Use resolver in more cases.

ognibuild/debian/apt.py

@@ -29,8 +29,6 @@ from debian.deb822 import Release
 from .. import DetailedFailure
 from ..session import Session, run_with_tee
 
-from .build import get_build_architecture
-
 
 class UnidentifiedError(Exception):
 
@@ -64,6 +62,12 @@ class AptManager(object):
     def __init__(self, session):
         self.session = session
 
+    def package_exists(self, package: str) -> bool:
+        raise NotImplementedError(self.package_exists)
+
+    def get_package_for_paths(self, paths, regex=False):
+        raise NotImplementedError(self.get_package_for_paths)
+
     def missing(self, packages):
         root = getattr(self.session, "location", "/")
         status_path = os.path.join(root, "var/lib/dpkg/status")
@@ -80,6 +84,7 @@ class AptManager(object):
         return list(missing)
 
     def install(self, packages: List[str]) -> None:
+        logging.info('Installing using apt: %r', packages)
         packages = self.missing(packages)
         if packages:
             run_apt(self.session, ["install"] + packages)
@@ -88,8 +93,30 @@ class AptManager(object):
         run_apt(self.session, ["satisfy"] + deps)
 
 
+class LocalAptManager(AptManager):
+
+    def __init__(self):
+        from ..session.plain import PlainSession
+        self.session = PlainSession()
+        self._apt_cache = None
+
+    def package_exists(self, package):
+        if self._apt_cache is None:
+            import apt_pkg
+
+            self._apt_cache = apt_pkg.Cache()
+        for p in self._apt_cache.packages:
+            if p.name == package:
+                return True
+        return False
+
+    def get_package_for_paths(self, paths, regex=False):
+        # TODO(jelmer): Make sure we use whatever is configured in self.session
+        return get_package_for_paths(paths, regex=regex)
+
+
 class FileSearcher(object):
-    def search_files(self, path, regex=False):
+    def search_files(self, path: str, regex: bool = False) -> Iterator[str]:
         raise NotImplementedError(self.search_files)
 
 
@@ -98,9 +125,6 @@ class ContentsFileNotFound(Exception):
 
 
 class AptContentsFileSearcher(FileSearcher):
-
-    _user_agent = 'ognibuild/0.1'
-
     def __init__(self):
         self._db = {}
 
@@ -137,6 +161,7 @@ class AptContentsFileSearcher(FileSearcher):
 
     @classmethod
     def from_repositories(cls, sources):
+        from .debian.build import get_build_architecture
         # TODO(jelmer): Verify signatures, etc.
         urls = []
         arches = [get_build_architecture(), "all"]
@@ -159,11 +184,11 @@ class AptContentsFileSearcher(FileSearcher):
                     urls.append("%s/%s/%s" % (base_url, name, entry["name"]))
         return cls.from_urls(urls)
 
-    @classmethod
+    @staticmethod
-    def _get(cls, url):
+    def _get(url):
         from urllib.request import urlopen, Request
 
-        request = Request(url, headers={"User-Agent": cls._user_agent})
+        request = Request(url, headers={"User-Agent": "Debian Janitor"})
         return urlopen(request)
 
     def load_url(self, url):
@@ -192,7 +217,7 @@ class GeneratedFileSearcher(FileSearcher):
     def __init__(self, db):
         self._db = db
 
-    def search_files(self, path, regex=False):
+    def search_files(self, path: str, regex: bool = False) -> Iterator[str]:
         for p, pkg in sorted(self._db.items()):
             if regex:
                 if re.match(path, p):
@@ -215,7 +240,7 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher(
 _apt_file_searcher = None
 
 
-def search_apt_file(path: str, regex: bool = False) -> Iterator[FileSearcher]:
+def search_apt_file(path: str, regex: bool = False) -> Iterator[str]:
     global _apt_file_searcher
     if _apt_file_searcher is None:
         # TODO(jelmer): cache file
@@ -223,3 +248,22 @@ def search_apt_file(path: str, regex: bool = False) -> Iterator[FileSearcher]:
     if _apt_file_searcher:
         yield from _apt_file_searcher.search_files(path, regex=regex)
     yield from GENERATED_FILE_SEARCHER.search_files(path, regex=regex)
+
+
+def get_package_for_paths(paths: List[str], regex: bool = False) -> Optional[str]:
+    candidates: Set[str] = set()
+    for path in paths:
+        candidates.update(search_apt_file(path, regex=regex))
+        if candidates:
+            break
+    if len(candidates) == 0:
+        logging.warning("No packages found that contain %r", paths)
+        return None
+    if len(candidates) > 1:
+        logging.warning(
+            "More than 1 packages found that contain %r: %r", path, candidates
+        )
+        # Euhr. Pick the one with the shortest name?
+        return sorted(candidates, key=len)[0]
+    else:
+        return candidates.pop()

ognibuild/debian/fix_build.py

@@ -21,7 +21,6 @@ __all__ = [
 
 import logging
 import os
-import subprocess
 import sys
 from typing import List, Callable, Type, Tuple, Set, Optional
 
@@ -106,9 +105,35 @@ from buildlog_consultant.sbuild import (
     SbuildFailure,
 )
 
-from ..apt import AptManager, LocalAptManager
+from .apt import AptManager, LocalAptManager
-from ..resolver.apt import AptResolver
+from ..resolver.apt import (
-from ..requirements import BinaryRequirement
+    AptResolver,
+    NoAptPackage,
+    get_package_for_python_module,
+    )
+from ..requirements import (
+    BinaryRequirement,
+    PathRequirement,
+    PkgConfigRequirement,
+    CHeaderRequirement,
+    JavaScriptRuntimeRequirement,
+    ValaPackageRequirement,
+    RubyGemRequirement,
+    GoPackageRequirement,
+    DhAddonRequirement,
+    PhpClassRequirement,
+    RPackageRequirement,
+    NodePackageRequirement,
+    LibraryRequirement,
+    RubyFileRequirement,
+    XmlEntityRequirement,
+    SprocketsFileRequirement,
+    JavaClassRequirement,
+    HaskellPackageRequirement,
+    MavenArtifactRequirement,
+    GnomeCommonRequirement,
+    JDKFileRequirement,
+    )
 from .build import attempt_build, DEFAULT_BUILDER
 
 
@@ -148,7 +173,7 @@ class DependencyContext(object):
 
 
 class BuildDependencyContext(DependencyContext):
-    def add_dependency(self, package, minimum_version=None):
+    def add_dependency(self, package: str, minimum_version: Optional[Version] = None):
         return add_build_dependency(
             self.tree,
             package,
@@ -181,12 +206,12 @@ class AutopkgtestDependencyContext(DependencyContext):
 
 
 def add_build_dependency(
-    tree,
+    tree: Tree,
-    package,
+    package: str,
-    minimum_version=None,
+    minimum_version: Optional[Version] = None,
-    committer=None,
+    committer: Optional[str] = None,
-    subpath="",
+    subpath: str = "",
-    update_changelog=True,
+    update_changelog: bool = True,
 ):
     if not isinstance(package, str):
         raise TypeError(package)
@@ -305,62 +330,6 @@ def commit_debian_changes(
             return True
 
 
-def get_package_for_python_module(apt, module, python_version):
-    if python_version == "python3":
-        paths = [
-            os.path.join(
-                "/usr/lib/python3/dist-packages",
-                module.replace(".", "/"),
-                "__init__.py",
-            ),
-            os.path.join(
-                "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py"
-            ),
-            os.path.join(
-                "/usr/lib/python3\\.[0-9]+/lib-dynload",
-                module.replace(".", "/") + "\\.cpython-.*\\.so",
-            ),
-            os.path.join(
-                "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py"
-            ),
-            os.path.join(
-                "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py"
-            ),
-        ]
-    elif python_version == "python2":
-        paths = [
-            os.path.join(
-                "/usr/lib/python2\\.[0-9]/dist-packages",
-                module.replace(".", "/"),
-                "__init__.py",
-            ),
-            os.path.join(
-                "/usr/lib/python2\\.[0-9]/dist-packages",
-                module.replace(".", "/") + ".py",
-            ),
-            os.path.join(
-                "/usr/lib/python2.\\.[0-9]/lib-dynload",
-                module.replace(".", "/") + ".so",
-            ),
-        ]
-    elif python_version == "pypy":
-        paths = [
-            os.path.join(
-                "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py"
-            ),
-            os.path.join(
-                "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py"
-            ),
-            os.path.join(
-                "/usr/lib/pypy/dist-packages",
-                module.replace(".", "/") + "\\.pypy-.*\\.so",
-            ),
-        ]
-    else:
-        raise AssertionError("unknown python version %r" % python_version)
-    return apt.get_package_for_paths(paths, regex=True)
-
-
 def targeted_python_versions(tree: Tree) -> Set[str]:
     with tree.get_file("debian/control") as f:
         control = Deb822(f)
@@ -378,13 +347,6 @@ def targeted_python_versions(tree: Tree) -> Set[str]:
     return targeted
 
 
-def fix_missing_javascript_runtime(error, context):
-    package = context.apt.get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False)
-    if package is None:
-        return False
-    return context.add_dependency(package)
-
-
 def fix_missing_python_distribution(error, context):  # noqa: C901
     targeted = targeted_python_versions(context.tree)
     default = not targeted
@@ -496,62 +458,60 @@ def fix_missing_python_module(error, context):
     return True
 
 
-def fix_missing_go_package(error, context):
+def fix_missing_requirement(error, context):
-    package = context.apt.get_package_for_paths(
+    if isinstance(error, MissingFile):
-        [os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True
+        req = PathRequirement(error.path)
-    )
+    elif isinstance(error, MissingCommand):
-    if package is None:
+        req = BinaryRequirement(error.command)
-        return False
+    elif isinstance(error, MissingPkgConfig):
-    return context.add_dependency(package)
+        req = PkgConfigRequirement(
-
+            error.module, error.minimum_version)
-
+    elif isinstance(error, MissingCHeader):
-def fix_missing_c_header(error, context):
+        req = CHeaderRequirement(error.header)
-    package = context.apt.get_package_for_paths(
+    elif isinstance(error, MissingJavaScriptRuntime):
-        [os.path.join("/usr/include", error.header)], regex=False
+        req = JavaScriptRuntimeRequirement()
-    )
+    elif isinstance(error, MissingRubyGem):
-    if package is None:
+        req = RubyGemRequirement(error.gem, error.version)
-        package = context.apt.get_package_for_paths(
+    elif isinstance(error, MissingValaPackage):
-            [os.path.join("/usr/include", ".*", error.header)], regex=True
+        req = ValaPackageRequirement(error.package)
-        )
+    elif isinstance(error, MissingGoPackage):
-    if package is None:
+        req = GoPackageRequirement(error.package)
-        return False
+    elif isinstance(error, DhAddonLoadFailure):
-    return context.add_dependency(package)
+        req = DhAddonRequirement(error.path)
-
+    elif isinstance(error, MissingPhpClass):
-
+        req = PhpClassRequirement(error.php_class)
-def fix_missing_pkg_config(error, context):
+    elif isinstance(error, MissingRPackage):
-    package = context.apt.get_package_for_paths(
+        req = RPackageRequirement(error.package, error.minimum_version)
-        [os.path.join("/usr/lib/pkgconfig", error.module + ".pc")]
+    elif isinstance(error, MissingNodeModule):
-    )
+        req = NodePackageRequirement(error.module)
-    if package is None:
+    elif isinstance(error, MissingLibrary):
-        package = context.apt.get_package_for_paths(
+        req = LibraryRequirement(error.library)
-            [os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")],
+    elif isinstance(error, MissingRubyFile):
-            regex=True,
+        req = RubyFileRequirement(error.filename)
-        )
+    elif isinstance(error, MissingXmlEntity):
-    if package is None:
+        req = XmlEntityRequirement(error.url)
-        return False
+    elif isinstance(error, MissingSprocketsFile):
-    return context.add_dependency(package, minimum_version=error.minimum_version)
+        req = SprocketsFileRequirement(error.content_type, error.name)
-
+    elif isinstance(error, MissingJavaClass):
-
+        req = JavaClassRequirement(error.classname)
-def fix_missing_command(error, context):
+    elif isinstance(error, MissingHaskellDependencies):
-    package = context.resolve_apt(BinaryRequirement(error.command))
+        # TODO(jelmer): Create multiple HaskellPackageRequirement objects?
-    return context.add_dependency(package)
+        req = HaskellPackageRequirement(error.package)
-
+    elif isinstance(error, MissingMavenArtifacts):
-
+        # TODO(jelmer): Create multiple MavenArtifactRequirement objects?
-def fix_missing_file(error, context):
+        req = MavenArtifactRequirement(error.artifacts)
-    package = context.apt.get_package_for_paths([error.path])
+    elif isinstance(error, MissingCSharpCompiler):
-    if package is None:
+        req = BinaryRequirement('msc')
-        return False
+    elif isinstance(error, GnomeCommonMissing):
-    return context.add_dependency(package)
+        req = GnomeCommonRequirement()
-
+    elif isinstance(error, MissingJDKFile):
-
+        req = JDKFileRequirement(error.jdk_path, error.filename)
-def fix_missing_sprockets_file(error, context):
-    if error.content_type == "application/javascript":
-        path = "/usr/share/.*/app/assets/javascripts/%s.js$" % error.name
     else:
-        logging.warning("unable to handle content type %s", error.content_type)
+        return None
-        return False
+
-    package = context.apt.get_package_for_paths([path], regex=True)
+    try:
-    if package is None:
+        package = context.resolve_apt(req)
+    except NoAptPackage:
         return False
     return context.add_dependency(package)
 
@@ -595,159 +555,10 @@ def fix_missing_perl_file(error, context):
     return context.add_dependency(package)
 
 
-def get_package_for_node_package(apt, node_package):
-    paths = [
-        "/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package,
-        "/usr/lib/nodejs/%s/package.json" % node_package,
-        "/usr/share/nodejs/%s/package.json" % node_package,
-    ]
-    return apt.get_package_for_paths(paths, regex=True)
-
-
-def fix_missing_node_module(error, context):
-    package = get_package_for_node_package(context.apt, error.module)
-    if package is None:
-        logging.warning("no node package found for %s.", error.module)
-        return False
-    return context.add_dependency(package)
-
-
-def fix_missing_dh_addon(error, context):
-    paths = [os.path.join("/usr/share/perl5", error.path)]
-    package = context.apt.get_package_for_paths(paths)
-    if package is None:
-        logging.warning("no package for debhelper addon %s", error.name)
-        return False
-    return context.add_dependency(package)
-
-
 def retry_apt_failure(error, context):
     return True
 
 
-def fix_missing_php_class(error, context):
-    path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/")
-    package = context.apt.get_package_for_paths([path])
-    if package is None:
-        logging.warning("no package for PHP class %s", error.php_class)
-        return False
-    return context.add_dependency(package)
-
-
-def fix_missing_jdk_file(error, context):
-    path = error.jdk_path + ".*/" + error.filename
-    package = context.apt.get_package_for_paths([path], regex=True)
-    if package is None:
-        logging.warning(
-            "no package found for %s (JDK: %s) - regex %s",
-            error.filename,
-            error.jdk_path,
-            path,
-        )
-        return False
-    return context.add_dependency(package)
-
-
-def fix_missing_vala_package(error, context):
-    path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package
-    package = context.apt.get_package_for_paths([path], regex=True)
-    if package is None:
-        logging.warning("no file found for package %s - regex %s", error.package, path)
-        return False
-    return context.add_dependency(package)
-
-
-def fix_missing_xml_entity(error, context):
-    # Ideally we should be using the XML catalog for this, but hardcoding
-    # a few URLs will do for now..
-    URL_MAP = {
-        "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/"
-    }
-    for url, path in URL_MAP.items():
-        if error.url.startswith(url):
-            search_path = os.path.join(path, error.url[len(url) :])
-            break
-    else:
-        return False
-
-    package = context.apt.get_package_for_paths([search_path], regex=False)
-    if package is None:
-        return False
-    return context.add_dependency(package)
-
-
-def fix_missing_library(error, context):
-    paths = [
-        os.path.join("/usr/lib/lib%s.so$" % error.library),
-        os.path.join("/usr/lib/.*/lib%s.so$" % error.library),
-        os.path.join("/usr/lib/lib%s.a$" % error.library),
-        os.path.join("/usr/lib/.*/lib%s.a$" % error.library),
-    ]
-    package = context.apt.get_package_for_paths(paths, regex=True)
-    if package is None:
-        logging.warning("no package for library %s", error.library)
-        return False
-    return context.add_dependency(package)
-
-
-def fix_missing_ruby_gem(error, context):
-    paths = [
-        os.path.join(
-            "/usr/share/rubygems-integration/all/"
-            "specifications/%s-.*\\.gemspec" % error.gem
-        )
-    ]
-    package = context.apt.get_package_for_paths(paths, regex=True)
-    if package is None:
-        logging.warning("no package for gem %s", error.gem)
-        return False
-    return context.add_dependency(package, minimum_version=error.version)
-
-
-def fix_missing_ruby_file(error, context):
-    paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)]
-    package = context.apt.get_package_for_paths(paths)
-    if package is not None:
-        return context.add_dependency(package)
-    paths = [
-        os.path.join(
-            r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
-            "lib/%s.rb" % error.filename
-        )
-    ]
-    package = context.apt.get_package_for_paths(paths, regex=True)
-    if package is not None:
-        return context.add_dependency(package)
-
-    logging.warning("no package for ruby file %s", error.filename)
-    return False
-
-
-def fix_missing_r_package(error, context):
-    paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)]
-    package = context.apt.get_package_for_paths(paths, regex=True)
-    if package is None:
-        logging.warning("no package for R package %s", error.package)
-        return False
-    return context.add_dependency(package, minimum_version=error.minimum_version)
-
-
-def fix_missing_java_class(error, context):
-    # Unfortunately this only finds classes in jars installed on the host
-    # system :(
-    output = subprocess.check_output(["java-propose-classpath", "-c" + error.classname])
-    classpath = [p for p in output.decode().strip(":").strip().split(":") if p]
-    if not classpath:
-        logging.warning("unable to find classpath for %s", error.classname)
-        return False
-    logging.info("Classpath for %s: %r", error.classname, classpath)
-    package = context.apt.get_package_for_paths(classpath)
-    if package is None:
-        logging.warning("no package for files in %r", classpath)
-        return False
-    return context.add_dependency(package)
-
-
 def enable_dh_autoreconf(context):
     # Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by
     # default.
@@ -768,9 +579,8 @@ def enable_dh_autoreconf(context):
 
 
 def fix_missing_configure(error, context):
-    if not context.tree.has_filename("configure.ac") and not context.tree.has_filename(
+    if (not context.tree.has_filename("configure.ac") and
-        "configure.in"
+            not context.tree.has_filename("configure.in")):
-    ):
         return False
 
     return enable_dh_autoreconf(context)
@@ -783,43 +593,6 @@ def fix_missing_automake_input(error, context):
     return enable_dh_autoreconf(context)
 
 
-def fix_missing_maven_artifacts(error, context):
-    artifact = error.artifacts[0]
-    parts = artifact.split(":")
-    if len(parts) == 4:
-        (group_id, artifact_id, kind, version) = parts
-        regex = False
-    elif len(parts) == 3:
-        (group_id, artifact_id, version) = parts
-        kind = "jar"
-        regex = False
-    elif len(parts) == 2:
-        version = ".*"
-        (group_id, artifact_id) = parts
-        kind = "jar"
-        regex = True
-    else:
-        raise AssertionError("invalid number of parts to artifact %s" % artifact)
-    paths = [
-        os.path.join(
-            "/usr/share/maven-repo",
-            group_id.replace(".", "/"),
-            artifact_id,
-            version,
-            "%s-%s.%s" % (artifact_id, version, kind),
-        )
-    ]
-    package = context.apt.get_package_for_paths(paths, regex=regex)
-    if package is None:
-        logging.warning("no package for artifact %s", artifact)
-        return False
-    return context.add_dependency(package)
-
-
-def install_gnome_common(error, context):
-    return context.add_dependency("gnome-common")
-
-
 def install_gnome_common_dep(error, context):
     if error.package == "glib-gettext":
         package = context.apt.get_package_for_paths(["/usr/bin/glib-gettextize"])
@@ -914,19 +687,6 @@ def fix_missing_autoconf_macro(error, context):
     return context.add_dependency(package)
 
 
-def fix_missing_c_sharp_compiler(error, context):
-    return context.add_dependency("mono-mcs")
-
-
-def fix_missing_haskell_dependencies(error, context):
-    path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0]
-    package = context.apt.get_package_for_paths([path], regex=True)
-    if package is None:
-        logging.warning("no package for macro file %s", path)
-        return False
-    return context.add_dependency(package)
-
-
 VERSIONED_PACKAGE_FIXERS: List[
     Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]
 ] = [
@@ -939,35 +699,14 @@ VERSIONED_PACKAGE_FIXERS: List[
 APT_FIXERS: List[Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [
     (MissingPythonModule, fix_missing_python_module),
     (MissingPythonDistribution, fix_missing_python_distribution),
-    (MissingCHeader, fix_missing_c_header),
-    (MissingPkgConfig, fix_missing_pkg_config),
-    (MissingCommand, fix_missing_command),
-    (MissingFile, fix_missing_file),
-    (MissingSprocketsFile, fix_missing_sprockets_file),
-    (MissingGoPackage, fix_missing_go_package),
     (MissingPerlFile, fix_missing_perl_file),
     (MissingPerlModule, fix_missing_perl_file),
-    (MissingXmlEntity, fix_missing_xml_entity),
-    (MissingNodeModule, fix_missing_node_module),
-    (MissingRubyGem, fix_missing_ruby_gem),
-    (MissingRPackage, fix_missing_r_package),
-    (MissingLibrary, fix_missing_library),
-    (MissingJavaClass, fix_missing_java_class),
-    (DhAddonLoadFailure, fix_missing_dh_addon),
-    (MissingPhpClass, fix_missing_php_class),
     (AptFetchFailure, retry_apt_failure),
-    (MissingMavenArtifacts, fix_missing_maven_artifacts),
-    (GnomeCommonMissing, install_gnome_common),
     (MissingGnomeCommonDependency, install_gnome_common_dep),
     (MissingXfceDependency, install_xfce_dep),
     (MissingConfigStatusInput, fix_missing_config_status_input),
-    (MissingJDKFile, fix_missing_jdk_file),
-    (MissingRubyFile, fix_missing_ruby_file),
-    (MissingJavaScriptRuntime, fix_missing_javascript_runtime),
     (MissingAutoconfMacro, fix_missing_autoconf_macro),
-    (MissingValaPackage, fix_missing_vala_package),
+    (Problem, fix_missing_requirement),
-    (MissingCSharpCompiler, fix_missing_c_sharp_compiler),
-    (MissingHaskellDependencies, fix_missing_haskell_dependencies),
 ]
 
 

ognibuild/requirements.py

@@ -16,6 +16,9 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
+import posixpath
+from typing import Optional, List, Tuple
+
 from . import UpstreamRequirement
 
 
@@ -62,3 +65,182 @@ class CargoCrateRequirement(UpstreamRequirement):
     def __init__(self, crate):
         super(CargoCrateRequirement, self).__init__('cargo-crate')
         self.crate = crate
+
+
+class PkgConfigRequirement(UpstreamRequirement):
+
+    module: str
+
+    def __init__(self, module, minimum_version=None):
+        super(PkgConfigRequirement, self).__init__('pkg-config')
+        self.module = module
+        self.minimum_version = minimum_version
+
+
+class PathRequirement(UpstreamRequirement):
+
+    path: str
+
+    def __init__(self, path):
+        super(PathRequirement, self).__init__('path')
+        self.path = path
+
+
+class CHeaderRequirement(UpstreamRequirement):
+
+    header: str
+
+    def __init__(self, header):
+        super(CHeaderRequirement, self).__init__('c-header')
+        self.header = header
+
+
+class JavaScriptRuntimeRequirement(UpstreamRequirement):
+
+    def __init__(self):
+        super(JavaScriptRuntimeRequirement, self).__init__(
+            'javascript-runtime')
+
+
+class ValaPackageRequirement(UpstreamRequirement):
+
+    package: str
+
+    def __init__(self, package: str):
+        super(ValaPackageRequirement, self).__init__('vala')
+        self.package = package
+
+
+class RubyGemRequirement(UpstreamRequirement):
+
+    gem: str
+    minimum_version: Optional[str]
+
+    def __init__(self, gem: str, minimum_version: Optional[str]):
+        super(RubyGemRequirement, self).__init__('gem')
+        self.gem = gem
+        self.minimum_version = minimum_version
+
+
+class GoPackageRequirement(UpstreamRequirement):
+
+    package: str
+
+    def __init__(self, package: str):
+        super(GoPackageRequirement, self).__init__('go')
+        self.package = package
+
+
+class DhAddonRequirement(UpstreamRequirement):
+
+    path: str
+
+    def __init__(self, path: str):
+        super(DhAddonRequirement, self).__init__('dh-addon')
+        self.path = path
+
+
+class PhpClassRequirement(UpstreamRequirement):
+
+    php_class: str
+
+    def __init__(self, php_class: str):
+        super(PhpClassRequirement, self).__init__('php-class')
+        self.php_class = php_class
+
+
+class RPackageRequirement(UpstreamRequirement):
+
+    package: str
+    minimum_version: Optional[str]
+
+    def __init__(self, package: str, minimum_version: Optional[str] = None):
+        super(RPackageRequirement, self).__init__('r-package')
+        self.package = package
+        self.minimum_version = minimum_version
+
+
+class LibraryRequirement(UpstreamRequirement):
+
+    library: str
+
+    def __init__(self, library: str):
+        super(LibraryRequirement, self).__init__('lib')
+        self.library = library
+
+
+class RubyFileRequirement(UpstreamRequirement):
+
+    filename: str
+
+    def __init__(self, filename: str):
+        super(RubyFileRequirement, self).__init__('ruby-file')
+        self.filename = filename
+
+
+class XmlEntityRequirement(UpstreamRequirement):
+
+    url: str
+
+    def __init__(self, url: str):
+        super(XmlEntityRequirement, self).__init__('xml-entity')
+        self.url = url
+
+
+class SprocketsFileRequirement(UpstreamRequirement):
+
+    content_type: str
+    name: str
+
+    def __init__(self, content_type: str, name: str):
+        super(SprocketsFileRequirement, self).__init__('sprockets-file')
+        self.content_type = content_type
+        self.name = name
+
+
+class JavaClassRequirement(UpstreamRequirement):
+
+    classname: str
+
+    def __init__(self, classname: str):
+        super(JavaClassRequirement, self).__init__('java-class')
+        self.classname = classname
+
+
+class HaskellPackageRequirement(UpstreamRequirement):
+
+    package: str
+
+    def __init__(self, package: str):
+        super(HaskellPackageRequirement, self).__init__('haskell-package')
+        self.package = package
+
+
+class MavenArtifactRequirement(UpstreamRequirement):
+
+    artifacts: List[Tuple[str, str, str]]
+
+    def __init__(self, artifacts):
+        super(MavenArtifactRequirement, self).__init__('maven-artifact')
+        self.artifacts = artifacts
+
+
+class GnomeCommonRequirement(UpstreamRequirement):
+
+    def __init__(self):
+        super(GnomeCommonRequirement, self).__init__('gnome-common')
+
+
+class JDKFileRequirement(UpstreamRequirement):
+
+    jdk_path: str
+    filename: str
+
+    def __init__(self, jdk_path: str, filename: str):
+        super(JDKFileRequirement, self).__init__('jdk-file')
+        self.jdk_path = jdk_path
+        self.filename = filename
+
+    @property
+    def path(self):
+        return posixpath.join(self.jdk_path, self.filename)

ognibuild/resolver/apt.py

@@ -15,13 +15,35 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
+import logging
 import posixpath
 
-from ..apt import AptManager
+from ..debian.apt import AptManager
 
 from . import Resolver
 from ..requirements import (
     BinaryRequirement,
+    CHeaderRequirement,
+    PkgConfigRequirement,
+    PathRequirement,
+    UpstreamRequirement,
+    JavaScriptRuntimeRequirement,
+    ValaPackageRequirement,
+    RubyGemRequirement,
+    GoPackageRequirement,
+    DhAddonRequirement,
+    PhpClassRequirement,
+    RPackageRequirement,
+    NodePackageRequirement,
+    LibraryRequirement,
+    RubyFileRequirement,
+    XmlEntityRequirement,
+    SprocketsFileRequirement,
+    JavaClassRequirement,
+    HaskellPackageRequirement,
+    MavenArtifactRequirement,
+    GnomeCommonRequirement,
+    JDKFileRequirement,
     )
 
 
@@ -29,6 +51,62 @@ class NoAptPackage(Exception):
     """No apt package."""
 
 
+def get_package_for_python_module(apt_mgr, module, python_version):
+    if python_version == "python3":
+        paths = [
+            posixpath.join(
+                "/usr/lib/python3/dist-packages",
+                module.replace(".", "/"),
+                "__init__.py",
+            ),
+            posixpath.join(
+                "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py"
+            ),
+            posixpath.join(
+                "/usr/lib/python3\\.[0-9]+/lib-dynload",
+                module.replace(".", "/") + "\\.cpython-.*\\.so",
+            ),
+            posixpath.join(
+                "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py"
+            ),
+            posixpath.join(
+                "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py"
+            ),
+        ]
+    elif python_version == "python2":
+        paths = [
+            posixpath.join(
+                "/usr/lib/python2\\.[0-9]/dist-packages",
+                module.replace(".", "/"),
+                "__init__.py",
+            ),
+            posixpath.join(
+                "/usr/lib/python2\\.[0-9]/dist-packages",
+                module.replace(".", "/") + ".py",
+            ),
+            posixpath.join(
+                "/usr/lib/python2.\\.[0-9]/lib-dynload",
+                module.replace(".", "/") + ".so",
+            ),
+        ]
+    elif python_version == "pypy":
+        paths = [
+            posixpath.join(
+                "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py"
+            ),
+            posixpath.join(
+                "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py"
+            ),
+            posixpath.join(
+                "/usr/lib/pypy/dist-packages",
+                module.replace(".", "/") + "\\.pypy-.*\\.so",
+            ),
+        ]
+    else:
+        raise AssertionError("unknown python version %r" % python_version)
+    return apt_mgr.get_package_for_paths(paths, regex=True)
+
+
 def resolve_binary_req(apt_mgr, req):
     if posixpath.isabs(req.binary_name):
         paths = [req.binary_name]
@@ -40,8 +118,218 @@ def resolve_binary_req(apt_mgr, req):
     return apt_mgr.get_package_for_paths(paths)
 
 
+def resolve_pkg_config_req(apt_mgr, req):
+    package = apt_mgr.get_package_for_paths(
+        [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")],
+        req.minimum_version
+    )
+    if package is None:
+        package = apt_mgr.get_package_for_paths(
+            [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")],
+            regex=True,
+            minimum_version=req.minimum_version)
+    return package
+
+
+def resolve_path_req(apt_mgr, req):
+    return apt_mgr.get_package_for_paths([req.path])
+
+
+def resolve_c_header_req(apt_mgr, req):
+    package = apt_mgr.get_package_for_paths(
+        [posixpath.join("/usr/include", req.header)], regex=False
+    )
+    if package is None:
+        package = apt_mgr.get_package_for_paths(
+            [posixpath.join("/usr/include", ".*", req.header)], regex=True
+        )
+    return package
+
+
+def resolve_js_runtime_req(apt_mgr, req):
+    return apt_mgr.get_package_for_paths(
+        ["/usr/bin/node", "/usr/bin/duk"], regex=False)
+
+
+def resolve_vala_package_req(apt_mgr, req):
+    path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package
+    return apt_mgr.get_package_for_paths([path], regex=True)
+
+
+def resolve_ruby_gem_req(apt_mgr, req):
+    paths = [
+        posixpath.join(
+            "/usr/share/rubygems-integration/all/"
+            "specifications/%s-.*\\.gemspec" % req.gem
+        )
+    ]
+    return apt_mgr.get_package_for_paths(
+        paths, regex=True, minimum_version=req.minimum_version)
+
+
+def resolve_go_package_req(apt_mgr, req):
+    return apt_mgr.get_package_for_paths(
+        [posixpath.join("/usr/share/gocode/src", req.package, ".*")],
+        regex=True
+    )
+
+
+def resolve_dh_addon_req(apt_mgr, req):
+    paths = [posixpath.join("/usr/share/perl5", req.path)]
+    return apt_mgr.get_package_for_paths(paths)
+
+
+def resolve_php_class_req(apt_mgr, req):
+    path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/")
+    return apt_mgr.get_package_for_paths([path])
+
+
+def resolve_r_package_req(apt_mgr, req):
+    paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)]
+    return apt_mgr.get_package_for_paths(paths, regex=True)
+
+
+def resolve_node_package_req(apt_mgr, req):
+    paths = [
+        "/usr/share/nodejs/.*/node_modules/%s/package.json" % req.package,
+        "/usr/lib/nodejs/%s/package.json" % req.package,
+        "/usr/share/nodejs/%s/package.json" % req.package,
+    ]
+    return apt_mgr.get_package_for_paths(paths, regex=True)
+
+
+def resolve_library_req(apt_mgr, req):
+    paths = [
+        posixpath.join("/usr/lib/lib%s.so$" % req.library),
+        posixpath.join("/usr/lib/.*/lib%s.so$" % req.library),
+        posixpath.join("/usr/lib/lib%s.a$" % req.library),
+        posixpath.join("/usr/lib/.*/lib%s.a$" % req.library),
+    ]
+    return apt_mgr.get_package_for_paths(paths, regex=True)
+
+
+def resolve_ruby_file_req(apt_mgr, req):
+    paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)]
+    package = apt_mgr.get_package_for_paths(paths)
+    if package is not None:
+        return package
+    paths = [
+        posixpath.join(
+            r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
+            "lib/%s.rb" % req.filename
+        )
+    ]
+    return apt_mgr.get_package_for_paths(paths, regex=True)
+
+
+def resolve_xml_entity_req(apt_mgr, req):
+    # Ideally we should be using the XML catalog for this, but hardcoding
+    # a few URLs will do for now..
+    URL_MAP = {
+        "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/"
+    }
+    for url, path in URL_MAP.items():
+        if req.url.startswith(url):
+            search_path = posixpath.join(path, req.url[len(url) :])
+            break
+    else:
+        return None
+
+    return apt_mgr.get_package_for_paths([search_path], regex=False)
+
+
+def resolve_sprockets_file_req(apt_mgr, req):
+    if req.content_type == "application/javascript":
+        path = "/usr/share/.*/app/assets/javascripts/%s.js$" % req.name
+    else:
+        logging.warning("unable to handle content type %s", req.content_type)
+        return None
+    return apt_mgr.get_package_for_paths([path], regex=True)
+
+
+def resolve_java_class_req(apt_mgr, req):
+    # Unfortunately this only finds classes in jars installed on the host
+    # system :(
+    # TODO(jelmer): Call in session
+    output = apt_mgr.session.check_output(
+        ["java-propose-classpath", "-c" + req.classname])
+    classpath = [p for p in output.decode().strip(":").strip().split(":") if p]
+    if not classpath:
+        logging.warning("unable to find classpath for %s", req.classname)
+        return False
+    logging.info("Classpath for %s: %r", req.classname, classpath)
+    package = apt_mgr.get_package_for_paths(classpath)
+    if package is None:
+        logging.warning("no package for files in %r", classpath)
+        return None
+    return package
+
+
+def resolve_haskell_package_req(apt_mgr, req):
+    path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0]
+    return apt_mgr.get_package_for_paths([path], regex=True)
+
+
+def resolve_maven_artifact_req(apt_mgr, req):
+    artifact = req.artifacts[0]
+    parts = artifact.split(":")
+    if len(parts) == 4:
+        (group_id, artifact_id, kind, version) = parts
+        regex = False
+    elif len(parts) == 3:
+        (group_id, artifact_id, version) = parts
+        kind = "jar"
+        regex = False
+    elif len(parts) == 2:
+        version = ".*"
+        (group_id, artifact_id) = parts
+        kind = "jar"
+        regex = True
+    else:
+        raise AssertionError("invalid number of parts to artifact %s" % artifact)
+    paths = [
+        posixpath.join(
+            "/usr/share/maven-repo",
+            group_id.replace(".", "/"),
+            artifact_id,
+            version,
+            "%s-%s.%s" % (artifact_id, version, kind),
+        )
+    ]
+    return apt_mgr.get_package_for_paths(paths, regex=regex)
+
+
+def resolve_gnome_common_req(apt_mgr, req):
+    return 'gnome-common'
+
+
+def resolve_jdk_file_req(apt_mgr, req):
+    path = req.jdk_path + ".*/" + req.filename
+    return apt_mgr.get_package_for_paths([path], regex=True)
+
+
 APT_REQUIREMENT_RESOLVERS = [
     (BinaryRequirement, resolve_binary_req),
+    (PkgConfigRequirement, resolve_pkg_config_req),
+    (PathRequirement, resolve_path_req),
+    (CHeaderRequirement, resolve_c_header_req),
+    (JavaScriptRuntimeRequirement, resolve_js_runtime_req),
+    (ValaPackageRequirement, resolve_vala_package_req),
+    (RubyGemRequirement, resolve_ruby_gem_req),
+    (GoPackageRequirement, resolve_go_package_req),
+    (DhAddonRequirement, resolve_dh_addon_req),
+    (PhpClassRequirement, resolve_php_class_req),
+    (RPackageRequirement, resolve_r_package_req),
+    (NodePackageRequirement, resolve_node_package_req),
+    (LibraryRequirement, resolve_library_req),
+    (RubyFileRequirement, resolve_ruby_file_req),
+    (XmlEntityRequirement, resolve_xml_entity_req),
+    (SprocketsFileRequirement, resolve_sprockets_file_req),
+    (JavaClassRequirement, resolve_java_class_req),
+    (HaskellPackageRequirement, resolve_haskell_package_req),
+    (MavenArtifactRequirement, resolve_maven_artifact_req),
+    (GnomeCommonRequirement, resolve_gnome_common_req),
+    (JDKFileRequirement, resolve_jdk_file_req),
 ]
 
 
@@ -70,14 +358,12 @@ class AptResolver(Resolver):
     def explain(self, requirements):
         raise NotImplementedError(self.explain)
 
-    def resolve(self, requirements):
+    def resolve(self, req: UpstreamRequirement):
-        for req in requirements:
+        for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
-            for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
+            if isinstance(req, rr_class):
-                if isinstance(req, rr_class):
+                deb_req = rr_fn(self.apt, req)
-                    package_name = rr_fn(self.apt, req)
+                if deb_req is None:
-                    if package_name is None:
+                    raise NoAptPackage()
-                        raise NoAptPackage()
+                return deb_req
-                    yield package_name
+        else:
-                    break
+            raise NotImplementedError
-            else:
-                raise NotImplementedError