More work on resolvers.

ognibuild/__init__.py

@@ -18,7 +18,6 @@
 
 import os
 import stat
-import sys
 
 
 class DetailedFailure(Exception):
@@ -44,9 +43,11 @@ def shebang_binary(p):
 
 class UpstreamRequirement(object):
 
-    def __init__(self, family, name):
+    # Name of the family of requirements - e.g. "python-package"
+    family: str
+
+    def __init__(self, family):
         self.family = family
-        self.name = name
 
 
 class UpstreamOutput(object):

ognibuild/__main__.py

@@ -25,12 +25,12 @@ from .clean import run_clean
 from .dist import run_dist
 from .install import run_install
 from .resolver import (
-    AptResolver,
     ExplainResolver,
     AutoResolver,
     NativeResolver,
     MissingDependencies,
 )
+from .resolver.apt import AptResolver
 from .test import run_test
 
 
@@ -84,6 +84,7 @@ def main():  # noqa: C901
         help="Ignore declared dependencies, follow build errors only",
     )
     args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO)
     if args.schroot:
         from .session.schroot import SchrootSession
 

ognibuild/buildsystem.py

@@ -22,7 +22,14 @@ import os
 import re
 import warnings
 
-from . import shebang_binary, UpstreamRequirement, UpstreamOutput
+from . import shebang_binary, UpstreamOutput
+from .requirements import (
+    BinaryRequirement,
+    PythonPackageRequirement,
+    PerlModuleRequirement,
+    NodePackageRequirement,
+    CargoCrateRequirement,
+    )
 from .apt import UnidentifiedError
 from .fix_build import run_with_build_fixer
 
@@ -66,7 +73,7 @@ class Pear(BuildSystem):
         self.path = path
 
     def setup(self, resolver):
-        resolver.install([UpstreamRequirement("binary", "pear")])
+        resolver.install([BinaryRequirement("pear")])
 
     def dist(self, session, resolver):
         self.setup(resolver)
@@ -94,18 +101,13 @@ class SetupPy(BuildSystem):
     name = "setup.py"
 
     def __init__(self, path):
+        self.path = path
         from distutils.core import run_setup
-
         self.result = run_setup(os.path.abspath(path), stop_after="init")
 
     def setup(self, resolver):
-        resolver.install(
+        resolver.install([PythonPackageRequirement('pip')])
-            [
+        with open(self.path, "r") as f:
-                UpstreamRequirement("python3", "pip"),
-                UpstreamRequirement("binary", "python3"),
-            ]
-        )
-        with open("setup.py", "r") as f:
             setup_py_contents = f.read()
         try:
             with open("setup.cfg", "r") as f:
@@ -114,7 +116,7 @@ class SetupPy(BuildSystem):
             setup_cfg_contents = ""
         if "setuptools" in setup_py_contents:
             logging.info("Reference to setuptools found, installing.")
-            resolver.install([UpstreamRequirement("python3", "setuptools")])
+            resolver.install([PythonPackageRequirement("setuptools")])
         if (
             "setuptools_scm" in setup_py_contents
             or "setuptools_scm" in setup_cfg_contents
@@ -122,9 +124,9 @@ class SetupPy(BuildSystem):
             logging.info("Reference to setuptools-scm found, installing.")
             resolver.install(
                 [
-                    UpstreamRequirement("python3", "setuptools-scm"),
+                    PythonPackageRequirement("setuptools-scm"),
-                    UpstreamRequirement("binary", "git"),
+                    BinaryRequirement("git"),
-                    UpstreamRequirement("binary", "mercurial"),
+                    BinaryRequirement("mercurial"),
                 ]
             )
 
@@ -150,24 +152,24 @@ class SetupPy(BuildSystem):
         interpreter = shebang_binary("setup.py")
         if interpreter is not None:
             if interpreter in ("python3", "python2", "python"):
-                resolver.install([UpstreamRequirement("binary", interpreter)])
+                resolver.install([BinaryRequirement(interpreter)])
             else:
                 raise ValueError("Unknown interpreter %r" % interpreter)
             run_with_build_fixer(session, ["./setup.py"] + args)
         else:
             # Just assume it's Python 3
-            resolver.install([UpstreamRequirement("binary", "python3")])
+            resolver.install([BinaryRequirement("python3")])
             run_with_build_fixer(session, ["python3", "./setup.py"] + args)
 
     def get_declared_dependencies(self):
         for require in self.result.get_requires():
-            yield "build", UpstreamRequirement("python3", require)
+            yield "build", PythonPackageRequirement(require)
         if self.result.install_requires:
             for require in self.result.install_requires:
-                yield "install", UpstreamRequirement("python3", require)
+                yield "install", PythonPackageRequirement(require)
         if self.result.tests_require:
             for require in self.result.tests_require:
-                yield "test", UpstreamRequirement("python3", require)
+                yield "test", PythonPackageRequirement(require)
 
     def get_declared_outputs(self):
         for script in self.result.scripts or []:
@@ -200,8 +202,8 @@ class PyProject(BuildSystem):
             )
             resolver.install(
                 [
-                    UpstreamRequirement("python3", "venv"),
+                    PythonPackageRequirement("venv"),
-                    UpstreamRequirement("python3", "pip"),
+                    PythonPackageRequirement("pip"),
                 ]
             )
             session.check_call(["pip3", "install", "poetry"], user="root")
@@ -220,8 +222,8 @@ class SetupCfg(BuildSystem):
     def setup(self, resolver):
         resolver.install(
             [
-                UpstreamRequirement("python3", "pep517"),
+                PythonPackageRequirement("pep517"),
-                UpstreamRequirement("python3", "pip"),
+                PythonPackageRequirement("pip"),
             ]
         )
 
@@ -244,10 +246,10 @@ class Npm(BuildSystem):
         if "devDependencies" in self.package:
             for name, unused_version in self.package["devDependencies"].items():
                 # TODO(jelmer): Look at version
-                yield "dev", UpstreamRequirement("npm", name)
+                yield "dev", NodePackageRequirement(name)
 
     def setup(self, resolver):
-        resolver.install([UpstreamRequirement("binary", "npm")])
+        resolver.install([BinaryRequirement("npm")])
 
     def dist(self, session, resolver):
         self.setup(resolver)
@@ -262,7 +264,7 @@ class Waf(BuildSystem):
         self.path = path
 
     def setup(self, resolver):
-        resolver.install([UpstreamRequirement("binary", "python3")])
+        resolver.install([BinaryRequirement("python3")])
 
     def dist(self, session, resolver):
         self.setup(resolver)
@@ -277,7 +279,7 @@ class Gem(BuildSystem):
         self.path = path
 
     def setup(self, resolver):
-        resolver.install([UpstreamRequirement("binary", "gem2deb")])
+        resolver.install([BinaryRequirement("gem2deb")])
 
     def dist(self, session, resolver):
         self.setup(resolver)
@@ -314,18 +316,18 @@ class DistInkt(BuildSystem):
     def setup(self, resolver):
         resolver.install(
             [
-                UpstreamRequirement("perl", "Dist::Inkt"),
+                PerlModuleRequirement("Dist::Inkt"),
             ]
         )
 
     def dist(self, session, resolver):
         self.setup(resolver)
         if self.name == "dist-inkt":
-            resolver.install([UpstreamRequirement("perl-module", self.dist_inkt_class)])
+            resolver.install([PerlModuleRequirement(self.dist_inkt_class)])
             run_with_build_fixer(session, ["distinkt-dist"])
         else:
             # Default to invoking Dist::Zilla
-            resolver.install([UpstreamRequirement("perl", "Dist::Zilla")])
+            resolver.install([PerlModuleRequirement("Dist::Zilla")])
             run_with_build_fixer(session, ["dzil", "build", "--in", ".."])
 
 
@@ -335,7 +337,7 @@ class Make(BuildSystem):
 
     def setup(self, session, resolver):
         if session.exists("Makefile.PL") and not session.exists("Makefile"):
-            resolver.install([UpstreamRequirement("binary", "perl")])
+            resolver.install([BinaryRequirement("perl")])
             run_with_build_fixer(session, ["perl", "Makefile.PL"])
 
         if not session.exists("Makefile") and not session.exists("configure"):
@@ -357,10 +359,10 @@ class Make(BuildSystem):
             elif session.exists("configure.ac") or session.exists("configure.in"):
                 resolver.install(
                     [
-                        UpstreamRequirement("binary", "autoconf"),
+                        BinaryRequirement("autoconf"),
-                        UpstreamRequirement("binary", "automake"),
+                        BinaryRequirement("automake"),
-                        UpstreamRequirement("binary", "gettextize"),
+                        BinaryRequirement("gettextize"),
-                        UpstreamRequirement("binary", "libtoolize"),
+                        BinaryRequirement("libtoolize"),
                     ]
                 )
                 run_with_build_fixer(session, ["autoreconf", "-i"])
@@ -370,7 +372,7 @@ class Make(BuildSystem):
 
     def dist(self, session, resolver):
         self.setup(session, resolver)
-        resolver.install([UpstreamRequirement("binary", "make")])
+        resolver.install([BinaryRequirement("make")])
         try:
             run_with_build_fixer(session, ["make", "dist"])
         except UnidentifiedError as e:
@@ -437,7 +439,7 @@ class Make(BuildSystem):
                     warnings.warn("Unable to parse META.yml: %s" % e)
                     return
                 for require in data.get("requires", []):
-                    yield "build", UpstreamRequirement("perl", require)
+                    yield "build", PerlModuleRequirement(require)
 
 
 class Cargo(BuildSystem):
@@ -454,7 +456,7 @@ class Cargo(BuildSystem):
         if "dependencies" in self.cargo:
             for name, details in self.cargo["dependencies"].items():
                 # TODO(jelmer): Look at details['features'], details['version']
-                yield "build", UpstreamRequirement("cargo-crate", name)
+                yield "build", CargoCrateRequirement(name)
 
 
 class Golang(BuildSystem):

ognibuild/debian/fix_build.py

@@ -21,15 +21,13 @@ __all__ = [
 
 import logging
 import os
-import re
 import subprocess
 import sys
-from typing import Iterator, List, Callable, Type, Tuple, Set, Optional
+from typing import List, Callable, Type, Tuple, Set, Optional
 
 from debian.deb822 import (
     Deb822,
     PkgRelation,
-    Release,
 )
 from debian.changelog import Version
 
@@ -113,6 +111,11 @@ from buildlog_consultant.sbuild import (
     SbuildFailure,
 )
 
+from ..apt import AptManager, LocalAptManager
+from ..resolver.apt import AptResolver
+from ..requirements import BinaryRequirement
+from .build import attempt_build
+
 
 DEFAULT_MAX_ITERATIONS = 10
 
@@ -128,15 +131,21 @@ class DependencyContext(object):
     def __init__(
         self,
         tree: MutableTree,
+        apt: AptManager,
         subpath: str = "",
         committer: Optional[str] = None,
         update_changelog: bool = True,
     ):
         self.tree = tree
+        self.apt = apt
+        self.resolver = AptResolver(apt)
         self.subpath = subpath
         self.committer = committer
         self.update_changelog = update_changelog
 
+    def resolve_apt(self, req):
+        return self.resolver.resolve(req)
+
     def add_dependency(
         self, package: str, minimum_version: Optional[Version] = None
     ) -> bool:
@@ -157,11 +166,11 @@ class BuildDependencyContext(DependencyContext):
 
 class AutopkgtestDependencyContext(DependencyContext):
     def __init__(
-        self, testname, tree, subpath="", committer=None, update_changelog=True
+        self, testname, tree, apt, subpath="", committer=None, update_changelog=True
     ):
         self.testname = testname
         super(AutopkgtestDependencyContext, self).__init__(
-            tree, subpath, committer, update_changelog
+            tree, apt, subpath, committer, update_changelog
         )
 
     def add_dependency(self, package, minimum_version=None):
@@ -301,27 +310,7 @@ def commit_debian_changes(
             return True
 
 
-def get_package_for_paths(paths, regex=False):
+def get_package_for_python_module(apt, module, python_version):
-    from .apt import search_apt_file
-    candidates = set()
-    for path in paths:
-        candidates.update(search_apt_file(path, regex=regex))
-        if candidates:
-            break
-    if len(candidates) == 0:
-        logging.warning("No packages found that contain %r", paths)
-        return None
-    if len(candidates) > 1:
-        logging.warning(
-            "More than 1 packages found that contain %r: %r", path, candidates
-        )
-        # Euhr. Pick the one with the shortest name?
-        return sorted(candidates, key=len)[0]
-    else:
-        return candidates.pop()
-
-
-def get_package_for_python_module(module, python_version):
     if python_version == "python3":
         paths = [
             os.path.join(
@@ -374,7 +363,7 @@ def get_package_for_python_module(module, python_version):
         ]
     else:
         raise AssertionError("unknown python version %r" % python_version)
-    return get_package_for_paths(paths, regex=True)
+    return apt.get_package_for_paths(paths, regex=True)
 
 
 def targeted_python_versions(tree: Tree) -> Set[str]:
@@ -394,23 +383,8 @@ def targeted_python_versions(tree: Tree) -> Set[str]:
     return targeted
 
 
-apt_cache = None
-
-
-def package_exists(package):
-    global apt_cache
-    if apt_cache is None:
-        import apt_pkg
-
-        apt_cache = apt_pkg.Cache()
-    for p in apt_cache.packages:
-        if p.name == package:
-            return True
-    return False
-
-
 def fix_missing_javascript_runtime(error, context):
-    package = get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False)
+    package = context.apt.get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False)
     if package is None:
         return False
     return context.add_dependency(package)
@@ -420,30 +394,30 @@ def fix_missing_python_distribution(error, context):  # noqa: C901
     targeted = targeted_python_versions(context.tree)
     default = not targeted
 
-    pypy_pkg = get_package_for_paths(
+    pypy_pkg = context.apt.get_package_for_paths(
         ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True
     )
     if pypy_pkg is None:
         pypy_pkg = "pypy-%s" % error.distribution
-        if not package_exists(pypy_pkg):
+        if not context.apt.package_exists(pypy_pkg):
             pypy_pkg = None
 
-    py2_pkg = get_package_for_paths(
+    py2_pkg = context.apt.get_package_for_paths(
         ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution],
         regex=True,
     )
     if py2_pkg is None:
         py2_pkg = "python-%s" % error.distribution
-        if not package_exists(py2_pkg):
+        if not context.apt.package_exists(py2_pkg):
             py2_pkg = None
 
-    py3_pkg = get_package_for_paths(
+    py3_pkg = context.apt.get_package_for_paths(
         ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution],
         regex=True,
     )
     if py3_pkg is None:
         py3_pkg = "python3-%s" % error.distribution
-        if not package_exists(py3_pkg):
+        if not context.apt.package_exists(py3_pkg):
             py3_pkg = None
 
     extra_build_deps = []
@@ -488,9 +462,9 @@ def fix_missing_python_module(error, context):
         targeted = set()
     default = not targeted
 
-    pypy_pkg = get_package_for_python_module(error.module, "pypy")
+    pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy")
-    py2_pkg = get_package_for_python_module(error.module, "python2")
+    py2_pkg = get_package_for_python_module(context.apt, error.module, "python2")
-    py3_pkg = get_package_for_python_module(error.module, "python3")
+    py3_pkg = get_package_for_python_module(context.apt, error.module, "python3")
 
     extra_build_deps = []
     if error.python_version == 2:
@@ -528,7 +502,7 @@ def fix_missing_python_module(error, context):
 
 
 def fix_missing_go_package(error, context):
-    package = get_package_for_paths(
+    package = context.apt.get_package_for_paths(
         [os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True
     )
     if package is None:
@@ -537,11 +511,11 @@ def fix_missing_go_package(error, context):
 
 
 def fix_missing_c_header(error, context):
-    package = get_package_for_paths(
+    package = context.apt.get_package_for_paths(
         [os.path.join("/usr/include", error.header)], regex=False
     )
     if package is None:
-        package = get_package_for_paths(
+        package = context.apt.get_package_for_paths(
             [os.path.join("/usr/include", ".*", error.header)], regex=True
         )
     if package is None:
@@ -550,11 +524,11 @@ def fix_missing_c_header(error, context):
 
 
 def fix_missing_pkg_config(error, context):
-    package = get_package_for_paths(
+    package = context.apt.get_package_for_paths(
         [os.path.join("/usr/lib/pkgconfig", error.module + ".pc")]
     )
     if package is None:
-        package = get_package_for_paths(
+        package = context.apt.get_package_for_paths(
             [os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")],
             regex=True,
         )
@@ -564,21 +538,12 @@ def fix_missing_pkg_config(error, context):
 
 
 def fix_missing_command(error, context):
-    if os.path.isabs(error.command):
+    package = context.resolve_apt(BinaryRequirement(error.command))
-        paths = [error.command]
-    else:
-        paths = [
-            os.path.join(dirname, error.command) for dirname in ["/usr/bin", "/bin"]
-        ]
-    package = get_package_for_paths(paths)
-    if package is None:
-        logging.info("No packages found that contain %r", paths)
-        return False
     return context.add_dependency(package)
 
 
 def fix_missing_file(error, context):
-    package = get_package_for_paths([error.path])
+    package = context.apt.get_package_for_paths([error.path])
     if package is None:
         return False
     return context.add_dependency(package)
@@ -590,7 +555,7 @@ def fix_missing_sprockets_file(error, context):
     else:
         logging.warning("unable to handle content type %s", error.content_type)
         return False
-    package = get_package_for_paths([path], regex=True)
+    package = context.apt.get_package_for_paths([path], regex=True)
     if package is None:
         return False
     return context.add_dependency(package)
@@ -619,7 +584,7 @@ def fix_missing_perl_file(error, context):
             paths = [error.filename]
     else:
         paths = [os.path.join(inc, error.filename) for inc in error.inc]
-    package = get_package_for_paths(paths, regex=False)
+    package = context.apt.get_package_for_paths(paths, regex=False)
     if package is None:
         if getattr(error, "module", None):
             logging.warning(
@@ -635,17 +600,17 @@ def fix_missing_perl_file(error, context):
     return context.add_dependency(package)
 
 
-def get_package_for_node_package(node_package):
+def get_package_for_node_package(apt, node_package):
     paths = [
         "/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package,
         "/usr/lib/nodejs/%s/package.json" % node_package,
         "/usr/share/nodejs/%s/package.json" % node_package,
     ]
-    return get_package_for_paths(paths, regex=True)
+    return apt.get_package_for_paths(paths, regex=True)
 
 
 def fix_missing_node_module(error, context):
-    package = get_package_for_node_package(error.module)
+    package = get_package_for_node_package(context.apt, error.module)
     if package is None:
         logging.warning("no node package found for %s.", error.module)
         return False
@@ -654,7 +619,7 @@ def fix_missing_node_module(error, context):
 
 def fix_missing_dh_addon(error, context):
     paths = [os.path.join("/usr/share/perl5", error.path)]
-    package = get_package_for_paths(paths)
+    package = context.apt.get_package_for_paths(paths)
     if package is None:
         logging.warning("no package for debhelper addon %s", error.name)
         return False
@@ -667,7 +632,7 @@ def retry_apt_failure(error, context):
 
 def fix_missing_php_class(error, context):
     path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/")
-    package = get_package_for_paths([path])
+    package = context.apt.get_package_for_paths([path])
     if package is None:
         logging.warning("no package for PHP class %s", error.php_class)
         return False
@@ -676,7 +641,7 @@ def fix_missing_php_class(error, context):
 
 def fix_missing_jdk_file(error, context):
     path = error.jdk_path + ".*/" + error.filename
-    package = get_package_for_paths([path], regex=True)
+    package = context.apt.get_package_for_paths([path], regex=True)
     if package is None:
         logging.warning(
             "no package found for %s (JDK: %s) - regex %s",
@@ -690,7 +655,7 @@ def fix_missing_jdk_file(error, context):
 
 def fix_missing_vala_package(error, context):
     path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package
-    package = get_package_for_paths([path], regex=True)
+    package = context.apt.get_package_for_paths([path], regex=True)
     if package is None:
         logging.warning("no file found for package %s - regex %s", error.package, path)
         return False
@@ -710,7 +675,7 @@ def fix_missing_xml_entity(error, context):
     else:
         return False
 
-    package = get_package_for_paths([search_path], regex=False)
+    package = context.apt.get_package_for_paths([search_path], regex=False)
     if package is None:
         return False
     return context.add_dependency(package)
@@ -723,7 +688,7 @@ def fix_missing_library(error, context):
         os.path.join("/usr/lib/lib%s.a$" % error.library),
         os.path.join("/usr/lib/.*/lib%s.a$" % error.library),
     ]
-    package = get_package_for_paths(paths, regex=True)
+    package = context.apt.get_package_for_paths(paths, regex=True)
     if package is None:
         logging.warning("no package for library %s", error.library)
         return False
@@ -737,7 +702,7 @@ def fix_missing_ruby_gem(error, context):
             "specifications/%s-.*\\.gemspec" % error.gem
         )
     ]
-    package = get_package_for_paths(paths, regex=True)
+    package = context.apt.get_package_for_paths(paths, regex=True)
     if package is None:
         logging.warning("no package for gem %s", error.gem)
         return False
@@ -746,7 +711,7 @@ def fix_missing_ruby_gem(error, context):
 
 def fix_missing_ruby_file(error, context):
     paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)]
-    package = get_package_for_paths(paths)
+    package = context.apt.get_package_for_paths(paths)
     if package is not None:
         return context.add_dependency(package)
     paths = [
@@ -755,7 +720,7 @@ def fix_missing_ruby_file(error, context):
             "lib/%s.rb" % error.filename
         )
     ]
-    package = get_package_for_paths(paths, regex=True)
+    package = context.apt.get_package_for_paths(paths, regex=True)
     if package is not None:
         return context.add_dependency(package)
 
@@ -765,7 +730,7 @@ def fix_missing_ruby_file(error, context):
 
 def fix_missing_r_package(error, context):
     paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)]
-    package = get_package_for_paths(paths, regex=True)
+    package = context.apt.get_package_for_paths(paths, regex=True)
     if package is None:
         logging.warning("no package for R package %s", error.package)
         return False
@@ -781,7 +746,7 @@ def fix_missing_java_class(error, context):
         logging.warning("unable to find classpath for %s", error.classname)
         return False
     logging.info("Classpath for %s: %r", error.classname, classpath)
-    package = get_package_for_paths(classpath)
+    package = context.apt.get_package_for_paths(classpath)
     if package is None:
         logging.warning("no package for files in %r", classpath)
         return False
@@ -849,7 +814,7 @@ def fix_missing_maven_artifacts(error, context):
             "%s-%s.%s" % (artifact_id, version, kind),
         )
     ]
-    package = get_package_for_paths(paths, regex=regex)
+    package = context.apt.get_package_for_paths(paths, regex=regex)
     if package is None:
         logging.warning("no package for artifact %s", artifact)
         return False
@@ -862,7 +827,7 @@ def install_gnome_common(error, context):
 
 def install_gnome_common_dep(error, context):
     if error.package == "glib-gettext":
-        package = get_package_for_paths(["/usr/bin/glib-gettextize"])
+        package = context.apt.get_package_for_paths(["/usr/bin/glib-gettextize"])
     else:
         package = None
     if package is None:
@@ -875,7 +840,7 @@ def install_gnome_common_dep(error, context):
 
 def install_xfce_dep(error, context):
     if error.package == "gtk-doc":
-        package = get_package_for_paths(["/usr/bin/gtkdocize"])
+        package = context.apt.get_package_for_paths(["/usr/bin/gtkdocize"])
     else:
         package = None
     if package is None:
@@ -947,7 +912,7 @@ def fix_missing_autoconf_macro(error, context):
     except KeyError:
         logging.info("No local m4 file found defining %s", error.macro)
         return False
-    package = get_package_for_paths([path])
+    package = context.apt.get_package_for_paths([path])
     if package is None:
         logging.warning("no package for macro file %s", path)
         return False
@@ -960,7 +925,7 @@ def fix_missing_c_sharp_compiler(error, context):
 
 def fix_missing_haskell_dependencies(error, context):
     path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0]
-    package = get_package_for_paths([path], regex=True)
+    package = context.apt.get_package_for_paths([path], regex=True)
     if package is None:
         logging.warning("no package for macro file %s", path)
         return False
@@ -1033,6 +998,7 @@ def resolve_error(error, context, fixers):
 
 def build_incrementally(
     local_tree,
+    apt,
     suffix,
     build_suite,
     output_directory,
@@ -1074,6 +1040,7 @@ def build_incrementally(
             if e.context[0] == "build":
                 context = BuildDependencyContext(
                     local_tree,
+                    apt,
                     subpath=subpath,
                     committer=committer,
                     update_changelog=update_changelog,
@@ -1082,6 +1049,7 @@ def build_incrementally(
                 context = AutopkgtestDependencyContext(
                     e.context[1],
                     local_tree,
+                    apt,
                     subpath=subpath,
                     committer=committer,
                     update_changelog=update_changelog,
@@ -1154,9 +1122,12 @@ def main(argv=None):
     args = parser.parse_args()
     from breezy.workingtree import WorkingTree
 
+    apt = LocalAptManager()
+
     tree = WorkingTree.open(".")
     build_incrementally(
         tree,
+        apt,
         args.suffix,
         args.suite,
         args.output_directory,

ognibuild/dist.py

@@ -124,7 +124,7 @@ def create_dist_schroot(
     subdir: Optional[str] = None,
 ) -> str:
     from .buildsystem import detect_buildsystems
-    from .resolver import AptResolver
+    from .resolver.apt import AptResolver
 
     if subdir is None:
         subdir = "package"

ognibuild/fix_build.py

@@ -22,6 +22,7 @@ from buildlog_consultant.common import (
     find_build_failure_description,
     Problem,
     MissingPerlModule,
+    MissingPythonDistribution,
     MissingCommand,
 )
 
@@ -69,10 +70,16 @@ def fix_npm_missing_command(error, context):
     return True
 
 
+def fix_python_package_from_pip(error, context):
+    context.session.check_call(["pip", "install", error.distribution])
+    return True
+
+
 GENERIC_INSTALL_FIXERS: List[
     Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]
 ] = [
     (MissingPerlModule, fix_perl_module_from_cpan),
+    (MissingPythonDistribution, fix_python_package_from_pip),
     (MissingCommand, fix_npm_missing_command),
 ]
 
@@ -84,11 +91,12 @@ def run_with_build_fixer(session: Session, args: List[str]):
         retcode, lines = run_with_tee(session, args)
         if retcode == 0:
             return
-        offset, line, error = find_build_failure_description(lines)
+        match, error = find_build_failure_description(lines)
         if error is None:
             logging.warning("Build failed with unidentified error. Giving up.")
-            if line is not None:
+            if match is not None:
-                raise UnidentifiedError(retcode, args, lines, secondary=(offset, line))
+                raise UnidentifiedError(
+                    retcode, args, lines, secondary=(match.lineno, match.line))
             raise UnidentifiedError(retcode, args, lines)
 
         logging.info("Identified error: %r", error)

ognibuild/requirements.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+# Copyright (C) 2019-2020 Jelmer Vernooij <jelmer@jelmer.uk>
+# encoding: utf-8
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+from . import UpstreamRequirement
+
+
+class PythonPackageRequirement(UpstreamRequirement):
+
+    package: str
+
+    def __init__(self, package):
+        super(PythonPackageRequirement, self).__init__('python-package')
+        self.package = package
+
+
+class BinaryRequirement(UpstreamRequirement):
+
+    binary_name: str
+
+    def __init__(self, binary_name):
+        super(BinaryRequirement, self).__init__('binary')
+        self.binary_name = binary_name
+
+
+class PerlModuleRequirement(UpstreamRequirement):
+
+    module: str
+
+    def __init__(self, module):
+        super(PerlModuleRequirement, self).__init__('perl-module')
+        self.module = module
+
+
+class NodePackageRequirement(UpstreamRequirement):
+
+    package: str
+
+    def __init__(self, package):
+        super(NodePackageRequirement, self).__init__('npm-package')
+        self.package = package
+
+
+class CargoCrateRequirement(UpstreamRequirement):
+
+    crate: str
+
+    def __init__(self, crate):
+        super(CargoCrateRequirement, self).__init__('cargo-crate')
+        self.crate = crate

ognibuild/resolver/__init__.py

@@ -17,11 +17,13 @@
 
 
 class MissingDependencies(Exception):
+
     def __init__(self, reqs):
         self.requirements = reqs
 
 
 class Resolver(object):
+
     def install(self, requirements):
         raise NotImplementedError(self.install)
 
@@ -29,43 +31,6 @@ class Resolver(object):
         raise NotImplementedError(self.explain)
 
 
-class AptResolver(Resolver):
-    def __init__(self, apt):
-        self.apt = apt
-
-    @classmethod
-    def from_session(cls, session):
-        from .apt import AptManager
-
-        return cls(AptManager(session))
-
-    def install(self, requirements):
-        missing = []
-        for req in requirements:
-            pps = list(self._possible_paths(req))
-            if not pps or not any(self.apt.session.exists(p) for p in pps):
-                missing.append(req)
-        if missing:
-            self.apt.install(list(self.resolve(missing)))
-
-    def explain(self, requirements):
-        raise NotImplementedError(self.explain)
-
-    def _possible_paths(self, req):
-        if req.family == "binary":
-            yield "/usr/bin/%s" % req.name
-        else:
-            return
-
-    def resolve(self, requirements):
-        for req in requirements:
-            if req.family == "python3":
-                yield "python3-%s" % req.name
-            else:
-                list(self._possible_paths(req))
-                raise NotImplementedError
-
-
 class NativeResolver(Resolver):
     def __init__(self, session):
         self.session = session
@@ -94,7 +59,8 @@ class ExplainResolver(Resolver):
 
 
 class AutoResolver(Resolver):
-    """Automatically find out the most appropriate way to instal dependencies."""
+    """Automatically find out the most appropriate way to install dependencies.
+    """
 
     def __init__(self, session):
         self.session = session

ognibuild/resolver/apt.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import posixpath
+
+from ..apt import AptManager
+
+from . import Resolver
+from ..requirements import (
+    BinaryRequirement,
+    PythonPackageRequirement,
+    )
+
+
+class NoAptPackage(Exception):
+    """No apt package."""
+
+
+def resolve_binary_req(apt_mgr, req):
+    if posixpath.isabs(req.binary_name):
+        paths = [req.binary_name]
+    else:
+        paths = [
+            posixpath.join(dirname, req.binary_name)
+            for dirname in ["/usr/bin", "/bin"]
+        ]
+    return apt_mgr.get_package_for_paths(paths)
+
+
+APT_REQUIREMENT_RESOLVERS = [
+    (BinaryRequirement, resolve_binary_req),
+]
+
+
+class AptResolver(Resolver):
+
+    def __init__(self, apt):
+        self.apt = apt
+
+    @classmethod
+    def from_session(cls, session):
+        return cls(AptManager(session))
+
+    def install(self, requirements):
+        missing = []
+        for req in requirements:
+            try:
+                pps = list(req.possible_paths())
+            except NotImplementedError:
+                missing.append(req)
+            else:
+                if not pps or not any(self.apt.session.exists(p) for p in pps):
+                    missing.append(req)
+        if missing:
+            self.apt.install(list(self.resolve(missing)))
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)
+
+    def resolve(self, requirements):
+        for req in requirements:
+            for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
+                if isinstance(req, rr_class):
+                    package_name = rr_fn(self.apt, req)
+                    if package_name is None:
+                        raise NoAptPackage()
+                    yield package_name
+                    break
+            else:
+                raise NotImplementedError

ognibuild/tests/test_debian_fix_build.py

@@ -31,6 +31,7 @@ from buildlog_consultant.common import (
     MissingValaPackage,
 )
 from ..debian import apt
+from ..debian.apt import LocalAptManager
 from ..debian.fix_build import (
     resolve_error,
     VERSIONED_PACKAGE_FIXERS,
@@ -88,8 +89,10 @@ blah (0.1) UNRELEASED; urgency=medium
                     yield pkg
 
     def resolve(self, error, context=("build",)):
+        apt = LocalAptManager()
         context = BuildDependencyContext(
             self.tree,
+            apt,
             subpath="",
             committer="Janitor <janitor@jelmer.uk>",
             update_changelog=True,