More fixes.

ognibuild/__main__.py

@@ -23,7 +23,7 @@ from .buildsystem import NoBuildToolsFound, detect_buildsystems
 from .resolver import (
     ExplainResolver,
     AutoResolver,
-    NativeResolver,
+    native_resolvers,
     MissingDependencies,
 )
 from .resolver.apt import AptResolver
@@ -91,6 +91,9 @@ def main():  # noqa: C901
         '--user', action='store_true', help='Install in local-user directories.')
 
     args = parser.parse_args()
+    if not args.subcommand:
+        parser.print_usage()
+        return 1
     if args.verbose:
         logging.basicConfig(level=logging.DEBUG)
     else:
@@ -109,7 +112,7 @@ def main():  # noqa: C901
         elif args.resolve == "explain":
             resolver = ExplainResolver.from_session(session)
         elif args.resolve == "native":
-            resolver = NativeResolver.from_session(session)
+            resolver = native_resolvers(session)
         elif args.resolver == "auto":
             resolver = AutoResolver.from_session(session)
         os.chdir(args.directory)
@@ -149,10 +152,10 @@ def main():  # noqa: C901
         except MissingDependencies as e:
             for req in e.requirements:
                 logging.info("Missing dependency (%s:%s)",
-                             req.family, req.name)
+                             req.family, req.package)
                 for resolver in [
                     AptResolver.from_session(session),
-                    NativeResolver.from_session(session),
+                    native_resolvers(session),
                 ]:
                     logging.info("  %s", resolver.explain([req]))
             return 2

ognibuild/buildlog.py

@@ -0,0 +1,192 @@
+#!/usr/bin/python3
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+"""Convert problems found in the buildlog to upstream requirements.
+"""
+
+import logging
+
+from buildlog_consultant.common import (
+    MissingConfigStatusInput,
+    MissingPythonModule,
+    MissingPythonDistribution,
+    MissingCHeader,
+    MissingPkgConfig,
+    MissingCommand,
+    MissingFile,
+    MissingJavaScriptRuntime,
+    MissingSprocketsFile,
+    MissingGoPackage,
+    MissingPerlFile,
+    MissingPerlModule,
+    MissingXmlEntity,
+    MissingJDKFile,
+    MissingNodeModule,
+    MissingPhpClass,
+    MissingRubyGem,
+    MissingLibrary,
+    MissingJavaClass,
+    MissingCSharpCompiler,
+    MissingConfigure,
+    MissingAutomakeInput,
+    MissingRPackage,
+    MissingRubyFile,
+    MissingAutoconfMacro,
+    MissingValaPackage,
+    MissingXfceDependency,
+    MissingHaskellDependencies,
+    NeedPgBuildExtUpdateControl,
+    DhAddonLoadFailure,
+    MissingMavenArtifacts,
+    GnomeCommonMissing,
+    MissingGnomeCommonDependency,
+)
+
+from .fix_build import BuildFixer
+from .requirements import (
+    BinaryRequirement,
+    PathRequirement,
+    PkgConfigRequirement,
+    CHeaderRequirement,
+    JavaScriptRuntimeRequirement,
+    ValaPackageRequirement,
+    RubyGemRequirement,
+    GoPackageRequirement,
+    DhAddonRequirement,
+    PhpClassRequirement,
+    RPackageRequirement,
+    NodePackageRequirement,
+    LibraryRequirement,
+    RubyFileRequirement,
+    XmlEntityRequirement,
+    SprocketsFileRequirement,
+    JavaClassRequirement,
+    HaskellPackageRequirement,
+    MavenArtifactRequirement,
+    GnomeCommonRequirement,
+    JDKFileRequirement,
+    PerlModuleRequirement,
+    PerlFileRequirement,
+    AutoconfMacroRequirement,
+    PythonModuleRequirement,
+    PythonPackageRequirement,
+    )
+
+
+def problem_to_upstream_requirement(problem):
+    if isinstance(problem, MissingFile):
+        return PathRequirement(problem.path)
+    elif isinstance(problem, MissingCommand):
+        return BinaryRequirement(problem.command)
+    elif isinstance(problem, MissingPkgConfig):
+        return PkgConfigRequirement(
+            problem.module, problem.minimum_version)
+    elif isinstance(problem, MissingCHeader):
+        return CHeaderRequirement(problem.header)
+    elif isinstance(problem, MissingJavaScriptRuntime):
+        return JavaScriptRuntimeRequirement()
+    elif isinstance(problem, MissingRubyGem):
+        return RubyGemRequirement(problem.gem, problem.version)
+    elif isinstance(problem, MissingValaPackage):
+        return ValaPackageRequirement(problem.package)
+    elif isinstance(problem, MissingGoPackage):
+        return GoPackageRequirement(problem.package)
+    elif isinstance(problem, DhAddonLoadFailure):
+        return DhAddonRequirement(problem.path)
+    elif isinstance(problem, MissingPhpClass):
+        return PhpClassRequirement(problem.php_class)
+    elif isinstance(problem, MissingRPackage):
+        return RPackageRequirement(problem.package, problem.minimum_version)
+    elif isinstance(problem, MissingNodeModule):
+        return NodePackageRequirement(problem.module)
+    elif isinstance(problem, MissingLibrary):
+        return LibraryRequirement(problem.library)
+    elif isinstance(problem, MissingRubyFile):
+        return RubyFileRequirement(problem.filename)
+    elif isinstance(problem, MissingXmlEntity):
+        return XmlEntityRequirement(problem.url)
+    elif isinstance(problem, MissingSprocketsFile):
+        return SprocketsFileRequirement(problem.content_type, problem.name)
+    elif isinstance(problem, MissingJavaClass):
+        return JavaClassRequirement(problem.classname)
+    elif isinstance(problem, MissingHaskellDependencies):
+        # TODO(jelmer): Create multiple HaskellPackageRequirement objects?
+        return HaskellPackageRequirement(problem.package)
+    elif isinstance(problem, MissingMavenArtifacts):
+        # TODO(jelmer): Create multiple MavenArtifactRequirement objects?
+        return MavenArtifactRequirement(problem.artifacts)
+    elif isinstance(problem, MissingCSharpCompiler):
+        return BinaryRequirement('msc')
+    elif isinstance(problem, GnomeCommonMissing):
+        return GnomeCommonRequirement()
+    elif isinstance(problem, MissingJDKFile):
+        return JDKFileRequirement(problem.jdk_path, problem.filename)
+    elif isinstance(problem, MissingGnomeCommonDependency):
+        if problem.package == "glib-gettext":
+            return BinaryRequirement('glib-gettextize')
+        else:
+            logging.warning(
+                "No known command for gnome-common dependency %s",
+                problem.package)
+            return None
+    elif isinstance(problem, MissingXfceDependency):
+        if problem.package == "gtk-doc":
+            return BinaryRequirement("gtkdocize")
+        else:
+            logging.warning(
+                "No known command for xfce dependency %s",
+                problem.package)
+            return None
+    elif isinstance(problem, MissingPerlModule):
+        return PerlModuleRequirement(
+            module=problem.module,
+            filename=problem.filename,
+            inc=problem.inc)
+    elif isinstance(problem, MissingPerlFile):
+        return PerlFileRequirement(filename=problem.filename)
+    elif isinstance(problem, MissingAutoconfMacro):
+        return AutoconfMacroRequirement(problem.macro)
+    elif isinstance(problem, MissingPythonModule):
+        return PythonModuleRequirement(
+            problem.module,
+            python_version=problem.python_version,
+            minimum_version=problem.minimum_version)
+    elif isinstance(problem, MissingPythonDistribution):
+        return PythonPackageRequirement(
+            problem.module,
+            python_version=problem.python_version,
+            minimum_version=problem.minimum_version)
+    else:
+        return None
+
+
+class UpstreamRequirementFixer(BuildFixer):
+
+    def __init__(self, resolver):
+        self.resolver = resolver
+
+    def can_fix(self, error):
+        req = problem_to_upstream_requirement(error)
+        return req is not None
+
+    def fix(self, error, context):
+        req = problem_to_upstream_requirement(error)
+        if req is None:
+            return False
+
+        package = self.resolver.resolve(req)
+        return context.add_dependency(package)

ognibuild/buildsystem.py

@@ -182,17 +182,19 @@ class SetupPy(BuildSystem):
     def get_declared_dependencies(self):
         for require in self.result.get_requires():
             yield "build", PythonPackageRequirement(require)
-        if self.result.install_requires:
+        # Not present for distutils-only packages
+        if getattr(self.result, 'install_requires', []):
             for require in self.result.install_requires:
                 yield "install", PythonPackageRequirement(require)
-        if self.result.tests_require:
+        # Not present for distutils-only packages
+        if getattr(self.result, 'tests_require', []):
             for require in self.result.tests_require:
                 yield "test", PythonPackageRequirement(require)
 
     def get_declared_outputs(self):
         for script in self.result.scripts or []:
             yield UpstreamOutput("binary", os.path.basename(script))
-        entry_points = self.result.entry_points or {}
+        entry_points = getattr(self.result, 'entry_points', None) or {}
         for script in entry_points.get("console_scripts", []):
             yield UpstreamOutput("binary", script.split("=")[0])
         for package in self.result.packages or []:

ognibuild/debian/apt.py

@@ -70,14 +70,9 @@ class AptManager(object):
 
     def package_exists(self, package):
         if self._apt_cache is None:
-            import apt_pkg
-
-            # TODO(jelmer): Load from self.session
-            self._apt_cache = apt_pkg.Cache()
-        for p in self._apt_cache.packages:
-            if p.name == package:
-                return True
-        return False
+            import apt
+            self._apt_cache = apt.Cache(rootdir=self.session.location)
+        return package in self._apt_cache
 
     def get_package_for_paths(self, paths, regex=False):
         logging.debug('Searching for packages containing %r', paths)
@@ -121,10 +116,12 @@ class RemoteAptContentsFileSearcher(FileSearcher):
     def from_session(cls, session):
         logging.info('Loading apt contents information')
         # TODO(jelmer): what about sources.list.d?
-        with open(os.path.join(session.location, 'etc/apt/sources.list'), 'r') as f:
-            return cls.from_repositories(
-                f.readlines(),
-                cache_dir=os.path.join(session.location, 'var/lib/apt/lists'))
+        from aptsources.sourceslist import SourcesList
+        sl = SourcesList()
+        sl.load(os.path.join(session.location, 'etc/apt/sources.list'))
+        return cls.from_sources_list(
+            sl,
+            cache_dir=os.path.join(session.location, 'var/lib/apt/lists'))
 
     def __setitem__(self, path, package):
         self._db[path] = package
@@ -174,32 +171,31 @@ class RemoteAptContentsFileSearcher(FileSearcher):
                     self.load_url(url)
                 except ContentsFileNotFound:
                     if mandatory:
-                        raise
-                    logging.debug(
-                        'Unable to fetch optional contents file %s', url)
+                        logging.warning(
+                            'Unable to fetch contents file %s', url)
+                    else:
+                        logging.debug(
+                            'Unable to fetch optional contents file %s', url)
         return self
 
     @classmethod
-    def from_repositories(cls, sources, cache_dir=None):
+    def from_sources_list(cls, sl, cache_dir=None):
         # TODO(jelmer): Use aptsources.sourceslist.SourcesList
         from .build import get_build_architecture
         # TODO(jelmer): Verify signatures, etc.
         urls = []
         arches = [(get_build_architecture(), True), ("all", False)]
-        for source in sources:
-            if not source.strip():
+        for source in sl.list:
+            if source.invalid or source.disabled:
                 continue
-            if source.strip().startswith('#'):
+            if source.type == 'deb-src':
                 continue
-            parts = source.split(" ")
-            if parts[0] == "deb-src":
-                continue
-            if parts[0] != "deb":
+            if source.type != 'deb':
                 logging.warning("Invalid line in sources: %r", source)
                 continue
-            base_url = parts[1].strip().rstrip("/")
-            name = parts[2].strip()
-            components = [c.strip() for c in parts[3:]]
+            base_url = source.uri.rstrip('/')
+            name = source.dist.rstrip('/')
+            components = source.comps
             if components:
                 dists_url = base_url + "/dists"
             else:

ognibuild/debian/fix_build.py

@@ -63,81 +63,28 @@ from debmutate._rules import (
 
 from breezy.plugins.debian.changelog import debcommit
 from buildlog_consultant import Problem
-from buildlog_consultant.common import (
-    MissingConfigStatusInput,
-    MissingPythonModule,
-    MissingPythonDistribution,
-    MissingCHeader,
-    MissingPkgConfig,
-    MissingCommand,
-    MissingFile,
-    MissingJavaScriptRuntime,
-    MissingSprocketsFile,
-    MissingGoPackage,
-    MissingPerlFile,
-    MissingPerlModule,
-    MissingXmlEntity,
-    MissingJDKFile,
-    MissingNodeModule,
-    MissingPhpClass,
-    MissingRubyGem,
-    MissingLibrary,
-    MissingJavaClass,
-    MissingCSharpCompiler,
-    MissingConfigure,
-    MissingAutomakeInput,
-    MissingRPackage,
-    MissingRubyFile,
-    MissingAutoconfMacro,
-    MissingValaPackage,
-    MissingXfceDependency,
-    MissingHaskellDependencies,
-    NeedPgBuildExtUpdateControl,
-    DhAddonLoadFailure,
-    MissingMavenArtifacts,
-    GnomeCommonMissing,
-    MissingGnomeCommonDependency,
-)
 from buildlog_consultant.apt import (
     AptFetchFailure,
 )
+from buildlog_consultant.common import (
+    MissingConfigStatusInput,
+    MissingAutomakeInput,
+    MissingConfigure,
+    NeedPgBuildExtUpdateControl,
+    MissingPythonModule,
+    MissingPythonDistribution,
+    MissingPerlFile,
+    )
 from buildlog_consultant.sbuild import (
     SbuildFailure,
 )
 
-from ..fix_build import BuildFixer, SimpleBuildFixer, resolve_error, DependencyContext
+from ..fix_build import BuildFixer, resolve_error, DependencyContext
+from ..buildlog import UpstreamRequirementFixer
 from ..resolver.apt import (
     NoAptPackage,
     get_package_for_python_module,
     )
-from ..requirements import (
-    BinaryRequirement,
-    PathRequirement,
-    PkgConfigRequirement,
-    CHeaderRequirement,
-    JavaScriptRuntimeRequirement,
-    ValaPackageRequirement,
-    RubyGemRequirement,
-    GoPackageRequirement,
-    DhAddonRequirement,
-    PhpClassRequirement,
-    RPackageRequirement,
-    NodePackageRequirement,
-    LibraryRequirement,
-    RubyFileRequirement,
-    XmlEntityRequirement,
-    SprocketsFileRequirement,
-    JavaClassRequirement,
-    HaskellPackageRequirement,
-    MavenArtifactRequirement,
-    GnomeCommonRequirement,
-    JDKFileRequirement,
-    PerlModuleRequirement,
-    PerlFileRequirement,
-    AutoconfMacroRequirement,
-    PythonModuleRequirement,
-    PythonPackageRequirement,
-    )
 from .build import attempt_build, DEFAULT_BUILDER
 
 
@@ -437,111 +384,6 @@ def fix_missing_python_module(error, context):
     return True
 
 
-def problem_to_upstream_requirement(problem):
-    if isinstance(problem, MissingFile):
-        return PathRequirement(problem.path)
-    elif isinstance(problem, MissingCommand):
-        return BinaryRequirement(problem.command)
-    elif isinstance(problem, MissingPkgConfig):
-        return PkgConfigRequirement(
-            problem.module, problem.minimum_version)
-    elif isinstance(problem, MissingCHeader):
-        return CHeaderRequirement(problem.header)
-    elif isinstance(problem, MissingJavaScriptRuntime):
-        return JavaScriptRuntimeRequirement()
-    elif isinstance(problem, MissingRubyGem):
-        return RubyGemRequirement(problem.gem, problem.version)
-    elif isinstance(problem, MissingValaPackage):
-        return ValaPackageRequirement(problem.package)
-    elif isinstance(problem, MissingGoPackage):
-        return GoPackageRequirement(problem.package)
-    elif isinstance(problem, DhAddonLoadFailure):
-        return DhAddonRequirement(problem.path)
-    elif isinstance(problem, MissingPhpClass):
-        return PhpClassRequirement(problem.php_class)
-    elif isinstance(problem, MissingRPackage):
-        return RPackageRequirement(problem.package, problem.minimum_version)
-    elif isinstance(problem, MissingNodeModule):
-        return NodePackageRequirement(problem.module)
-    elif isinstance(problem, MissingLibrary):
-        return LibraryRequirement(problem.library)
-    elif isinstance(problem, MissingRubyFile):
-        return RubyFileRequirement(problem.filename)
-    elif isinstance(problem, MissingXmlEntity):
-        return XmlEntityRequirement(problem.url)
-    elif isinstance(problem, MissingSprocketsFile):
-        return SprocketsFileRequirement(problem.content_type, problem.name)
-    elif isinstance(problem, MissingJavaClass):
-        return JavaClassRequirement(problem.classname)
-    elif isinstance(problem, MissingHaskellDependencies):
-        # TODO(jelmer): Create multiple HaskellPackageRequirement objects?
-        return HaskellPackageRequirement(problem.package)
-    elif isinstance(problem, MissingMavenArtifacts):
-        # TODO(jelmer): Create multiple MavenArtifactRequirement objects?
-        return MavenArtifactRequirement(problem.artifacts)
-    elif isinstance(problem, MissingCSharpCompiler):
-        return BinaryRequirement('msc')
-    elif isinstance(problem, GnomeCommonMissing):
-        return GnomeCommonRequirement()
-    elif isinstance(problem, MissingJDKFile):
-        return JDKFileRequirement(problem.jdk_path, problem.filename)
-    elif isinstance(problem, MissingGnomeCommonDependency):
-        if problem.package == "glib-gettext":
-            return BinaryRequirement('glib-gettextize')
-        else:
-            logging.warning(
-                "No known command for gnome-common dependency %s",
-                problem.package)
-            return None
-    elif isinstance(problem, MissingXfceDependency):
-        if problem.package == "gtk-doc":
-            return BinaryRequirement("gtkdocize")
-        else:
-            logging.warning(
-                "No known command for xfce dependency %s",
-                problem.package)
-            return None
-    elif isinstance(problem, MissingPerlModule):
-        return PerlModuleRequirement(
-            module=problem.module,
-            filename=problem.filename,
-            inc=problem.inc)
-    elif isinstance(problem, MissingPerlFile):
-        return PerlFileRequirement(filename=problem.filename)
-    elif isinstance(problem, MissingAutoconfMacro):
-        return AutoconfMacroRequirement(problem.macro)
-    elif isinstance(problem, MissingPythonModule):
-        return PythonModuleRequirement(
-            problem.module,
-            python_version=problem.python_version,
-            minimum_version=problem.minimum_version)
-    elif isinstance(problem, MissingPythonDistribution):
-        return PythonPackageRequirement(
-            problem.module,
-            python_version=problem.python_version,
-            minimum_version=problem.minimum_version)
-    else:
-        return None
-
-
-class UpstreamRequirementFixer(BuildFixer):
-
-    def can_fix(self, error):
-        req = problem_to_upstream_requirement(error)
-        return req is not None
-
-    def fix(self, error, context):
-        req = problem_to_upstream_requirement(error)
-        if req is None:
-            return False
-
-        try:
-            package = context.resolver.resolve(req)
-        except NoAptPackage:
-            return False
-        return context.add_dependency(package)
-
-
 def retry_apt_failure(error, context):
     return True
 
@@ -635,26 +477,39 @@ def fix_missing_makefile_pl(error, context):
     return False
 
 
-VERSIONED_PACKAGE_FIXERS: List[BuildFixer] = [
-    SimpleBuildFixer(
-        NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol),
-    SimpleBuildFixer(MissingConfigure, fix_missing_configure),
-    SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input),
-    SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input),
-]
+class SimpleBuildFixer(BuildFixer):
+
+    def __init__(self, problem_cls, fn):
+        self._problem_cls = problem_cls
+        self._fn = fn
+
+    def can_fix(self, problem):
+        return isinstance(problem, self._problem_cls)
+
+    def _fix(self, problem, context):
+        return self._fn(problem, context)
 
 
-APT_FIXERS: List[BuildFixer] = [
-    SimpleBuildFixer(MissingPythonModule, fix_missing_python_module),
-    SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution),
-    SimpleBuildFixer(AptFetchFailure, retry_apt_failure),
-    UpstreamRequirementFixer(),
-]
+def versioned_package_fixers():
+    return [
+        SimpleBuildFixer(
+            NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol),
+        SimpleBuildFixer(MissingConfigure, fix_missing_configure),
+        SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input),
+        SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input),
+        SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl),
+    ]
 
 
-GENERIC_FIXERS: List[BuildFixer] = [
-    SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl),
-]
+def apt_fixers(apt) -> List[BuildFixer]:
+    from ..resolver.apt import AptResolver
+    resolver = AptResolver(apt)
+    return [
+        SimpleBuildFixer(MissingPythonModule, fix_missing_python_module),
+        SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution),
+        SimpleBuildFixer(AptFetchFailure, retry_apt_failure),
+        UpstreamRequirementFixer(resolver),
+    ]
 
 
 def build_incrementally(
@@ -720,7 +575,7 @@ def build_incrementally(
                 raise
             try:
                 if not resolve_error(
-                    e.error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS + GENERIC_FIXERS
+                    e.error, context, versioned_package_fixers() + apt_fixers(apt)
                 ):
                     logging.warning("Failed to resolve error %r. Giving up.", e.error)
                     raise

ognibuild/fix_build.py

@@ -47,19 +47,6 @@ class BuildFixer(object):
         return self._fix(problem, context)
 
 
-class SimpleBuildFixer(BuildFixer):
-
-    def __init__(self, problem_cls, fn):
-        self._problem_cls = problem_cls
-        self._fn = fn
-
-    def can_fix(self, problem):
-        return isinstance(problem, self._problem_cls)
-
-    def _fix(self, problem, context):
-        return self._fn(problem, context)
-
-
 class DependencyContext(object):
     def __init__(
         self,
@@ -71,8 +58,6 @@ class DependencyContext(object):
     ):
         self.tree = tree
         self.apt = apt
-        from .resolver.apt import AptResolver
-        self.resolver = AptResolver(apt)
         self.subpath = subpath
         self.committer = committer
         self.update_changelog = update_changelog
@@ -94,47 +79,23 @@ class SchrootDependencyContext(DependencyContext):
         return True
 
 
-def fix_perl_module_from_cpan(error, context):
-    # TODO(jelmer): Specify -T to skip tests?
-    context.session.check_call(
-        ["cpan", "-i", error.module], user="root", env={"PERL_MM_USE_DEFAULT": "1"}
-    )
-    return True
-
-
-NPM_COMMAND_PACKAGES = {
-    "del-cli": "del-cli",
-}
-
-
-def fix_npm_missing_command(error, context):
-    try:
-        package = NPM_COMMAND_PACKAGES[error.command]
-    except KeyError:
-        return False
-
-    context.session.check_call(["npm", "-g", "install", package])
-    return True
-
-
-def fix_python_package_from_pip(error, context):
-    context.session.check_call(["pip", "install", error.distribution])
-    return True
-
-
-GENERIC_INSTALL_FIXERS: List[BuildFixer] = [
-    SimpleBuildFixer(MissingPerlModule, fix_perl_module_from_cpan),
-    SimpleBuildFixer(MissingPythonDistribution, fix_python_package_from_pip),
-    SimpleBuildFixer(MissingCommand, fix_npm_missing_command),
-]
+def generic_install_fixers(session):
+    from .buildlog import UpstreamRequirementFixer
+    from .resolver import CPANResolver, PypiResolver, NpmResolver
+    return [
+        UpstreamRequirementFixer(CPANResolver(session)),
+        UpstreamRequirementFixer(PypiResolver(session)),
+        UpstreamRequirementFixer(NpmResolver(session)),
+        ]
 
 
 def run_with_build_fixer(
         session: Session, args: List[str],
         fixers: Optional[List[BuildFixer]] = None):
     if fixers is None:
-        from .debian.fix_build import APT_FIXERS
-        fixers = GENERIC_INSTALL_FIXERS + APT_FIXERS
+        from .debian.fix_build import apt_fixers
+        from .resolver.apt import AptResolver
+        fixers = generic_install_fixers(session) + apt_fixers(AptResolver.from_session(session))
     logging.info("Running %r", args)
     fixed_errors = []
     while True:

ognibuild/resolver/__init__.py

@@ -34,20 +34,109 @@ class Resolver(object):
         raise NotImplementedError(self.met)
 
 
-class NativeResolver(Resolver):
+class CPANResolver(object):
+
     def __init__(self, session):
         self.session = session
 
-    @classmethod
-    def from_session(cls, session):
-        return cls(session)
-
     def install(self, requirements):
-        raise NotImplementedError(self.install)
+        from ..requirements import PerlModuleRequirement
+        missing = []
+        for requirement in requirements:
+            if not isinstance(requirement, PerlModuleRequirement):
+                missing.append(requirement)
+                continue
+            # TODO(jelmer): Specify -T to skip tests?
+            self.session.check_call(
+                ["cpan", "-i", requirement.module],
+                user="root", env={"PERL_MM_USE_DEFAULT": "1"}
+            )
+        if missing:
+            raise MissingDependencies(missing)
 
     def explain(self, requirements):
         raise NotImplementedError(self.explain)
 
+    def met(self, requirement):
+        raise NotImplementedError(self.met)
+
+
+class PypiResolver(object):
+
+    def __init__(self, session):
+        self.session = session
+
+    def install(self, requirements):
+        from ..requirements import PythonPackageRequirement
+        missing = []
+        for requirement in requirements:
+            if not isinstance(requirement, PythonPackageRequirement):
+                missing.append(requirement)
+                continue
+            self.session.check_call(["pip", "install", requirement.package])
+        if missing:
+            raise MissingDependencies(missing)
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)
+
+    def met(self, requirement):
+        raise NotImplementedError(self.met)
+
+
+NPM_COMMAND_PACKAGES = {
+    "del-cli": "del-cli",
+}
+
+
+class NpmResolver(object):
+
+    def __init__(self, session):
+        self.session = session
+
+    def install(self, requirements):
+        from ..requirements import NodePackageRequirement
+        missing = []
+        for requirement in requirements:
+            if not isinstance(requirement, NodePackageRequirement):
+                missing.append(requirement)
+                continue
+            try:
+                package = NPM_COMMAND_PACKAGES[requirement.command]
+            except KeyError:
+                missing.append(requirement)
+                continue
+            self.session.check_call(["npm", "-g", "install", package])
+        if missing:
+            raise MissingDependencies(missing)
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)
+
+    def met(self, requirement):
+        raise NotImplementedError(self.met)
+
+
+class StackedResolver(Resolver):
+    def __init__(self, subs):
+        self.subs = subs
+
+    def install(self, requirements):
+        for sub in self.subs:
+            try:
+                sub.install(requirements)
+            except MissingDependencies as e:
+                requirements = e.requirements
+            else:
+                return
+
+
+def native_resolvers(session):
+    return StackedResolver([
+        CPANResolver(session),
+        PypiResolver(session),
+        NpmResolver(session)])
+
 
 class ExplainResolver(Resolver):
     def __init__(self, session):

ognibuild/resolver/apt.py

@@ -21,7 +21,7 @@ import posixpath
 
 from ..debian.apt import AptManager
 
-from . import Resolver
+from . import Resolver, MissingDependencies
 from ..requirements import (
     BinaryRequirement,
     CHeaderRequirement,
@@ -57,24 +57,35 @@ class NoAptPackage(Exception):
     """No apt package."""
 
 
+class AptRequirement(object):
+
+    def __init__(self, package, minimum_version=None):
+        self.package = package
+        self.minimum_version = minimum_version
+
+
 def get_package_for_python_package(apt_mgr, package, python_version, minimum_version=None):
     if python_version == "pypy":
-        return apt_mgr.get_package_for_paths(
+        pkg_name = apt_mgr.get_package_for_paths(
             ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package],
             regex=True)
     elif python_version == "cpython2":
-        return apt_mgr.get_package_for_paths(
+        pkg_name = apt_mgr.get_package_for_paths(
             ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info/PKG-INFO" % package],
             regex=True)
     elif python_version == "cpython3":
-        return apt_mgr.get_package_for_paths(
+        pkg_name = apt_mgr.get_package_for_paths(
             ["/usr/lib/python3/dist-packages/%s-.*.egg-info/PKG-INFO" % package],
             regex=True)
     else:
         raise NotImplementedError
+    # TODO(jelmer): Dealing with epoch, etc?
+    if pkg_name is not None:
+        return AptRequirement(pkg_name, minimum_version)
+    return None
 
 
-def get_package_for_python_module(apt_mgr, module, python_version):
+def get_package_for_python_module(apt_mgr, module, python_version, minimum_version):
     if python_version == "python3":
         paths = [
             posixpath.join(
@@ -127,7 +138,10 @@ def get_package_for_python_module(apt_mgr, module, python_version):
         ]
     else:
         raise AssertionError("unknown python version %r" % python_version)
-    return apt_mgr.get_package_for_paths(paths, regex=True)
+    pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name, minimum_version=minimum_version)
+    return None
 
 
 def resolve_binary_req(apt_mgr, req):
@@ -138,7 +152,10 @@ def resolve_binary_req(apt_mgr, req):
             posixpath.join(dirname, req.binary_name)
             for dirname in ["/usr/bin", "/bin"]
         ]
-    return apt_mgr.get_package_for_paths(paths)
+    pkg_name = apt_mgr.get_package_for_paths(paths)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_pkg_config_req(apt_mgr, req):
@@ -151,11 +168,16 @@ def resolve_pkg_config_req(apt_mgr, req):
             [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")],
             regex=True,
             minimum_version=req.minimum_version)
-    return package
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_path_req(apt_mgr, req):
-    return apt_mgr.get_package_for_paths([req.path])
+    package = apt_mgr.get_package_for_paths([req.path])
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_c_header_req(apt_mgr, req):
@@ -166,17 +188,25 @@ def resolve_c_header_req(apt_mgr, req):
         package = apt_mgr.get_package_for_paths(
             [posixpath.join("/usr/include", ".*", req.header)], regex=True
         )
-    return package
+    if package is None:
+        return None
+    return AptRequirement(package)
 
 
 def resolve_js_runtime_req(apt_mgr, req):
-    return apt_mgr.get_package_for_paths(
+    package = apt_mgr.get_package_for_paths(
         ["/usr/bin/node", "/usr/bin/duk"], regex=False)
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_vala_package_req(apt_mgr, req):
     path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package
-    return apt_mgr.get_package_for_paths([path], regex=True)
+    package = apt_mgr.get_package_for_paths([path], regex=True)
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_ruby_gem_req(apt_mgr, req):
@@ -186,30 +216,45 @@ def resolve_ruby_gem_req(apt_mgr, req):
             "specifications/%s-.*\\.gemspec" % req.gem
         )
     ]
-    return apt_mgr.get_package_for_paths(
-        paths, regex=True, minimum_version=req.minimum_version)
+    package = apt_mgr.get_package_for_paths(
+        paths, regex=True)
+    if package is not None:
+        return AptRequirement(package, minimum_version=req.minimum_version)
+    return None
 
 
 def resolve_go_package_req(apt_mgr, req):
-    return apt_mgr.get_package_for_paths(
+    package = apt_mgr.get_package_for_paths(
         [posixpath.join("/usr/share/gocode/src", req.package, ".*")],
         regex=True
     )
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_dh_addon_req(apt_mgr, req):
     paths = [posixpath.join("/usr/share/perl5", req.path)]
-    return apt_mgr.get_package_for_paths(paths)
+    package = apt_mgr.get_package_for_paths(paths)
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_php_class_req(apt_mgr, req):
     path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/")
-    return apt_mgr.get_package_for_paths([path])
+    package = apt_mgr.get_package_for_paths([path])
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_r_package_req(apt_mgr, req):
     paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)]
-    return apt_mgr.get_package_for_paths(paths, regex=True)
+    package = apt_mgr.get_package_for_paths(paths, regex=True)
+    if package is not None:
+        return AptRequirement(package)
+    return None
 
 
 def resolve_node_package_req(apt_mgr, req):
@@ -218,7 +263,10 @@ def resolve_node_package_req(apt_mgr, req):
         "/usr/lib/nodejs/%s/package.json" % req.package,
         "/usr/share/nodejs/%s/package.json" % req.package,
     ]
-    return apt_mgr.get_package_for_paths(paths, regex=True)
+    pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_library_req(apt_mgr, req):
@@ -228,21 +276,27 @@ def resolve_library_req(apt_mgr, req):
         posixpath.join("/usr/lib/lib%s.a$" % req.library),
         posixpath.join("/usr/lib/.*/lib%s.a$" % req.library),
     ]
-    return apt_mgr.get_package_for_paths(paths, regex=True)
+    pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_ruby_file_req(apt_mgr, req):
     paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)]
     package = apt_mgr.get_package_for_paths(paths)
     if package is not None:
-        return package
+        return AptRequirement(package)
     paths = [
         posixpath.join(
             r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
             "lib/%s.rb" % req.filename
         )
     ]
-    return apt_mgr.get_package_for_paths(paths, regex=True)
+    pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_xml_entity_req(apt_mgr, req):
@@ -258,7 +312,10 @@ def resolve_xml_entity_req(apt_mgr, req):
     else:
         return None
 
-    return apt_mgr.get_package_for_paths([search_path], regex=False)
+    pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_sprockets_file_req(apt_mgr, req):
@@ -267,7 +324,10 @@ def resolve_sprockets_file_req(apt_mgr, req):
     else:
         logging.warning("unable to handle content type %s", req.content_type)
         return None
-    return apt_mgr.get_package_for_paths([path], regex=True)
+    pkg_name = apt_mgr.get_package_for_paths([path], regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_java_class_req(apt_mgr, req):
@@ -285,12 +345,15 @@ def resolve_java_class_req(apt_mgr, req):
     if package is None:
         logging.warning("no package for files in %r", classpath)
         return None
-    return package
+    return AptRequirement(package)
 
 
 def resolve_haskell_package_req(apt_mgr, req):
     path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0]
-    return apt_mgr.get_package_for_paths([path], regex=True)
+    pkg_name = apt_mgr.get_package_for_paths([path], regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_maven_artifact_req(apt_mgr, req):
@@ -319,16 +382,22 @@ def resolve_maven_artifact_req(apt_mgr, req):
             "%s-%s.%s" % (artifact_id, version, kind),
         )
     ]
-    return apt_mgr.get_package_for_paths(paths, regex=regex)
+    pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_gnome_common_req(apt_mgr, req):
-    return 'gnome-common'
+    return AptRequirement('gnome-common')
 
 
 def resolve_jdk_file_req(apt_mgr, req):
     path = req.jdk_path + ".*/" + req.filename
-    return apt_mgr.get_package_for_paths([path], regex=True)
+    pkg_name = apt_mgr.get_package_for_paths([path], regex=True)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_perl_module_req(apt_mgr, req):
@@ -344,11 +413,17 @@ def resolve_perl_module_req(apt_mgr, req):
             paths = [req.filename]
     else:
         paths = [posixpath.join(inc, req.filename) for inc in req.inc]
-    return apt_mgr.get_package_for_paths(paths, regex=False)
+    pkg_name = apt_mgr.get_package_for_paths(paths, regex=False)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_perl_file_req(apt_mgr, req):
-    return apt_mgr.get_package_for_paths([req.filename], regex=False)
+    pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False)
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def _find_aclocal_fun(macro):
@@ -370,7 +445,10 @@ def resolve_autoconf_macro_req(apt_mgr, req):
     except KeyError:
         logging.info("No local m4 file found defining %s", req.macro)
         return None
-    return apt_mgr.get_package_for_paths([path])
+    pkg_name = apt_mgr.get_package_for_paths([path])
+    if pkg_name is not None:
+        return AptRequirement(pkg_name)
+    return None
 
 
 def resolve_python_module_req(apt_mgr, req):
@@ -421,14 +499,7 @@ APT_REQUIREMENT_RESOLVERS = [
 ]
 
 
-class AptRequirement(object):
-
-    def __init__(self, package, minimum_version=None):
-        self.package = package
-        self.minimum_version = minimum_version
-
-
-def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement):
+def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement) -> AptRequirement:
     for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
         if isinstance(req, rr_class):
             deb_req = rr_fn(apt_mgr, req)
@@ -456,7 +527,17 @@ class AptResolver(Resolver):
             except NotImplementedError:
                 missing.append(req)
         if missing:
-            self.apt.install([self.resolve(m) for m in missing])
+            still_missing = []
+            apt_requirements = []
+            for m in missing:
+                try:
+                    apt_requirements.append(self.resolve(m))
+                except NoAptPackage:
+                    still_missing.append(m)
+            self.apt.install(
+                [req.package for req in apt_requirements])
+            if still_missing:
+                raise MissingDependencies(still_missing)
 
     def explain(self, requirements):
         raise NotImplementedError(self.explain)

ognibuild/tests/test_debian_fix_build.py

@@ -34,8 +34,8 @@ from ..debian import apt
 from ..debian.apt import AptManager
 from ..debian.fix_build import (
     resolve_error,
-    VERSIONED_PACKAGE_FIXERS,
-    APT_FIXERS,
+    versioned_package_fixers,
+    apt_fixers,
     BuildDependencyContext,
 )
 from breezy.tests import TestCaseWithTransport
@@ -95,10 +95,10 @@ blah (0.1) UNRELEASED; urgency=medium
             self.tree,
             apt,
             subpath="",
-            committer="Janitor <janitor@jelmer.uk>",
+            committer="ognibuild <ognibuild@jelmer.uk>",
             update_changelog=True,
         )
-        return resolve_error(error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS)
+        return resolve_error(error, context, versioned_package_fixers() + apt_fixers(apt))
 
     def get_build_deps(self):
         with open(self.tree.abspath("debian/control"), "r") as f: