More work on resolvers.

notes/structure.md

@@ -0,0 +1,30 @@
+Upstream requirements are expressed as objects derived from UpstreamRequirement.
+
+They can either be:
+
+ * extracted from the build system
+ * extracted from errors in build logs
+
+The details of UpstreamRequirements are specific to the kind of requirement,
+and otherwise opaque to ognibuild.
+
+When building a package, we first make sure that all declared upstream
+requirements are met.
+
+Then we attempt to build.
+
+If any problems are found in the log, buildlog-consultant will report them.
+
+ognibuild can then invoke "fixers" to address Problems.
+
+Problems can be converted to UpstreamRequirements by UpstreamRequirementFixer
+
+Other Fixer can do things like e.g. upgrade configure.ac to a newer version.
+
+UpstreamRequirementFixer uses a UpstreamRequirementResolver object that
+can translate UpstreamRequirement objects into apt package names or
+e.g. cpan commands.
+
+ognibuild keeps finding problems, resolving them and rebuilding until it finds
+a problem it can not resolve or that it thinks it has already resolved
+(i.e. seen before).

ognibuild/__main__.py

@@ -38,10 +38,9 @@ def main():
     )
     parser.add_argument("--schroot", type=str, help="schroot to run in.")
     parser.add_argument(
-        "--resolve",
+        '--resolve', choices=['explain', 'apt', 'native'],
-        choices=["explain", "apt", "native"],
+        default='apt',
-        help="What to do about missing dependencies",
+        help='What to do about missing dependencies')
-    )
     args = parser.parse_args()
     if args.schroot:
         from .session.schroot import SchrootSession
@@ -52,18 +51,27 @@ def main():
 
         session = PlainSession()
     with session:
+        if args.resolve == 'apt':
+            from .resolver import AptResolver
+            resolver = AptResolver.from_session(session)
+        elif args.resolve == 'explain':
+            from .resolver import ExplainResolver
+            resolver = ExplainResolver.from_session(session)
+        elif args.resolve == 'native':
+            from .resolver import NativeResolver
+            resolver = NativeResolver.from_session(session)
         os.chdir(args.directory)
         try:
-            if args.subcommand == "dist":
+            if args.subcommand == 'dist':
-                run_dist(session)
+                run_dist(session=session, resolver=resolver)
-            if args.subcommand == "build":
+            if args.subcommand == 'build':
-                run_build(session)
+                run_build(session, resolver=resolver)
-            if args.subcommand == "clean":
+            if args.subcommand == 'clean':
-                run_clean(session)
+                run_clean(session, resolver=resolver)
-            if args.subcommand == "install":
+            if args.subcommand == 'install':
-                run_install(session)
+                run_install(session, resolver=resolver)
-            if args.subcommand == "test":
+            if args.subcommand == 'test':
-                run_test(session)
+                run_test(session, resolver=resolver)
         except NoBuildToolsFound:
             logging.info("No build tools found.")
             return 1

ognibuild/build.py

@@ -18,13 +18,13 @@
 from .buildsystem import detect_buildsystems, NoBuildToolsFound
 
 
-def run_build(session):
+def run_build(session, resolver):
     # Some things want to write to the user's home directory,
     # e.g. pip caches in ~/.cache
     session.create_home()
 
     for buildsystem in detect_buildsystems(session):
-        buildsystem.build()
+        buildsystem.build(resolver)
         return
 
     raise NoBuildToolsFound()

ognibuild/buildsystem.py

@@ -20,8 +20,8 @@
 import logging
 import re
 
-from . import shebang_binary
+from . import shebang_binary, UpstreamPackage
-from .apt import AptManager, UnidentifiedError
+from .apt import UnidentifiedError
 from .fix_build import run_with_build_fixer
 
 
@@ -35,105 +35,106 @@ class BuildSystem(object):
     def __init__(self, session):
         self.session = session
 
-    def dist(self):
+    def dist(self, resolver):
         raise NotImplementedError(self.dist)
 
-    def test(self):
+    def test(self, resolver):
         raise NotImplementedError(self.test)
 
-    def build(self):
+    def build(self, resolver):
         raise NotImplementedError(self.build)
 
-    def clean(self):
+    def clean(self, resolver):
         raise NotImplementedError(self.clean)
 
-    def install(self):
+    def install(self, resolver):
         raise NotImplementedError(self.install)
 
 
 class Pear(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        apt.install(["php-pear"])
 
-    def dist(self):
+    def setup(self, resolver):
-        self.setup()
+        resolver.install([UpstreamPackage('binary', 'pear')])
-        run_with_build_fixer(self.session, ["pear", "package"])
 
-    def test(self):
+    def dist(self, resolver):
+        self.setup(resolver)
+        run_with_build_fixer(self.session, ['pear', 'package'])
+
+    def test(self, resolver):
         self.setup()
         run_with_build_fixer(self.session, ["pear", "run-tests"])
 
-    def build(self):
+    def build(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        run_with_build_fixer(self.session, ["pear", "build"])
+        run_with_build_fixer(self.session, ['pear', 'build'])
 
-    def clean(self):
+    def clean(self, resolver):
-        self.setup()
+        self.setup(resolver)
         # TODO
 
-    def install(self):
+    def install(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        run_with_build_fixer(self.session, ["pear", "install"])
+        run_with_build_fixer(self.session, ['pear', 'install'])
 
 
 class SetupPy(BuildSystem):
-    def setup(self):
+
-        apt = AptManager(self.session)
+    def setup(self, resolver):
-        apt.install(["python3", "python3-pip"])
+        resolver.install([
-        with open("setup.py", "r") as f:
+            UpstreamPackage('python3', 'pip'),
+            UpstreamPackage('binary', 'python3'),
+            ])
+        with open('setup.py', 'r') as f:
             setup_py_contents = f.read()
         try:
             with open("setup.cfg", "r") as f:
                 setup_cfg_contents = f.read()
         except FileNotFoundError:
-            setup_cfg_contents = ""
+            setup_cfg_contents = ''
-        if "setuptools" in setup_py_contents:
+        if 'setuptools' in setup_py_contents:
-            logging.info("Reference to setuptools found, installing.")
+            logging.info('Reference to setuptools found, installing.')
-            apt.install(["python3-setuptools"])
+            resolver.install([UpstreamPackage('python3', 'setuptools')])
-        if (
+        if ('setuptools_scm' in setup_py_contents or
-            "setuptools_scm" in setup_py_contents
+                'setuptools_scm' in setup_cfg_contents):
-            or "setuptools_scm" in setup_cfg_contents
+            logging.info('Reference to setuptools-scm found, installing.')
-        ):
+            resolver.install([
-            logging.info("Reference to setuptools-scm found, installing.")
+                UpstreamPackage('python3', 'setuptools-scm'),
-            apt.install(["python3-setuptools-scm", "git", "mercurial"])
+                UpstreamPackage('binary', 'git'),
+                UpstreamPackage('binary', 'mercurial'),
+                ])
 
         # TODO(jelmer): Install setup_requires
 
-    def test(self):
+    def test(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        self._run_setup(["test"])
+        self._run_setup(resolver, ['test'])
 
-    def dist(self):
+    def dist(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        self._run_setup(["sdist"])
+        self._run_setup(resolver, ['sdist'])
 
-    def clean(self):
+    def clean(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        self._run_setup(["clean"])
+        self._run_setup(resolver, ['clean'])
 
-    def install(self):
+    def install(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        self._run_setup(["install"])
+        self._run_setup(resolver, ['install'])
 
-    def _run_setup(self, args):
+    def _run_setup(self, resolver, args):
-        apt = AptManager(self.session)
+        interpreter = shebang_binary('setup.py')
-        interpreter = shebang_binary("setup.py")
         if interpreter is not None:
-            if interpreter == "python3":
+            if interpreter in ('python3', 'python2', 'python'):
-                apt.install(["python3"])
+                resolver.install([UpstreamPackage('binary', interpreter)])
-            elif interpreter == "python2":
-                apt.install(["python2"])
-            elif interpreter == "python":
-                apt.install(["python"])
             else:
-                raise ValueError("Unknown interpreter %r" % interpreter)
+                raise ValueError('Unknown interpreter %r' % interpreter)
-            apt.install(["python2", "python3"])
+            run_with_build_fixer(
-            run_with_build_fixer(self.session, ["./setup.py"] + args)
+                self.session, ['./setup.py'] + args)
         else:
             # Just assume it's Python 3
-            apt.install(["python3"])
+            resolver.install([UpstreamPackage('binary', 'python3')])
-            run_with_build_fixer(self.session, ["python3", "./setup.py"] + args)
+            run_with_build_fixer(
+                self.session, ['python3', './setup.py'] + args)
 
 
 class PyProject(BuildSystem):
@@ -143,75 +144,79 @@ class PyProject(BuildSystem):
         with open("pyproject.toml", "r") as pf:
             return toml.load(pf)
 
-    def dist(self):
+    def dist(self, resolver):
-        apt = AptManager(self.session)
         pyproject = self.load_toml()
         if "poetry" in pyproject.get("tool", []):
             logging.info(
-                "Found pyproject.toml with poetry section, " "assuming poetry project."
+                'Found pyproject.toml with poetry section, '
-            )
+                'assuming poetry project.')
-            apt.install(["python3-venv", "python3-pip"])
+            resolver.install([
-            self.session.check_call(["pip3", "install", "poetry"], user="root")
+                UpstreamPackage('python3', 'venv'),
-            self.session.check_call(["poetry", "build", "-f", "sdist"])
+                UpstreamPackage('python3', 'pip'),
+                ])
+            self.session.check_call(['pip3', 'install', 'poetry'], user='root')
+            self.session.check_call(['poetry', 'build', '-f', 'sdist'])
             return
         raise AssertionError("no supported section in pyproject.toml")
 
 
 class SetupCfg(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        apt.install(["python3-pep517", "python3-pip"])
 
-    def dist(self):
+    def setup(self, resolver):
-        self.session.check_call(["python3", "-m", "pep517.build", "-s", "."])
+        resolver.install([
+            UpstreamPackage('python3', 'pep517'),
+            UpstreamPackage('python3', 'pip'),
+            ])
+
+    def dist(self, resolver):
+        self.setup(resolver)
+        self.session.check_call(['python3', '-m', 'pep517.build', '-s', '.'])
 
 
 class NpmPackage(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        apt.install(["npm"])
 
-    def dist(self):
+    def setup(self, resolver):
-        self.setup()
+        resolver.install([UpstreamPackage('binary', 'npm')])
-        run_with_build_fixer(self.session, ["npm", "pack"])
+
+    def dist(self, resolver):
+        self.setup(resolver)
+        run_with_build_fixer(self.session, ['npm', 'pack'])
 
 
 class Waf(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        apt.install(["python3"])
 
-    def dist(self):
+    def setup(self, resolver):
-        self.setup()
+        resolver.install([UpstreamPackage('binary', 'python3')])
-        run_with_build_fixer(self.session, ["./waf", "dist"])
+
+    def dist(self, resolver):
+        self.setup(resolver)
+        run_with_build_fixer(self.session, ['./waf', 'dist'])
 
 
 class Gem(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        apt.install(["gem2deb"])
 
-    def dist(self):
+    def setup(self, resolver):
-        self.setup()
+        resolver.install([UpstreamPackage('binary', 'gem2deb')])
-        gemfiles = [
+
-            entry.name
+    def dist(self, resolver):
-            for entry in self.session.scandir(".")
+        self.setup(resolver)
-            if entry.name.endswith(".gem")
+        gemfiles = [entry.name for entry in self.session.scandir('.')
-        ]
+                    if entry.name.endswith('.gem')]
         if len(gemfiles) > 1:
             logging.warning("More than one gemfile. Trying the first?")
         run_with_build_fixer(self.session, ["gem2tgz", gemfiles[0]])
 
 
 class DistInkt(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        apt.install(["libdist-inkt-perl"])
 
-    def dist(self):
+    def setup(self, resolver):
-        self.setup()
+        resolver.install([
-        apt = AptManager(self.session)
+            UpstreamPackage('perl', 'Dist::Inkt'),
-        with open("dist.ini", "rb") as f:
+            ])
+
+    def dist(self, resolver):
+        self.setup(resolver)
+        with open('dist.ini', 'rb') as f:
             for line in f:
                 if not line.startswith(b";;"):
                     continue
@@ -230,22 +235,23 @@ class DistInkt(BuildSystem):
                     run_with_build_fixer(self.session, ["distinkt-dist"])
                     return
         # Default to invoking Dist::Zilla
-        logging.info("Found dist.ini, assuming dist-zilla.")
+        logging.info('Found dist.ini, assuming dist-zilla.')
-        apt.install(["libdist-zilla-perl"])
+        resolver.install([UpstreamPackage('perl', 'Dist::Zilla')])
-        run_with_build_fixer(self.session, ["dzil", "build", "--in", ".."])
+        run_with_build_fixer(self.session, ['dzil', 'build', '--in', '..'])
 
 
 class Make(BuildSystem):
-    def setup(self):
-        apt = AptManager(self.session)
-        if self.session.exists("Makefile.PL") and not self.session.exists("Makefile"):
-            apt.install(["perl"])
-            run_with_build_fixer(self.session, ["perl", "Makefile.PL"])
 
-        if not self.session.exists("Makefile") and not self.session.exists("configure"):
+    def setup(self, resolver):
-            if self.session.exists("autogen.sh"):
+        if self.session.exists('Makefile.PL') and not self.session.exists('Makefile'):
-                if shebang_binary("autogen.sh") is None:
+            resolver.install([UpstreamPackage('binary', 'perl')])
-                    run_with_build_fixer(self.session, ["/bin/sh", "./autogen.sh"])
+            run_with_build_fixer(self.session, ['perl', 'Makefile.PL'])
+
+        if not self.session.exists('Makefile') and not self.session.exists('configure'):
+            if self.session.exists('autogen.sh'):
+                if shebang_binary('autogen.sh') is None:
+                    run_with_build_fixer(
+                        self.session, ['/bin/sh', './autogen.sh'])
                 try:
                     run_with_build_fixer(self.session, ["./autogen.sh"])
                 except UnidentifiedError as e:
@@ -269,10 +275,9 @@ class Make(BuildSystem):
         if not self.session.exists("Makefile") and self.session.exists("configure"):
             self.session.check_call(["./configure"])
 
-    def dist(self):
+    def dist(self, resolver):
-        self.setup()
+        self.setup(resolver)
-        apt = AptManager(self.session)
+        resolver.install([UpstreamPackage('binary', 'make')])
-        apt.install(["make"])
         try:
             run_with_build_fixer(self.session, ["make", "dist"])
         except UnidentifiedError as e:

ognibuild/clean.py

@@ -18,13 +18,13 @@
 from .buildsystem import detect_buildsystems, NoBuildToolsFound
 
 
-def run_clean(session):
+def run_clean(session, resolver):
     # Some things want to write to the user's home directory,
     # e.g. pip caches in ~/.cache
     session.create_home()
 
     for buildsystem in detect_buildsystems(session):
-        buildsystem.clean()
+        buildsystem.clean(resolver)
         return
 
     raise NoBuildToolsFound()

ognibuild/dist.py

@@ -62,13 +62,13 @@ class DistNoTarball(Exception):
     """Dist operation did not create a tarball."""
 
 
-def run_dist(session):
+def run_dist(session, resolver):
     # Some things want to write to the user's home directory,
     # e.g. pip caches in ~/.cache
     session.create_home()
 
     for buildsystem in detect_buildsystems(session):
-        buildsystem.dist()
+        buildsystem.dist(resolver)
         return
 
     raise NoBuildToolsFound()

ognibuild/install.py

@@ -18,13 +18,13 @@
 from .buildsystem import detect_buildsystems, NoBuildToolsFound
 
 
-def run_install(session):
+def run_install(session, resolver):
     # Some things want to write to the user's home directory,
     # e.g. pip caches in ~/.cache
     session.create_home()
 
     for buildsystem in detect_buildsystems(session):
-        buildsystem.install()
+        buildsystem.install(resolver)
         return
 
     raise NoBuildToolsFound()

ognibuild/resolver.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+
+class Resolver(object):
+
+    def install(self, requirements):
+        raise NotImplementedError(self.install)
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)
+
+
+class AptResolver(Resolver):
+
+    def __init__(self, apt):
+        self.apt = apt
+
+    @classmethod
+    def from_session(cls, session):
+        from .apt import AptManager
+        return cls(AptManager(session))
+
+    def install(self, requirements):
+        self.apt.install(list(self.resolve(requirements)))
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)
+
+    def resolve(self, requirements):
+        for req in requirements:
+            if req.family == 'python3':
+                yield 'python3-%s' % req.name
+            else:
+                yield self.apt.find_file('/usr/bin/%s' % req.name)
+
+
+class NativeResolver(Resolver):
+
+    def __init__(self, session):
+        self.session = session
+
+    @classmethod
+    def from_session(cls, session):
+        return cls(session)
+
+    def install(self, requirements):
+        raise NotImplementedError(self.install)
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)
+
+
+class ExplainResolver(Resolver):
+
+    def __init__(self, session):
+        self.session = session
+
+    @classmethod
+    def from_session(cls, session):
+        return cls(session)
+
+    def install(self, requirements):
+        raise NotImplementedError(self.install)
+
+    def explain(self, requirements):
+        raise NotImplementedError(self.explain)

ognibuild/test.py

@@ -18,13 +18,13 @@
 from .buildsystem import detect_buildsystems, NoBuildToolsFound
 
 
-def run_test(session):
+def run_test(session, resolver):
     # Some things want to write to the user's home directory,
     # e.g. pip caches in ~/.cache
     session.create_home()
 
     for buildsystem in detect_buildsystems(session):
-        buildsystem.test()
+        buildsystem.test(resolver)
         return
 
     raise NoBuildToolsFound()