More refactoring.

TODO

@@ -1 +1,2 @@
 - Need to be able to check up front whether a requirement is satisfied, before attempting to install it (which is more expensive)
+- Cache parsed Contents files during test suite runs and/or speed up reading

ognibuild/__init__.py

@@ -58,8 +58,8 @@ class UpstreamRequirement(object):
     def __init__(self, family):
         self.family = family
 
-    def possible_paths(self):
+    def met(self, session):
-        raise NotImplementedError
+        raise NotImplementedError(self)
 
 
 class UpstreamOutput(object):

ognibuild/__main__.py

@@ -20,10 +20,6 @@ import os
 import sys
 from . import UnidentifiedError
 from .buildsystem import NoBuildToolsFound, detect_buildsystems
-from .build import run_build
-from .clean import run_clean
-from .dist import run_dist
-from .install import run_install
 from .resolver import (
     ExplainResolver,
     AutoResolver,
@@ -31,7 +27,6 @@ from .resolver import (
     MissingDependencies,
 )
 from .resolver.apt import AptResolver
-from .test import run_test
 
 
 def get_necessary_declared_requirements(resolver, requirements, stages):
@@ -54,6 +49,7 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages):
 
 STAGE_MAP = {
     "dist": [],
+    "info": [],
     "install": ["build"],
     "test": ["test", "dev"],
     "build": ["build"],
@@ -65,9 +61,6 @@ def main():  # noqa: C901
     import argparse
 
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "subcommand", type=str, choices=["dist", "build", "clean", "test", "install"]
-    )
     parser.add_argument(
         "--directory", "-d", type=str, help="Directory for project.", default="."
     )
@@ -87,6 +80,16 @@ def main():  # noqa: C901
         "--verbose",
         action="store_true",
         help="Be verbose")
+    subparsers = parser.add_subparsers(dest='subcommand')
+    subparsers.add_parser('dist')
+    subparsers.add_parser('build')
+    subparsers.add_parser('clean')
+    subparsers.add_parser('test')
+    subparsers.add_parser('info')
+    install_parser = subparsers.add_parser('install')
+    install_parser.add_argument(
+        '--user', action='store_true', help='Install in local-user directories.')
+
     args = parser.parse_args()
     if args.verbose:
         logging.basicConfig(level=logging.DEBUG)
@@ -112,21 +115,32 @@ def main():  # noqa: C901
         os.chdir(args.directory)
         try:
             bss = list(detect_buildsystems(args.directory))
+            logging.info('Detected buildsystems: %r', bss)
             if not args.ignore_declared_dependencies:
                 stages = STAGE_MAP[args.subcommand]
                 if stages:
                     for bs in bss:
                         install_necessary_declared_requirements(resolver, bs, stages)
             if args.subcommand == "dist":
+                from .dist import run_dist
                 run_dist(session=session, buildsystems=bss, resolver=resolver)
             if args.subcommand == "build":
+                from .build import run_build
                 run_build(session, buildsystems=bss, resolver=resolver)
             if args.subcommand == "clean":
+                from .clean import run_clean
                 run_clean(session, buildsystems=bss, resolver=resolver)
             if args.subcommand == "install":
-                run_install(session, buildsystems=bss, resolver=resolver)
+                from .install import run_install
+                run_install(
+                    session, buildsystems=bss, resolver=resolver,
+                    user=args.user)
             if args.subcommand == "test":
+                from .test import run_test
                 run_test(session, buildsystems=bss, resolver=resolver)
+            if args.subcommand == "info":
+                from .info import run_info
+                run_info(session, buildsystems=bss, resolver=resolver)
         except UnidentifiedError:
             return 1
         except NoBuildToolsFound:

ognibuild/buildsystem.py

@@ -20,6 +20,7 @@
 import logging
 import os
 import re
+from typing import Optional
 import warnings
 
 from . import shebang_binary, UpstreamOutput, UnidentifiedError
@@ -37,6 +38,14 @@ class NoBuildToolsFound(Exception):
     """No supported build tools were found."""
 
 
+class InstallTarget(object):
+
+    # Whether to prefer user-specific installation
+    user: Optional[bool]
+
+    # TODO(jelmer): Add information about target directory, layout, etc.
+
+
 class BuildSystem(object):
     """A particular buildsystem."""
 
@@ -54,7 +63,7 @@ class BuildSystem(object):
     def clean(self, session, resolver):
         raise NotImplementedError(self.clean)
 
-    def install(self, session, resolver):
+    def install(self, session, resolver, install_target):
         raise NotImplementedError(self.install)
 
     def get_declared_dependencies(self):
@@ -84,15 +93,15 @@ class Pear(BuildSystem):
 
     def build(self, session, resolver):
         self.setup(resolver)
-        run_with_build_fixer(session, ["pear", "build"])
+        run_with_build_fixer(session, ["pear", "build", self.path])
 
     def clean(self, session, resolver):
         self.setup(resolver)
         # TODO
 
-    def install(self, session, resolver):
+    def install(self, session, resolver, install_target):
         self.setup(resolver)
-        run_with_build_fixer(session, ["pear", "install"])
+        run_with_build_fixer(session, ["pear", "install", self.path])
 
 
 class SetupPy(BuildSystem):
@@ -104,6 +113,9 @@ class SetupPy(BuildSystem):
         from distutils.core import run_setup
         self.result = run_setup(os.path.abspath(path), stop_after="init")
 
+    def __repr__(self):
+        return "%s(%r)" % (type(self).__name__, self.path)
+
     def setup(self, resolver):
         resolver.install([PythonPackageRequirement('pip')])
         with open(self.path, "r") as f:
@@ -147,9 +159,12 @@ class SetupPy(BuildSystem):
         self.setup(resolver)
         self._run_setup(session, resolver, ["clean"])
 
-    def install(self, session, resolver):
+    def install(self, session, resolver, install_target):
         self.setup(resolver)
-        self._run_setup(session, resolver, ["install"])
+        extra_args = []
+        if install_target.user:
+            extra_args.append('--user')
+        self._run_setup(session, resolver, ["install"] + extra_args)
 
     def _run_setup(self, session, resolver, args):
         interpreter = shebang_binary("setup.py")
@@ -338,7 +353,12 @@ class Make(BuildSystem):
 
     name = "make"
 
+    def __repr__(self):
+        return "%s()" % type(self).__name__
+
     def setup(self, session, resolver):
+        resolver.install([BinaryRequirement("make")])
+
         if session.exists("Makefile.PL") and not session.exists("Makefile"):
             resolver.install([BinaryRequirement("perl")])
             run_with_build_fixer(session, ["perl", "Makefile.PL"])
@@ -375,12 +395,14 @@ class Make(BuildSystem):
 
     def build(self, session, resolver):
         self.setup(session, resolver)
-        resolver.install([BinaryRequirement("make")])
         run_with_build_fixer(session, ["make", "all"])
 
+    def install(self, session, resolver, install_target):
+        self.setup(session, resolver)
+        run_with_build_fixer(session, ["make", "install"])
+
     def dist(self, session, resolver):
         self.setup(session, resolver)
-        resolver.install([BinaryRequirement("make")])
         try:
             run_with_build_fixer(session, ["make", "dist"])
         except UnidentifiedError as e:

ognibuild/debian/fix_build.py

@@ -437,7 +437,7 @@ def fix_missing_python_module(error, context):
     return True
 
 
-def problem_to_upstream_requirement(problem, context):
+def problem_to_upstream_requirement(problem):
     if isinstance(problem, MissingFile):
         return PathRequirement(problem.path)
     elif isinstance(problem, MissingCommand):
@@ -526,7 +526,11 @@ def problem_to_upstream_requirement(problem, context):
 
 class UpstreamRequirementFixer(BuildFixer):
 
-    def fix_missing_requirement(self, error, context):
+    def can_fix(self, error):
+        req = problem_to_upstream_requirement(error)
+        return req is not None
+
+    def fix(self, error, context):
         req = problem_to_upstream_requirement(error)
         if req is None:
             return False

ognibuild/fix_build.py

@@ -167,15 +167,15 @@ def run_with_build_fixer(
 
 def resolve_error(error, context, fixers):
     relevant_fixers = []
-    for error_cls, fixer in fixers:
+    for fixer in fixers:
-        if isinstance(error, error_cls):
+        if fixer.can_fix(error):
             relevant_fixers.append(fixer)
     if not relevant_fixers:
         logging.warning("No fixer found for %r", error)
         return False
     for fixer in relevant_fixers:
         logging.info("Attempting to use fixer %r to address %r", fixer, error)
-        made_changes = fixer(error, context)
+        made_changes = fixer.fix(error, context)
         if made_changes:
             return True
     return False

ognibuild/info.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+# Copyright (C) 2020-2021 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+from .buildsystem import NoBuildToolsFound, InstallTarget
+
+
+def run_info(session, buildsystems, resolver):
+    for buildsystem in buildsystems:
+        print('%r:' % buildsystem)
+        deps = {}
+        try:
+            for kind, dep in buildsystem.get_declared_dependencies():
+                deps.setdefault(kind, []).append(dep)
+        except NotImplementedError:
+            print('\tUnable to detect declared dependencies for this type of build system')
+        if deps:
+            print('\tDeclared dependencies:')
+            for kind in deps:
+                print('\t\t%s:' % kind)
+                for dep in deps[kind]:
+                    print('\t\t\t%s' % dep)
+            print('')
+        try:
+            outputs = list(buildsystem.get_declared_outputs())
+        except NotImplementedError:
+            print('\tUnable to detect declared outputs for this type of build system')
+            outputs = []
+        if outputs:
+            print('\tDeclared outputs:')
+            for output in outputs:
+                print('\t\t%s' % output)

ognibuild/install.py

@@ -15,16 +15,19 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-from .buildsystem import NoBuildToolsFound
+from .buildsystem import NoBuildToolsFound, InstallTarget
 
 
-def run_install(session, buildsystems, resolver):
+def run_install(session, buildsystems, resolver, user: bool = False):
     # Some things want to write to the user's home directory,
     # e.g. pip caches in ~/.cache
     session.create_home()
 
+    install_target = InstallTarget()
+    install_target.user = user
+
     for buildsystem in buildsystems:
-        buildsystem.install(session, resolver)
+        buildsystem.install(session, resolver, install_target)
         return
 
     raise NoBuildToolsFound()

ognibuild/requirements.py

@@ -33,10 +33,13 @@ class PythonPackageRequirement(UpstreamRequirement):
         self.minimum_version = minimum_version
 
     def __repr__(self):
-        return "%s(%r, %r, %r)" % (
+        return "%s(%r, python_version=%r, minimum_version=%r)" % (
             type(self).__name__, self.package, self.python_version,
             self.minimum_version)
 
+    def __str__(self):
+        return "python package: %s" % self.package
+
 
 class BinaryRequirement(UpstreamRequirement):
 

ognibuild/resolver/apt.py

@@ -57,7 +57,7 @@ class NoAptPackage(Exception):
     """No apt package."""
 
 
-def get_package_for_python_package(apt_mgr, package, python_version):
+def get_package_for_python_package(apt_mgr, package, python_version, minimum_version=None):
     if python_version == "pypy":
         return apt_mgr.get_package_for_paths(
             ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package],
@@ -375,18 +375,18 @@ def resolve_autoconf_macro_req(apt_mgr, req):
 
 def resolve_python_module_req(apt_mgr, req):
     if req.python_version == 2:
-        return get_package_for_python_module(apt_mgr, req.module, "cpython2")
+        return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.minimum_version)
     elif req.python_version in (None, 3):
-        return get_package_for_python_module(apt_mgr, req.module, "cpython3")
+        return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.minimum_version)
     else:
         return None
 
 
 def resolve_python_package_req(apt_mgr, req):
     if req.python_version == 2:
-        return get_package_for_python_package(apt_mgr, req.package, "cpython2")
+        return get_package_for_python_package(apt_mgr, req.package, "cpython2", req.minimum_version)
     elif req.python_version in (None, 3):
-        return get_package_for_python_package(apt_mgr, req.package, "cpython3")
+        return get_package_for_python_package(apt_mgr, req.package, "cpython3", req.minimum_version)
     else:
         return None
 
@@ -421,6 +421,13 @@ APT_REQUIREMENT_RESOLVERS = [
 ]
 
 
+class AptRequirement(object):
+
+    def __init__(self, package, minimum_version=None):
+        self.package = package
+        self.minimum_version = minimum_version
+
+
 def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement):
     for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
         if isinstance(req, rr_class):
@@ -440,15 +447,11 @@ class AptResolver(Resolver):
     def from_session(cls, session):
         return cls(AptManager(session))
 
-    def met(self, requirement):
-        pps = list(requirement.possible_paths())
-        return any(self.apt.session.exists(p) for p in pps)
-
     def install(self, requirements):
         missing = []
         for req in requirements:
             try:
-                if not self.met(req):
+                if not req.met(self.apt.session):
                     missing.append(req)
             except NotImplementedError:
                 missing.append(req)

ognibuild/tests/test_debian_fix_build.py

@@ -31,7 +31,7 @@ from buildlog_consultant.common import (
     MissingValaPackage,
 )
 from ..debian import apt
-from ..debian.apt import LocalAptManager
+from ..debian.apt import AptManager
 from ..debian.fix_build import (
     resolve_error,
     VERSIONED_PACKAGE_FIXERS,
@@ -89,7 +89,8 @@ blah (0.1) UNRELEASED; urgency=medium
                     yield pkg
 
     def resolve(self, error, context=("build",)):
-        apt = LocalAptManager()
+        from ..session.plain import PlainSession
+        apt = AptManager(PlainSession())
         context = BuildDependencyContext(
             self.tree,
             apt,