diff --git a/TODO b/TODO index 29105b7..eab3954 100644 --- a/TODO +++ b/TODO @@ -1 +1,2 @@ - Need to be able to check up front whether a requirement is satisfied, before attempting to install it (which is more expensive) +- Cache parsed Contents files during test suite runs and/or speed up reading diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index eb32b9d..552f109 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -58,8 +58,8 @@ class UpstreamRequirement(object): def __init__(self, family): self.family = family - def possible_paths(self): - raise NotImplementedError + def met(self, session): + raise NotImplementedError(self) class UpstreamOutput(object): diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index f395f45..b6dbde2 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -20,10 +20,6 @@ import os import sys from . import UnidentifiedError from .buildsystem import NoBuildToolsFound, detect_buildsystems -from .build import run_build -from .clean import run_clean -from .dist import run_dist -from .install import run_install from .resolver import ( ExplainResolver, AutoResolver, @@ -31,7 +27,6 @@ from .resolver import ( MissingDependencies, ) from .resolver.apt import AptResolver -from .test import run_test def get_necessary_declared_requirements(resolver, requirements, stages): @@ -54,6 +49,7 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages): STAGE_MAP = { "dist": [], + "info": [], "install": ["build"], "test": ["test", "dev"], "build": ["build"], @@ -65,9 +61,6 @@ def main(): # noqa: C901 import argparse parser = argparse.ArgumentParser() - parser.add_argument( - "subcommand", type=str, choices=["dist", "build", "clean", "test", "install"] - ) parser.add_argument( "--directory", "-d", type=str, help="Directory for project.", default="." ) @@ -87,6 +80,16 @@ def main(): # noqa: C901 "--verbose", action="store_true", help="Be verbose") + subparsers = parser.add_subparsers(dest='subcommand') + subparsers.add_parser('dist') + subparsers.add_parser('build') + subparsers.add_parser('clean') + subparsers.add_parser('test') + subparsers.add_parser('info') + install_parser = subparsers.add_parser('install') + install_parser.add_argument( + '--user', action='store_true', help='Install in local-user directories.') + args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) @@ -112,21 +115,32 @@ def main(): # noqa: C901 os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) + logging.info('Detected buildsystems: %r', bss) if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: for bs in bss: install_necessary_declared_requirements(resolver, bs, stages) if args.subcommand == "dist": + from .dist import run_dist run_dist(session=session, buildsystems=bss, resolver=resolver) if args.subcommand == "build": + from .build import run_build run_build(session, buildsystems=bss, resolver=resolver) if args.subcommand == "clean": + from .clean import run_clean run_clean(session, buildsystems=bss, resolver=resolver) if args.subcommand == "install": - run_install(session, buildsystems=bss, resolver=resolver) + from .install import run_install + run_install( + session, buildsystems=bss, resolver=resolver, + user=args.user) if args.subcommand == "test": + from .test import run_test run_test(session, buildsystems=bss, resolver=resolver) + if args.subcommand == "info": + from .info import run_info + run_info(session, buildsystems=bss, resolver=resolver) except UnidentifiedError: return 1 except NoBuildToolsFound: diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 6d311f8..8f9f6ba 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,6 +20,7 @@ import logging import os import re +from typing import Optional import warnings from . import shebang_binary, UpstreamOutput, UnidentifiedError @@ -37,6 +38,14 @@ class NoBuildToolsFound(Exception): """No supported build tools were found.""" +class InstallTarget(object): + + # Whether to prefer user-specific installation + user: Optional[bool] + + # TODO(jelmer): Add information about target directory, layout, etc. + + class BuildSystem(object): """A particular buildsystem.""" @@ -54,7 +63,7 @@ class BuildSystem(object): def clean(self, session, resolver): raise NotImplementedError(self.clean) - def install(self, session, resolver): + def install(self, session, resolver, install_target): raise NotImplementedError(self.install) def get_declared_dependencies(self): @@ -84,15 +93,15 @@ class Pear(BuildSystem): def build(self, session, resolver): self.setup(resolver) - run_with_build_fixer(session, ["pear", "build"]) + run_with_build_fixer(session, ["pear", "build", self.path]) def clean(self, session, resolver): self.setup(resolver) # TODO - def install(self, session, resolver): + def install(self, session, resolver, install_target): self.setup(resolver) - run_with_build_fixer(session, ["pear", "install"]) + run_with_build_fixer(session, ["pear", "install", self.path]) class SetupPy(BuildSystem): @@ -104,6 +113,9 @@ class SetupPy(BuildSystem): from distutils.core import run_setup self.result = run_setup(os.path.abspath(path), stop_after="init") + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + def setup(self, resolver): resolver.install([PythonPackageRequirement('pip')]) with open(self.path, "r") as f: @@ -147,9 +159,12 @@ class SetupPy(BuildSystem): self.setup(resolver) self._run_setup(session, resolver, ["clean"]) - def install(self, session, resolver): + def install(self, session, resolver, install_target): self.setup(resolver) - self._run_setup(session, resolver, ["install"]) + extra_args = [] + if install_target.user: + extra_args.append('--user') + self._run_setup(session, resolver, ["install"] + extra_args) def _run_setup(self, session, resolver, args): interpreter = shebang_binary("setup.py") @@ -338,7 +353,12 @@ class Make(BuildSystem): name = "make" + def __repr__(self): + return "%s()" % type(self).__name__ + def setup(self, session, resolver): + resolver.install([BinaryRequirement("make")]) + if session.exists("Makefile.PL") and not session.exists("Makefile"): resolver.install([BinaryRequirement("perl")]) run_with_build_fixer(session, ["perl", "Makefile.PL"]) @@ -375,12 +395,14 @@ class Make(BuildSystem): def build(self, session, resolver): self.setup(session, resolver) - resolver.install([BinaryRequirement("make")]) run_with_build_fixer(session, ["make", "all"]) + def install(self, session, resolver, install_target): + self.setup(session, resolver) + run_with_build_fixer(session, ["make", "install"]) + def dist(self, session, resolver): self.setup(session, resolver) - resolver.install([BinaryRequirement("make")]) try: run_with_build_fixer(session, ["make", "dist"]) except UnidentifiedError as e: diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index d76d305..36f6139 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -437,7 +437,7 @@ def fix_missing_python_module(error, context): return True -def problem_to_upstream_requirement(problem, context): +def problem_to_upstream_requirement(problem): if isinstance(problem, MissingFile): return PathRequirement(problem.path) elif isinstance(problem, MissingCommand): @@ -526,7 +526,11 @@ def problem_to_upstream_requirement(problem, context): class UpstreamRequirementFixer(BuildFixer): - def fix_missing_requirement(self, error, context): + def can_fix(self, error): + req = problem_to_upstream_requirement(error) + return req is not None + + def fix(self, error, context): req = problem_to_upstream_requirement(error) if req is None: return False diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 4b4bbf0..d2d6a9f 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -167,15 +167,15 @@ def run_with_build_fixer( def resolve_error(error, context, fixers): relevant_fixers = [] - for error_cls, fixer in fixers: - if isinstance(error, error_cls): + for fixer in fixers: + if fixer.can_fix(error): relevant_fixers.append(fixer) if not relevant_fixers: logging.warning("No fixer found for %r", error) return False for fixer in relevant_fixers: logging.info("Attempting to use fixer %r to address %r", fixer, error) - made_changes = fixer(error, context) + made_changes = fixer.fix(error, context) if made_changes: return True return False diff --git a/ognibuild/info.py b/ognibuild/info.py new file mode 100644 index 0000000..a5e4c9f --- /dev/null +++ b/ognibuild/info.py @@ -0,0 +1,45 @@ +#!/usr/bin/python3 +# Copyright (C) 2020-2021 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from .buildsystem import NoBuildToolsFound, InstallTarget + + +def run_info(session, buildsystems, resolver): + for buildsystem in buildsystems: + print('%r:' % buildsystem) + deps = {} + try: + for kind, dep in buildsystem.get_declared_dependencies(): + deps.setdefault(kind, []).append(dep) + except NotImplementedError: + print('\tUnable to detect declared dependencies for this type of build system') + if deps: + print('\tDeclared dependencies:') + for kind in deps: + print('\t\t%s:' % kind) + for dep in deps[kind]: + print('\t\t\t%s' % dep) + print('') + try: + outputs = list(buildsystem.get_declared_outputs()) + except NotImplementedError: + print('\tUnable to detect declared outputs for this type of build system') + outputs = [] + if outputs: + print('\tDeclared outputs:') + for output in outputs: + print('\t\t%s' % output) diff --git a/ognibuild/install.py b/ognibuild/install.py index df0e61f..c30967a 100644 --- a/ognibuild/install.py +++ b/ognibuild/install.py @@ -15,16 +15,19 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from .buildsystem import NoBuildToolsFound +from .buildsystem import NoBuildToolsFound, InstallTarget -def run_install(session, buildsystems, resolver): +def run_install(session, buildsystems, resolver, user: bool = False): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() + install_target = InstallTarget() + install_target.user = user + for buildsystem in buildsystems: - buildsystem.install(session, resolver) + buildsystem.install(session, resolver, install_target) return raise NoBuildToolsFound() diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 56483ba..44c9f27 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -33,10 +33,13 @@ class PythonPackageRequirement(UpstreamRequirement): self.minimum_version = minimum_version def __repr__(self): - return "%s(%r, %r, %r)" % ( + return "%s(%r, python_version=%r, minimum_version=%r)" % ( type(self).__name__, self.package, self.python_version, self.minimum_version) + def __str__(self): + return "python package: %s" % self.package + class BinaryRequirement(UpstreamRequirement): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 39f4e0f..0c6a783 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -57,7 +57,7 @@ class NoAptPackage(Exception): """No apt package.""" -def get_package_for_python_package(apt_mgr, package, python_version): +def get_package_for_python_package(apt_mgr, package, python_version, minimum_version=None): if python_version == "pypy": return apt_mgr.get_package_for_paths( ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package], @@ -375,18 +375,18 @@ def resolve_autoconf_macro_req(apt_mgr, req): def resolve_python_module_req(apt_mgr, req): if req.python_version == 2: - return get_package_for_python_module(apt_mgr, req.module, "cpython2") + return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.minimum_version) elif req.python_version in (None, 3): - return get_package_for_python_module(apt_mgr, req.module, "cpython3") + return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.minimum_version) else: return None def resolve_python_package_req(apt_mgr, req): if req.python_version == 2: - return get_package_for_python_package(apt_mgr, req.package, "cpython2") + return get_package_for_python_package(apt_mgr, req.package, "cpython2", req.minimum_version) elif req.python_version in (None, 3): - return get_package_for_python_package(apt_mgr, req.package, "cpython3") + return get_package_for_python_package(apt_mgr, req.package, "cpython3", req.minimum_version) else: return None @@ -421,6 +421,13 @@ APT_REQUIREMENT_RESOLVERS = [ ] +class AptRequirement(object): + + def __init__(self, package, minimum_version=None): + self.package = package + self.minimum_version = minimum_version + + def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement): for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): @@ -440,15 +447,11 @@ class AptResolver(Resolver): def from_session(cls, session): return cls(AptManager(session)) - def met(self, requirement): - pps = list(requirement.possible_paths()) - return any(self.apt.session.exists(p) for p in pps) - def install(self, requirements): missing = [] for req in requirements: try: - if not self.met(req): + if not req.met(self.apt.session): missing.append(req) except NotImplementedError: missing.append(req) diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index 07725f3..0cd80ec 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -31,7 +31,7 @@ from buildlog_consultant.common import ( MissingValaPackage, ) from ..debian import apt -from ..debian.apt import LocalAptManager +from ..debian.apt import AptManager from ..debian.fix_build import ( resolve_error, VERSIONED_PACKAGE_FIXERS, @@ -89,7 +89,8 @@ blah (0.1) UNRELEASED; urgency=medium yield pkg def resolve(self, error, context=("build",)): - apt = LocalAptManager() + from ..session.plain import PlainSession + apt = AptManager(PlainSession()) context = BuildDependencyContext( self.tree, apt,