From dd14deb00d1be5ebff24f2a6edbf248cee940dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Feb 2021 16:05:36 +0000 Subject: [PATCH] Fix all tests. --- ognibuild/__init__.py | 3 + ognibuild/__main__.py | 56 +++++----- ognibuild/build.py | 4 +- ognibuild/buildlog.py | 2 + ognibuild/buildsystem.py | 136 ++++++++++++----------- ognibuild/clean.py | 4 +- ognibuild/debian/fix_build.py | 118 +++++++++++--------- ognibuild/dist.py | 4 +- ognibuild/fix_build.py | 19 +--- ognibuild/info.py | 2 +- ognibuild/install.py | 4 +- ognibuild/resolver/__init__.py | 79 +++++++++---- ognibuild/resolver/apt.py | 25 ++--- ognibuild/session/plain.py | 3 + ognibuild/test.py | 4 +- ognibuild/tests/test_debian_fix_build.py | 34 +++--- setup.py | 5 +- 17 files changed, 280 insertions(+), 222 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 552f109..9b7b07f 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -67,3 +67,6 @@ class UpstreamOutput(object): def __init__(self, family, name): self.family = family self.name = name + + def __repr__(self): + return "%s(%r, %r)" % (type(self).__name__, self.family, self.name) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index d1ffb02..3db88bf 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -21,8 +21,7 @@ import sys from . import UnidentifiedError from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( - ExplainResolver, - AutoResolver, + auto_resolver, native_resolvers, MissingDependencies, ) @@ -56,6 +55,11 @@ STAGE_MAP = { "clean": [], } +def determine_fixers(session, resolver): + from .buildlog import UpstreamRequirementFixer + from .resolver.apt import AptResolver + return [UpstreamRequirementFixer(resolver)] + def main(): # noqa: C901 import argparse @@ -67,12 +71,17 @@ def main(): # noqa: C901 parser.add_argument("--schroot", type=str, help="schroot to run in.") parser.add_argument( "--resolve", - choices=["explain", "apt", "native"], - default="apt", + choices=["apt", "native", "auto"], + default="auto", help="What to do about missing dependencies", ) + parser.add_argument( + "--explain", + action='store_true', + help="Explain what needs to be done rather than making changes") parser.add_argument( "--ignore-declared-dependencies", + "--optimistic", action="store_true", help="Ignore declared dependencies, follow build errors only", ) @@ -109,56 +118,53 @@ def main(): # noqa: C901 with session: if args.resolve == "apt": resolver = AptResolver.from_session(session) - elif args.resolve == "explain": - resolver = ExplainResolver.from_session(session) elif args.resolve == "native": resolver = native_resolvers(session) - elif args.resolver == "auto": - resolver = AutoResolver.from_session(session) + elif args.resolve == "auto": + resolver = auto_resolver(session) + logging.info('Using requirement resolver: %s', resolver) os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) logging.info('Detected buildsystems: %r', bss) - if not args.ignore_declared_dependencies: + if not args.ignore_declared_dependencies and not args.explain: stages = STAGE_MAP[args.subcommand] if stages: for bs in bss: install_necessary_declared_requirements(resolver, bs, stages) + fixers = determine_fixers(session, resolver) if args.subcommand == "dist": from .dist import run_dist - run_dist(session=session, buildsystems=bss, resolver=resolver) + run_dist( + session=session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "build": from .build import run_build - run_build(session, buildsystems=bss, resolver=resolver) + run_build( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "clean": from .clean import run_clean - run_clean(session, buildsystems=bss, resolver=resolver) + run_clean( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "install": from .install import run_install run_install( session, buildsystems=bss, resolver=resolver, - user=args.user) + fixers=fixers, user=args.user) if args.subcommand == "test": from .test import run_test - run_test(session, buildsystems=bss, resolver=resolver) + run_test(session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "info": from .info import run_info - run_info(session, buildsystems=bss, resolver=resolver) + run_info(session, buildsystems=bss) except UnidentifiedError: return 1 except NoBuildToolsFound: logging.info("No build tools found.") return 1 - except MissingDependencies as e: - for req in e.requirements: - logging.info("Missing dependency (%s:%s)", - req.family, req.package) - for resolver in [ - AptResolver.from_session(session), - native_resolvers(session), - ]: - logging.info(" %s", resolver.explain([req])) - return 2 return 0 diff --git a/ognibuild/build.py b/ognibuild/build.py index b58db3a..1b03bf5 100644 --- a/ognibuild/build.py +++ b/ognibuild/build.py @@ -18,13 +18,13 @@ from .buildsystem import NoBuildToolsFound -def run_build(session, buildsystems, resolver): +def run_build(session, buildsystems, resolver, fixers): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.build(session, resolver) + buildsystem.build(session, resolver, fixers) return raise NoBuildToolsFound() diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 0ff19a8..ae358d5 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -189,4 +189,6 @@ class UpstreamRequirementFixer(BuildFixer): return False package = self.resolver.resolve(req) + if package is None: + return False return context.add_dependency(package) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index cc8b30e..37261aa 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -31,7 +31,7 @@ from .requirements import ( NodePackageRequirement, CargoCrateRequirement, ) -from .fix_build import run_with_build_fixer +from .fix_build import run_with_build_fixers class NoBuildToolsFound(Exception): @@ -51,19 +51,19 @@ class BuildSystem(object): name: str - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): raise NotImplementedError(self.dist) - def test(self, session, resolver): + def test(self, session, resolver, fixers): raise NotImplementedError(self.test) - def build(self, session, resolver): + def build(self, session, resolver, fixers): raise NotImplementedError(self.build) - def clean(self, session, resolver): + def clean(self, session, resolver, fixers): raise NotImplementedError(self.clean) - def install(self, session, resolver, install_target): + def install(self, session, resolver, fixers, install_target): raise NotImplementedError(self.install) def get_declared_dependencies(self): @@ -83,25 +83,25 @@ class Pear(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("pear")]) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) - run_with_build_fixer(session, ["pear", "package"]) + run_with_build_fixers(session, ["pear", "package"], fixers) - def test(self, session, resolver): + def test(self, session, resolver, fixers): self.setup(resolver) - run_with_build_fixer(session, ["pear", "run-tests"]) + run_with_build_fixers(session, ["pear", "run-tests"], fixers) - def build(self, session, resolver): + def build(self, session, resolver, fixers): self.setup(resolver) - run_with_build_fixer(session, ["pear", "build", self.path]) + run_with_build_fixers(session, ["pear", "build", self.path], fixers) - def clean(self, session, resolver): + def clean(self, session, resolver, fixers): self.setup(resolver) # TODO - def install(self, session, resolver, install_target): + def install(self, session, resolver, fixers, install_target): self.setup(resolver) - run_with_build_fixer(session, ["pear", "install", self.path]) + run_with_build_fixers(session, ["pear", "install", self.path], fixers) class SetupPy(BuildSystem): @@ -143,41 +143,40 @@ class SetupPy(BuildSystem): # TODO(jelmer): Install setup_requires - def test(self, session, resolver): + def test(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["test"]) + self._run_setup(session, resolver, ["test"], fixers) - def build(self, session, resolver): + def build(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["build"]) + self._run_setup(session, resolver, ["build"], fixers) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["sdist"]) + self._run_setup(session, resolver, ["sdist"], fixers) - def clean(self, session, resolver): + def clean(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["clean"]) + self._run_setup(session, resolver, ["clean"], fixers) - def install(self, session, resolver, install_target): + def install(self, session, resolver, fixers, install_target): self.setup(resolver) extra_args = [] if install_target.user: extra_args.append('--user') - self._run_setup(session, resolver, ["install"] + extra_args) + self._run_setup(session, resolver, ["install"] + extra_args, fixers) - def _run_setup(self, session, resolver, args): + def _run_setup(self, session, resolver, args, fixers): interpreter = shebang_binary("setup.py") if interpreter is not None: - if interpreter in ("python3", "python2", "python"): - resolver.install([BinaryRequirement(interpreter)]) - else: - raise ValueError("Unknown interpreter %r" % interpreter) - run_with_build_fixer(session, ["./setup.py"] + args) + resolver.install([BinaryRequirement(interpreter)]) + run_with_build_fixers(session, ["./setup.py"] + args, fixers) else: # Just assume it's Python 3 resolver.install([BinaryRequirement("python3")]) - run_with_build_fixer(session, ["python3", "./setup.py"] + args) + run_with_build_fixers( + session, ["python3", "./setup.py"] + args, + fixers) def get_declared_dependencies(self): for require in self.result.get_requires(): @@ -215,7 +214,7 @@ class PyProject(BuildSystem): with open(self.path, "r") as pf: return toml.load(pf) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): if "poetry" in self.pyproject.get("tool", []): logging.info( "Found pyproject.toml with poetry section, " "assuming poetry project." @@ -247,7 +246,7 @@ class SetupCfg(BuildSystem): ] ) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) session.check_call(["python3", "-m", "pep517.build", "-s", "."]) @@ -271,9 +270,9 @@ class Npm(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("npm")]) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) - run_with_build_fixer(session, ["npm", "pack"]) + run_with_build_fixers(session, ["npm", "pack"], fixers) class Waf(BuildSystem): @@ -286,9 +285,9 @@ class Waf(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("python3")]) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) - run_with_build_fixer(session, ["./waf", "dist"]) + run_with_build_fixers(session, ["./waf", "dist"], fixers) class Gem(BuildSystem): @@ -301,14 +300,14 @@ class Gem(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("gem2deb")]) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) gemfiles = [ entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") ] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") - run_with_build_fixer(session, ["gem2tgz", gemfiles[0]]) + run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) class DistInkt(BuildSystem): @@ -340,15 +339,16 @@ class DistInkt(BuildSystem): ] ) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(resolver) if self.name == "dist-inkt": resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) - run_with_build_fixer(session, ["distinkt-dist"]) + run_with_build_fixers(session, ["distinkt-dist"], fixers) else: # Default to invoking Dist::Zilla resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixer(session, ["dzil", "build", "--in", ".."]) + run_with_build_fixers( + session, ["dzil", "build", "--in", ".."], fixers) class Make(BuildSystem): @@ -358,26 +358,30 @@ class Make(BuildSystem): def __repr__(self): return "%s()" % type(self).__name__ - def setup(self, session, resolver): + def setup(self, session, resolver, fixers): resolver.install([BinaryRequirement("make")]) if session.exists("Makefile.PL") and not session.exists("Makefile"): resolver.install([BinaryRequirement("perl")]) - run_with_build_fixer(session, ["perl", "Makefile.PL"]) + run_with_build_fixers(session, ["perl", "Makefile.PL"], fixers) if not session.exists("Makefile") and not session.exists("configure"): if session.exists("autogen.sh"): if shebang_binary("autogen.sh") is None: - run_with_build_fixer(session, ["/bin/sh", "./autogen.sh"]) + run_with_build_fixers( + session, ["/bin/sh", "./autogen.sh"], fixers) try: - run_with_build_fixer(session, ["./autogen.sh"]) + run_with_build_fixers( + session, ["./autogen.sh"], fixers) except UnidentifiedError as e: if ( "Gnulib not yet bootstrapped; " "run ./bootstrap instead.\n" in e.lines ): - run_with_build_fixer(session, ["./bootstrap"]) - run_with_build_fixer(session, ["./autogen.sh"]) + run_with_build_fixers( + session, ["./bootstrap"], fixers) + run_with_build_fixers( + session, ["./autogen.sh"], fixers) else: raise @@ -390,23 +394,23 @@ class Make(BuildSystem): BinaryRequirement("libtoolize"), ] ) - run_with_build_fixer(session, ["autoreconf", "-i"]) + run_with_build_fixers(session, ["autoreconf", "-i"], fixers) if not session.exists("Makefile") and session.exists("configure"): session.check_call(["./configure"]) - def build(self, session, resolver): + def build(self, session, resolver, fixers): self.setup(session, resolver) - run_with_build_fixer(session, ["make", "all"]) + run_with_build_fixers(session, ["make", "all"], fixers) - def install(self, session, resolver, install_target): + def install(self, session, resolver, fixers, install_target): self.setup(session, resolver) - run_with_build_fixer(session, ["make", "install"]) + run_with_build_fixers(session, ["make", "install"], fixers) - def dist(self, session, resolver): + def dist(self, session, resolver, fixers): self.setup(session, resolver) try: - run_with_build_fixer(session, ["make", "dist"]) + run_with_build_fixers(session, ["make", "dist"], fixers) except UnidentifiedError as e: if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: pass @@ -416,17 +420,17 @@ class Make(BuildSystem): "Reconfigure the source tree " "(via './config' or 'perl Configure'), please.\n" ) in e.lines: - run_with_build_fixer(session, ["./config"]) - run_with_build_fixer(session, ["make", "dist"]) + run_with_build_fixers(session, ["./config"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) elif ( "Please try running 'make manifest' and then run " "'make dist' again.\n" in e.lines ): - run_with_build_fixer(session, ["make", "manifest"]) - run_with_build_fixer(session, ["make", "dist"]) + run_with_build_fixers(session, ["make", "manifest"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) elif "Please run ./configure first\n" in e.lines: - run_with_build_fixer(session, ["./configure"]) - run_with_build_fixer(session, ["make", "dist"]) + run_with_build_fixers(session, ["./configure"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) elif any( [ re.match( @@ -437,8 +441,8 @@ class Make(BuildSystem): for line in e.lines ] ): - run_with_build_fixer(session, ["./configure"]) - run_with_build_fixer(session, ["make", "dist"]) + run_with_build_fixers(session, ["./configure"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) elif any( [ re.match( @@ -449,8 +453,8 @@ class Make(BuildSystem): for line in e.lines ] ): - run_with_build_fixer(session, ["make", "manifest"]) - run_with_build_fixer(session, ["make", "dist"]) + run_with_build_fixers(session, ["make", "manifest"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) else: raise else: diff --git a/ognibuild/clean.py b/ognibuild/clean.py index 9f1c4d1..6bbb3ee 100644 --- a/ognibuild/clean.py +++ b/ognibuild/clean.py @@ -18,13 +18,13 @@ from .buildsystem import NoBuildToolsFound -def run_clean(session, buildsystems, resolver): +def run_clean(session, buildsystems, resolver, fixers): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.clean(session, resolver) + buildsystem.clean(session, resolver, fixers) return raise NoBuildToolsFound() diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index a32219b..1ccc9bf 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -36,7 +36,6 @@ from breezy.tree import Tree from debmutate.control import ( ensure_some_version, ensure_minimum_version, - pg_buildext_updatecontrol, ControlEditor, ) from debmutate.debhelper import ( @@ -82,7 +81,7 @@ from buildlog_consultant.sbuild import ( from ..fix_build import BuildFixer, resolve_error, DependencyContext from ..buildlog import UpstreamRequirementFixer from ..resolver.apt import ( - NoAptPackage, + AptRequirement, get_package_for_python_module, ) from .build import attempt_build, DEFAULT_BUILDER @@ -99,11 +98,11 @@ class CircularDependency(Exception): class BuildDependencyContext(DependencyContext): - def add_dependency(self, package: str, minimum_version: Optional[Version] = None): + + def add_dependency(self, requirement: AptRequirement): return add_build_dependency( self.tree, - package, - minimum_version=minimum_version, + requirement, committer=self.committer, subpath=self.subpath, update_changelog=self.update_changelog, @@ -119,12 +118,11 @@ class AutopkgtestDependencyContext(DependencyContext): tree, apt, subpath, committer, update_changelog ) - def add_dependency(self, package, minimum_version=None): + def add_dependency(self, requirement): return add_test_dependency( self.tree, self.testname, - package, - minimum_version=minimum_version, + requirement, committer=self.committer, subpath=self.subpath, update_changelog=self.update_changelog, @@ -133,37 +131,38 @@ class AutopkgtestDependencyContext(DependencyContext): def add_build_dependency( tree: Tree, - package: str, - minimum_version: Optional[Version] = None, + requirement: AptRequirement, committer: Optional[str] = None, subpath: str = "", update_changelog: bool = True, ): - if not isinstance(package, str): - raise TypeError(package) + if not isinstance(requirement, AptRequirement): + raise TypeError(requirement) control_path = os.path.join(tree.abspath(subpath), "debian/control") try: with ControlEditor(path=control_path) as updater: for binary in updater.binaries: - if binary["Package"] == package: - raise CircularDependency(package) - if minimum_version: + if binary["Package"] == requirement.package: + raise CircularDependency(requirement.package) + if requirement.minimum_version: updater.source["Build-Depends"] = ensure_minimum_version( - updater.source.get("Build-Depends", ""), package, minimum_version + updater.source.get("Build-Depends", ""), + requirement.package, requirement.minimum_version ) else: updater.source["Build-Depends"] = ensure_some_version( - updater.source.get("Build-Depends", ""), package + updater.source.get("Build-Depends", ""), + requirement.package ) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False - if minimum_version: - desc = "%s (>= %s)" % (package, minimum_version) + if requirement.minimum_version: + desc = "%s (>= %s)" % (requirement.package, requirement.minimum_version) else: - desc = package + desc = requirement.package if not updater.changed: logging.info("Giving up; dependency %s was already present.", desc) @@ -182,14 +181,13 @@ def add_build_dependency( def add_test_dependency( tree, testname, - package, - minimum_version=None, + requirement, committer=None, subpath="", update_changelog=True, ): - if not isinstance(package, str): - raise TypeError(package) + if not isinstance(requirement, AptRequirement): + raise TypeError(requirement) tests_control_path = os.path.join(tree.abspath(subpath), "debian/tests/control") @@ -204,13 +202,14 @@ def add_test_dependency( command_counter += 1 if name != testname: continue - if minimum_version: + if requirement.minimum_version: control["Depends"] = ensure_minimum_version( - control.get("Depends", ""), package, minimum_version + control.get("Depends", ""), + requirement.package, requirement.minimum_version ) else: control["Depends"] = ensure_some_version( - control.get("Depends", ""), package + control.get("Depends", ""), requirement.package ) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) @@ -218,10 +217,11 @@ def add_test_dependency( if not updater.changed: return False - if minimum_version: - desc = "%s (>= %s)" % (package, minimum_version) + if requirement.minimum_version: + desc = "%s (>= %s)" % ( + requirement.package, requirement.minimum_version) else: - desc = package + desc = requirement.package logging.info("Adding dependency to test %s: %s", testname, desc) return commit_debian_changes( @@ -333,7 +333,9 @@ def fix_missing_python_distribution(error, context): # noqa: C901 for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency(dep_pkg, minimum_version=error.minimum_version): + if not context.add_dependency( + AptRequirement( + dep_pkg.package, minimum_version=error.minimum_version)): return False return True @@ -345,9 +347,9 @@ def fix_missing_python_module(error, context): targeted = set() default = not targeted - pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy") - py2_pkg = get_package_for_python_module(context.apt, error.module, "python2") - py3_pkg = get_package_for_python_module(context.apt, error.module, "python3") + pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", None) + py2_pkg = get_package_for_python_module(context.apt, error.module, "python2", None) + py3_pkg = get_package_for_python_module(context.apt, error.module, "python3", None) extra_build_deps = [] if error.python_version == 2: @@ -379,7 +381,8 @@ def fix_missing_python_module(error, context): for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency(dep_pkg, error.minimum_version): + if not context.add_dependency( + AptRequirement(dep_pkg.package, error.minimum_version)): return False return True @@ -402,7 +405,7 @@ def enable_dh_autoreconf(context): return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return context.add_dependency("dh-autoreconf") + return context.add_dependency(AptRequirement("dh-autoreconf")) return False @@ -453,17 +456,27 @@ def fix_missing_config_status_input(error, context): return True -def run_pgbuildext_updatecontrol(error, context): - logging.info("Running 'pg_buildext updatecontrol'") - # TODO(jelmer): run in the schroot - pg_buildext_updatecontrol(context.tree.abspath(context.subpath)) - return commit_debian_changes( - context.tree, - context.subpath, - "Run 'pgbuildext updatecontrol'.", - committer=context.committer, - update_changelog=False, - ) +class PgBuildExtOutOfDateControlFixer(BuildFixer): + + def __init__(self, session): + self.session = session + + def can_fix(self, problem): + return isinstance(problem, NeedPgBuildExtUpdateControl) + + def _fix(self, problem, context): + return self._fn(problem, context) + + def _fix(self, error, context): + logging.info("Running 'pg_buildext updatecontrol'") + self.session.check_call(["pg_buildext", "updatecontrol"]) + return commit_debian_changes( + context.tree, + context.subpath, + "Run 'pgbuildext updatecontrol'.", + committer=context.committer, + update_changelog=False, + ) def fix_missing_makefile_pl(error, context): @@ -490,10 +503,9 @@ class SimpleBuildFixer(BuildFixer): return self._fn(problem, context) -def versioned_package_fixers(): +def versioned_package_fixers(session): return [ - SimpleBuildFixer( - NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol), + PgBuildExtOutOfDateControlFixer(session), SimpleBuildFixer(MissingConfigure, fix_missing_configure), SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input), SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input), @@ -527,6 +539,8 @@ def build_incrementally( update_changelog=True, ): fixed_errors = [] + fixers = versioned_package_fixers(apt.session) + apt_fixers(apt) + logging.info('Using fixers: %r', fixers) while True: try: return attempt_build( @@ -574,9 +588,7 @@ def build_incrementally( logging.warning("unable to install for context %r", e.phase) raise try: - if not resolve_error( - e.error, context, versioned_package_fixers() + apt_fixers(apt) - ): + if not resolve_error(e.error, context, fixers): logging.warning("Failed to resolve error %r. Giving up.", e.error) raise except GeneratedFile: diff --git a/ognibuild/dist.py b/ognibuild/dist.py index d226e0f..cfe38d2 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -62,13 +62,13 @@ class DistNoTarball(Exception): """Dist operation did not create a tarball.""" -def run_dist(session, buildsystems, resolver): +def run_dist(session, buildsystems, resolver, fixers): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.dist(session, resolver) + buildsystem.dist(session, resolver, fixers) return raise NoBuildToolsFound() diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index c6d25a1..5520e31 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -79,23 +79,8 @@ class SchrootDependencyContext(DependencyContext): return True -def generic_install_fixers(session): - from .buildlog import UpstreamRequirementFixer - from .resolver import CPANResolver, PypiResolver, NpmResolver - return [ - UpstreamRequirementFixer(CPANResolver(session)), - UpstreamRequirementFixer(PypiResolver(session)), - UpstreamRequirementFixer(NpmResolver(session)), - ] - - -def run_with_build_fixer( - session: Session, args: List[str], - fixers: Optional[List[BuildFixer]] = None): - if fixers is None: - from .debian.fix_build import apt_fixers - from .resolver.apt import AptResolver - fixers = generic_install_fixers(session) + apt_fixers(AptResolver.from_session(session)) +def run_with_build_fixers( + session: Session, args: List[str], fixers: List[BuildFixer]): logging.info("Running %r", args) fixed_errors = [] while True: diff --git a/ognibuild/info.py b/ognibuild/info.py index a5e4c9f..3848b2b 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -18,7 +18,7 @@ from .buildsystem import NoBuildToolsFound, InstallTarget -def run_info(session, buildsystems, resolver): +def run_info(session, buildsystems): for buildsystem in buildsystems: print('%r:' % buildsystem) deps = {} diff --git a/ognibuild/install.py b/ognibuild/install.py index c30967a..bf7bf62 100644 --- a/ognibuild/install.py +++ b/ognibuild/install.py @@ -18,7 +18,7 @@ from .buildsystem import NoBuildToolsFound, InstallTarget -def run_install(session, buildsystems, resolver, user: bool = False): +def run_install(session, buildsystems, resolver, fixers, user: bool = False): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() @@ -27,7 +27,7 @@ def run_install(session, buildsystems, resolver, user: bool = False): install_target.user = user for buildsystem in buildsystems: - buildsystem.install(session, resolver, install_target) + buildsystem.install(session, resolver, fixers, install_target) return raise NoBuildToolsFound() diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 90e40c7..dd12b60 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -34,11 +34,14 @@ class Resolver(object): raise NotImplementedError(self.met) -class CPANResolver(object): +class CPANResolver(Resolver): def __init__(self, session): self.session = session + def __str__(self): + return "cpan" + def install(self, requirements): from ..requirements import PerlModuleRequirement missing = [] @@ -61,11 +64,42 @@ class CPANResolver(object): raise NotImplementedError(self.met) -class PypiResolver(object): +class CargoResolver(Resolver): def __init__(self, session): self.session = session + def __str__(self): + return "cargo" + + def install(self, requirements): + from ..requirements import CargoCrateRequirement + missing = [] + for requirement in requirements: + if not isinstance(requirement, CargoCrateRequirement): + missing.append(requirement) + continue + self.session.check_call( + ["cargo", "install", requirement.crate], + user="root") + if missing: + raise MissingDependencies(missing) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + def met(self, requirement): + raise NotImplementedError(self.met) + + +class PypiResolver(Resolver): + + def __init__(self, session): + self.session = session + + def __str__(self): + return "pypi" + def install(self, requirements): from ..requirements import PythonPackageRequirement missing = [] @@ -89,11 +123,14 @@ NPM_COMMAND_PACKAGES = { } -class NpmResolver(object): +class NpmResolver(Resolver): def __init__(self, session): self.session = session + def __str__(self): + return "npm" + def install(self, requirements): from ..requirements import NodePackageRequirement missing = [] @@ -121,6 +158,9 @@ class StackedResolver(Resolver): def __init__(self, subs): self.subs = subs + def __str__(self): + return "[" + ", ".join(map(str, self.subs)) + "]" + def install(self, requirements): for sub in self.subs: try: @@ -135,7 +175,8 @@ def native_resolvers(session): return StackedResolver([ CPANResolver(session), PypiResolver(session), - NpmResolver(session)]) + NpmResolver(session), + CargoResolver(session)]) class ExplainResolver(Resolver): @@ -150,19 +191,17 @@ class ExplainResolver(Resolver): raise MissingDependencies(requirements) -class AutoResolver(Resolver): - """Automatically find out the most appropriate way to install dependencies. - """ - - def __init__(self, session): - self.session = session - - @classmethod - def from_session(cls, session): - return cls(session) - - def install(self, requirements): - raise NotImplementedError(self.install) - - def explain(self, requirements): - raise NotImplementedError(self.explain) +def auto_resolver(session): + # TODO(jelmer): if session is SchrootSession or if we're root, use apt + from .apt import AptResolver + from ..session.schroot import SchrootSession + user = session.check_output(['echo', '$USER']).decode().strip() + resolvers = [] + if isinstance(session, SchrootSession) or user == 'root': + resolvers.append(AptResolver.from_session(session)) + resolvers.extend([ + CPANResolver(session), + PypiResolver(session), + NpmResolver(session), + CargoResolver(session)]) + return StackedResolver(resolvers) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index e34be07..49d9470 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -53,10 +53,6 @@ from ..requirements import ( ) -class NoAptPackage(Exception): - """No apt package.""" - - class AptRequirement(object): def __init__(self, package, minimum_version=None): @@ -161,15 +157,13 @@ def resolve_binary_req(apt_mgr, req): def resolve_pkg_config_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")], - req.minimum_version ) if package is None: package = apt_mgr.get_package_for_paths( [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], - regex=True, - minimum_version=req.minimum_version) + regex=True) if package is not None: - return AptRequirement(package) + return AptRequirement(package, minimum_version=req.minimum_version) return None @@ -502,10 +496,7 @@ APT_REQUIREMENT_RESOLVERS = [ def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement) -> AptRequirement: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): - deb_req = rr_fn(apt_mgr, req) - if deb_req is None: - raise NoAptPackage(req) - return deb_req + return rr_fn(apt_mgr, req) raise NotImplementedError(type(req)) @@ -514,6 +505,9 @@ class AptResolver(Resolver): def __init__(self, apt): self.apt = apt + def __str__(self): + return "apt" + @classmethod def from_session(cls, session): return cls(AptManager(session)) @@ -530,10 +524,11 @@ class AptResolver(Resolver): still_missing = [] apt_requirements = [] for m in missing: - try: - apt_requirements.append(self.resolve(m)) - except NoAptPackage: + apt_req = self.resolve(m) + if apt_req is None: still_missing.append(m) + else: + apt_requirements.append(m) self.apt.install( [req.package for req in apt_requirements]) if still_missing: diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 7a1eb6c..084fa1b 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -33,6 +33,9 @@ class PlainSession(Session): def check_call(self, args): return subprocess.check_call(args) + def check_output(self, args): + return subprocess.check_output(args) + def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) diff --git a/ognibuild/test.py b/ognibuild/test.py index 8560347..750143f 100644 --- a/ognibuild/test.py +++ b/ognibuild/test.py @@ -18,13 +18,13 @@ from .buildsystem import NoBuildToolsFound -def run_test(session, buildsystems, resolver): +def run_test(session, buildsystems, resolver, fixers): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.test(session, resolver) + buildsystem.test(session, resolver, fixers) return raise NoBuildToolsFound() diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index c978008..6246c03 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -31,7 +31,7 @@ from buildlog_consultant.common import ( MissingValaPackage, ) from ..debian import apt -from ..debian.apt import AptManager +from ..debian.apt import AptManager, FileSearcher from ..debian.fix_build import ( resolve_error, versioned_package_fixers, @@ -41,6 +41,21 @@ from ..debian.fix_build import ( from breezy.tests import TestCaseWithTransport +class DummyAptSearcher(FileSearcher): + + def __init__(self, files): + self._apt_files = files + + def search_files(self, path, regex=False): + for p, pkg in sorted(self._apt_files.items()): + if regex: + if re.match(path, p): + yield pkg + else: + if path == p: + yield pkg + + class ResolveErrorTests(TestCaseWithTransport): def setUp(self): super(ResolveErrorTests, self).setUp() @@ -76,21 +91,13 @@ blah (0.1) UNRELEASED; urgency=medium ) self.tree.add(["debian", "debian/control", "debian/changelog"]) self.tree.commit("Initial commit") - self.overrideAttr(apt, "search_apt_file", self._search_apt_file) self._apt_files = {} - def _search_apt_file(self, path, regex=False): - for p, pkg in sorted(self._apt_files.items()): - if regex: - if re.match(path, p): - yield pkg - else: - if path == p: - yield pkg - def resolve(self, error, context=("build",)): from ..session.plain import PlainSession - apt = AptManager(PlainSession()) + session = PlainSession() + apt = AptManager(session) + apt._searchers = [DummyAptSearcher(self._apt_files)] context = BuildDependencyContext( self.tree, apt, @@ -98,7 +105,8 @@ blah (0.1) UNRELEASED; urgency=medium committer="ognibuild ", update_changelog=True, ) - return resolve_error(error, context, versioned_package_fixers() + apt_fixers(apt)) + fixers = versioned_package_fixers(session) + apt_fixers(apt) + return resolve_error(error, context, fixers) def get_build_deps(self): with open(self.tree.abspath("debian/control"), "r") as f: diff --git a/setup.py b/setup.py index d78a7ae..26a9cce 100755 --- a/setup.py +++ b/setup.py @@ -30,9 +30,10 @@ setup(name="ognibuild", install_requires=[ 'breezy', 'buildlog-consultant', - 'python_debian', - 'debmutate', ], + extras_require={ + 'debian': ['debmutate', 'python_debian', 'python_apt'], + }, tests_require=['python_debian', 'buildlog-consultant', 'breezy'], test_suite='ognibuild.tests.test_suite', )