From ee5a8462f315134e74e98116cf742eb3dc6ed640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 13 Feb 2021 14:50:09 +0000 Subject: [PATCH] More work on resolvers. --- ognibuild/__init__.py | 7 +- ognibuild/__main__.py | 3 +- ognibuild/buildsystem.py | 78 ++++----- ognibuild/debian/fix_build.py | 149 +++++++----------- ognibuild/dist.py | 2 +- ognibuild/fix_build.py | 14 +- ognibuild/requirements.py | 64 ++++++++ .../{resolver.py => resolver/__init__.py} | 42 +---- ognibuild/resolver/apt.py | 84 ++++++++++ ognibuild/tests/test_debian_fix_build.py | 3 + 10 files changed, 273 insertions(+), 173 deletions(-) create mode 100644 ognibuild/requirements.py rename ognibuild/{resolver.py => resolver/__init__.py} (66%) create mode 100644 ognibuild/resolver/apt.py diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index c693ada..132e417 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -18,7 +18,6 @@ import os import stat -import sys class DetailedFailure(Exception): @@ -44,9 +43,11 @@ def shebang_binary(p): class UpstreamRequirement(object): - def __init__(self, family, name): + # Name of the family of requirements - e.g. "python-package" + family: str + + def __init__(self, family): self.family = family - self.name = name class UpstreamOutput(object): diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index d7061f6..ab562ce 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -25,12 +25,12 @@ from .clean import run_clean from .dist import run_dist from .install import run_install from .resolver import ( - AptResolver, ExplainResolver, AutoResolver, NativeResolver, MissingDependencies, ) +from .resolver.apt import AptResolver from .test import run_test @@ -84,6 +84,7 @@ def main(): # noqa: C901 help="Ignore declared dependencies, follow build errors only", ) args = parser.parse_args() + logging.basicConfig(level=logging.INFO) if args.schroot: from .session.schroot import SchrootSession diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9e2d832..d36f019 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -22,7 +22,14 @@ import os import re import warnings -from . import shebang_binary, UpstreamRequirement, UpstreamOutput +from . import shebang_binary, UpstreamOutput +from .requirements import ( + BinaryRequirement, + PythonPackageRequirement, + PerlModuleRequirement, + NodePackageRequirement, + CargoCrateRequirement, + ) from .apt import UnidentifiedError from .fix_build import run_with_build_fixer @@ -66,7 +73,7 @@ class Pear(BuildSystem): self.path = path def setup(self, resolver): - resolver.install([UpstreamRequirement("binary", "pear")]) + resolver.install([BinaryRequirement("pear")]) def dist(self, session, resolver): self.setup(resolver) @@ -94,18 +101,13 @@ class SetupPy(BuildSystem): name = "setup.py" def __init__(self, path): + self.path = path from distutils.core import run_setup - self.result = run_setup(os.path.abspath(path), stop_after="init") def setup(self, resolver): - resolver.install( - [ - UpstreamRequirement("python3", "pip"), - UpstreamRequirement("binary", "python3"), - ] - ) - with open("setup.py", "r") as f: + resolver.install([PythonPackageRequirement('pip')]) + with open(self.path, "r") as f: setup_py_contents = f.read() try: with open("setup.cfg", "r") as f: @@ -114,7 +116,7 @@ class SetupPy(BuildSystem): setup_cfg_contents = "" if "setuptools" in setup_py_contents: logging.info("Reference to setuptools found, installing.") - resolver.install([UpstreamRequirement("python3", "setuptools")]) + resolver.install([PythonPackageRequirement("setuptools")]) if ( "setuptools_scm" in setup_py_contents or "setuptools_scm" in setup_cfg_contents @@ -122,9 +124,9 @@ class SetupPy(BuildSystem): logging.info("Reference to setuptools-scm found, installing.") resolver.install( [ - UpstreamRequirement("python3", "setuptools-scm"), - UpstreamRequirement("binary", "git"), - UpstreamRequirement("binary", "mercurial"), + PythonPackageRequirement("setuptools-scm"), + BinaryRequirement("git"), + BinaryRequirement("mercurial"), ] ) @@ -150,24 +152,24 @@ class SetupPy(BuildSystem): interpreter = shebang_binary("setup.py") if interpreter is not None: if interpreter in ("python3", "python2", "python"): - resolver.install([UpstreamRequirement("binary", interpreter)]) + resolver.install([BinaryRequirement(interpreter)]) else: raise ValueError("Unknown interpreter %r" % interpreter) run_with_build_fixer(session, ["./setup.py"] + args) else: # Just assume it's Python 3 - resolver.install([UpstreamRequirement("binary", "python3")]) + resolver.install([BinaryRequirement("python3")]) run_with_build_fixer(session, ["python3", "./setup.py"] + args) def get_declared_dependencies(self): for require in self.result.get_requires(): - yield "build", UpstreamRequirement("python3", require) + yield "build", PythonPackageRequirement(require) if self.result.install_requires: for require in self.result.install_requires: - yield "install", UpstreamRequirement("python3", require) + yield "install", PythonPackageRequirement(require) if self.result.tests_require: for require in self.result.tests_require: - yield "test", UpstreamRequirement("python3", require) + yield "test", PythonPackageRequirement(require) def get_declared_outputs(self): for script in self.result.scripts or []: @@ -200,8 +202,8 @@ class PyProject(BuildSystem): ) resolver.install( [ - UpstreamRequirement("python3", "venv"), - UpstreamRequirement("python3", "pip"), + PythonPackageRequirement("venv"), + PythonPackageRequirement("pip"), ] ) session.check_call(["pip3", "install", "poetry"], user="root") @@ -220,8 +222,8 @@ class SetupCfg(BuildSystem): def setup(self, resolver): resolver.install( [ - UpstreamRequirement("python3", "pep517"), - UpstreamRequirement("python3", "pip"), + PythonPackageRequirement("pep517"), + PythonPackageRequirement("pip"), ] ) @@ -244,10 +246,10 @@ class Npm(BuildSystem): if "devDependencies" in self.package: for name, unused_version in self.package["devDependencies"].items(): # TODO(jelmer): Look at version - yield "dev", UpstreamRequirement("npm", name) + yield "dev", NodePackageRequirement(name) def setup(self, resolver): - resolver.install([UpstreamRequirement("binary", "npm")]) + resolver.install([BinaryRequirement("npm")]) def dist(self, session, resolver): self.setup(resolver) @@ -262,7 +264,7 @@ class Waf(BuildSystem): self.path = path def setup(self, resolver): - resolver.install([UpstreamRequirement("binary", "python3")]) + resolver.install([BinaryRequirement("python3")]) def dist(self, session, resolver): self.setup(resolver) @@ -277,7 +279,7 @@ class Gem(BuildSystem): self.path = path def setup(self, resolver): - resolver.install([UpstreamRequirement("binary", "gem2deb")]) + resolver.install([BinaryRequirement("gem2deb")]) def dist(self, session, resolver): self.setup(resolver) @@ -314,18 +316,18 @@ class DistInkt(BuildSystem): def setup(self, resolver): resolver.install( [ - UpstreamRequirement("perl", "Dist::Inkt"), + PerlModuleRequirement("Dist::Inkt"), ] ) def dist(self, session, resolver): self.setup(resolver) if self.name == "dist-inkt": - resolver.install([UpstreamRequirement("perl-module", self.dist_inkt_class)]) + resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) run_with_build_fixer(session, ["distinkt-dist"]) else: # Default to invoking Dist::Zilla - resolver.install([UpstreamRequirement("perl", "Dist::Zilla")]) + resolver.install([PerlModuleRequirement("Dist::Zilla")]) run_with_build_fixer(session, ["dzil", "build", "--in", ".."]) @@ -335,7 +337,7 @@ class Make(BuildSystem): def setup(self, session, resolver): if session.exists("Makefile.PL") and not session.exists("Makefile"): - resolver.install([UpstreamRequirement("binary", "perl")]) + resolver.install([BinaryRequirement("perl")]) run_with_build_fixer(session, ["perl", "Makefile.PL"]) if not session.exists("Makefile") and not session.exists("configure"): @@ -357,10 +359,10 @@ class Make(BuildSystem): elif session.exists("configure.ac") or session.exists("configure.in"): resolver.install( [ - UpstreamRequirement("binary", "autoconf"), - UpstreamRequirement("binary", "automake"), - UpstreamRequirement("binary", "gettextize"), - UpstreamRequirement("binary", "libtoolize"), + BinaryRequirement("autoconf"), + BinaryRequirement("automake"), + BinaryRequirement("gettextize"), + BinaryRequirement("libtoolize"), ] ) run_with_build_fixer(session, ["autoreconf", "-i"]) @@ -370,7 +372,7 @@ class Make(BuildSystem): def dist(self, session, resolver): self.setup(session, resolver) - resolver.install([UpstreamRequirement("binary", "make")]) + resolver.install([BinaryRequirement("make")]) try: run_with_build_fixer(session, ["make", "dist"]) except UnidentifiedError as e: @@ -437,7 +439,7 @@ class Make(BuildSystem): warnings.warn("Unable to parse META.yml: %s" % e) return for require in data.get("requires", []): - yield "build", UpstreamRequirement("perl", require) + yield "build", PerlModuleRequirement(require) class Cargo(BuildSystem): @@ -454,7 +456,7 @@ class Cargo(BuildSystem): if "dependencies" in self.cargo: for name, details in self.cargo["dependencies"].items(): # TODO(jelmer): Look at details['features'], details['version'] - yield "build", UpstreamRequirement("cargo-crate", name) + yield "build", CargoCrateRequirement(name) class Golang(BuildSystem): diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 1a651ad..2e8848e 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -21,15 +21,13 @@ __all__ = [ import logging import os -import re import subprocess import sys -from typing import Iterator, List, Callable, Type, Tuple, Set, Optional +from typing import List, Callable, Type, Tuple, Set, Optional from debian.deb822 import ( Deb822, PkgRelation, - Release, ) from debian.changelog import Version @@ -113,6 +111,11 @@ from buildlog_consultant.sbuild import ( SbuildFailure, ) +from ..apt import AptManager, LocalAptManager +from ..resolver.apt import AptResolver +from ..requirements import BinaryRequirement +from .build import attempt_build + DEFAULT_MAX_ITERATIONS = 10 @@ -128,15 +131,21 @@ class DependencyContext(object): def __init__( self, tree: MutableTree, + apt: AptManager, subpath: str = "", committer: Optional[str] = None, update_changelog: bool = True, ): self.tree = tree + self.apt = apt + self.resolver = AptResolver(apt) self.subpath = subpath self.committer = committer self.update_changelog = update_changelog + def resolve_apt(self, req): + return self.resolver.resolve(req) + def add_dependency( self, package: str, minimum_version: Optional[Version] = None ) -> bool: @@ -157,11 +166,11 @@ class BuildDependencyContext(DependencyContext): class AutopkgtestDependencyContext(DependencyContext): def __init__( - self, testname, tree, subpath="", committer=None, update_changelog=True + self, testname, tree, apt, subpath="", committer=None, update_changelog=True ): self.testname = testname super(AutopkgtestDependencyContext, self).__init__( - tree, subpath, committer, update_changelog + tree, apt, subpath, committer, update_changelog ) def add_dependency(self, package, minimum_version=None): @@ -301,27 +310,7 @@ def commit_debian_changes( return True -def get_package_for_paths(paths, regex=False): - from .apt import search_apt_file - candidates = set() - for path in paths: - candidates.update(search_apt_file(path, regex=regex)) - if candidates: - break - if len(candidates) == 0: - logging.warning("No packages found that contain %r", paths) - return None - if len(candidates) > 1: - logging.warning( - "More than 1 packages found that contain %r: %r", path, candidates - ) - # Euhr. Pick the one with the shortest name? - return sorted(candidates, key=len)[0] - else: - return candidates.pop() - - -def get_package_for_python_module(module, python_version): +def get_package_for_python_module(apt, module, python_version): if python_version == "python3": paths = [ os.path.join( @@ -374,7 +363,7 @@ def get_package_for_python_module(module, python_version): ] else: raise AssertionError("unknown python version %r" % python_version) - return get_package_for_paths(paths, regex=True) + return apt.get_package_for_paths(paths, regex=True) def targeted_python_versions(tree: Tree) -> Set[str]: @@ -394,23 +383,8 @@ def targeted_python_versions(tree: Tree) -> Set[str]: return targeted -apt_cache = None - - -def package_exists(package): - global apt_cache - if apt_cache is None: - import apt_pkg - - apt_cache = apt_pkg.Cache() - for p in apt_cache.packages: - if p.name == package: - return True - return False - - def fix_missing_javascript_runtime(error, context): - package = get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False) + package = context.apt.get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False) if package is None: return False return context.add_dependency(package) @@ -420,30 +394,30 @@ def fix_missing_python_distribution(error, context): # noqa: C901 targeted = targeted_python_versions(context.tree) default = not targeted - pypy_pkg = get_package_for_paths( + pypy_pkg = context.apt.get_package_for_paths( ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True ) if pypy_pkg is None: pypy_pkg = "pypy-%s" % error.distribution - if not package_exists(pypy_pkg): + if not context.apt.package_exists(pypy_pkg): pypy_pkg = None - py2_pkg = get_package_for_paths( + py2_pkg = context.apt.get_package_for_paths( ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution], regex=True, ) if py2_pkg is None: py2_pkg = "python-%s" % error.distribution - if not package_exists(py2_pkg): + if not context.apt.package_exists(py2_pkg): py2_pkg = None - py3_pkg = get_package_for_paths( + py3_pkg = context.apt.get_package_for_paths( ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution], regex=True, ) if py3_pkg is None: py3_pkg = "python3-%s" % error.distribution - if not package_exists(py3_pkg): + if not context.apt.package_exists(py3_pkg): py3_pkg = None extra_build_deps = [] @@ -488,9 +462,9 @@ def fix_missing_python_module(error, context): targeted = set() default = not targeted - pypy_pkg = get_package_for_python_module(error.module, "pypy") - py2_pkg = get_package_for_python_module(error.module, "python2") - py3_pkg = get_package_for_python_module(error.module, "python3") + pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy") + py2_pkg = get_package_for_python_module(context.apt, error.module, "python2") + py3_pkg = get_package_for_python_module(context.apt, error.module, "python3") extra_build_deps = [] if error.python_version == 2: @@ -528,7 +502,7 @@ def fix_missing_python_module(error, context): def fix_missing_go_package(error, context): - package = get_package_for_paths( + package = context.apt.get_package_for_paths( [os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True ) if package is None: @@ -537,11 +511,11 @@ def fix_missing_go_package(error, context): def fix_missing_c_header(error, context): - package = get_package_for_paths( + package = context.apt.get_package_for_paths( [os.path.join("/usr/include", error.header)], regex=False ) if package is None: - package = get_package_for_paths( + package = context.apt.get_package_for_paths( [os.path.join("/usr/include", ".*", error.header)], regex=True ) if package is None: @@ -550,11 +524,11 @@ def fix_missing_c_header(error, context): def fix_missing_pkg_config(error, context): - package = get_package_for_paths( + package = context.apt.get_package_for_paths( [os.path.join("/usr/lib/pkgconfig", error.module + ".pc")] ) if package is None: - package = get_package_for_paths( + package = context.apt.get_package_for_paths( [os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")], regex=True, ) @@ -564,21 +538,12 @@ def fix_missing_pkg_config(error, context): def fix_missing_command(error, context): - if os.path.isabs(error.command): - paths = [error.command] - else: - paths = [ - os.path.join(dirname, error.command) for dirname in ["/usr/bin", "/bin"] - ] - package = get_package_for_paths(paths) - if package is None: - logging.info("No packages found that contain %r", paths) - return False + package = context.resolve_apt(BinaryRequirement(error.command)) return context.add_dependency(package) def fix_missing_file(error, context): - package = get_package_for_paths([error.path]) + package = context.apt.get_package_for_paths([error.path]) if package is None: return False return context.add_dependency(package) @@ -590,7 +555,7 @@ def fix_missing_sprockets_file(error, context): else: logging.warning("unable to handle content type %s", error.content_type) return False - package = get_package_for_paths([path], regex=True) + package = context.apt.get_package_for_paths([path], regex=True) if package is None: return False return context.add_dependency(package) @@ -619,7 +584,7 @@ def fix_missing_perl_file(error, context): paths = [error.filename] else: paths = [os.path.join(inc, error.filename) for inc in error.inc] - package = get_package_for_paths(paths, regex=False) + package = context.apt.get_package_for_paths(paths, regex=False) if package is None: if getattr(error, "module", None): logging.warning( @@ -635,17 +600,17 @@ def fix_missing_perl_file(error, context): return context.add_dependency(package) -def get_package_for_node_package(node_package): +def get_package_for_node_package(apt, node_package): paths = [ "/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package, "/usr/lib/nodejs/%s/package.json" % node_package, "/usr/share/nodejs/%s/package.json" % node_package, ] - return get_package_for_paths(paths, regex=True) + return apt.get_package_for_paths(paths, regex=True) def fix_missing_node_module(error, context): - package = get_package_for_node_package(error.module) + package = get_package_for_node_package(context.apt, error.module) if package is None: logging.warning("no node package found for %s.", error.module) return False @@ -654,7 +619,7 @@ def fix_missing_node_module(error, context): def fix_missing_dh_addon(error, context): paths = [os.path.join("/usr/share/perl5", error.path)] - package = get_package_for_paths(paths) + package = context.apt.get_package_for_paths(paths) if package is None: logging.warning("no package for debhelper addon %s", error.name) return False @@ -667,7 +632,7 @@ def retry_apt_failure(error, context): def fix_missing_php_class(error, context): path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/") - package = get_package_for_paths([path]) + package = context.apt.get_package_for_paths([path]) if package is None: logging.warning("no package for PHP class %s", error.php_class) return False @@ -676,7 +641,7 @@ def fix_missing_php_class(error, context): def fix_missing_jdk_file(error, context): path = error.jdk_path + ".*/" + error.filename - package = get_package_for_paths([path], regex=True) + package = context.apt.get_package_for_paths([path], regex=True) if package is None: logging.warning( "no package found for %s (JDK: %s) - regex %s", @@ -690,7 +655,7 @@ def fix_missing_jdk_file(error, context): def fix_missing_vala_package(error, context): path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package - package = get_package_for_paths([path], regex=True) + package = context.apt.get_package_for_paths([path], regex=True) if package is None: logging.warning("no file found for package %s - regex %s", error.package, path) return False @@ -710,7 +675,7 @@ def fix_missing_xml_entity(error, context): else: return False - package = get_package_for_paths([search_path], regex=False) + package = context.apt.get_package_for_paths([search_path], regex=False) if package is None: return False return context.add_dependency(package) @@ -723,7 +688,7 @@ def fix_missing_library(error, context): os.path.join("/usr/lib/lib%s.a$" % error.library), os.path.join("/usr/lib/.*/lib%s.a$" % error.library), ] - package = get_package_for_paths(paths, regex=True) + package = context.apt.get_package_for_paths(paths, regex=True) if package is None: logging.warning("no package for library %s", error.library) return False @@ -737,7 +702,7 @@ def fix_missing_ruby_gem(error, context): "specifications/%s-.*\\.gemspec" % error.gem ) ] - package = get_package_for_paths(paths, regex=True) + package = context.apt.get_package_for_paths(paths, regex=True) if package is None: logging.warning("no package for gem %s", error.gem) return False @@ -746,7 +711,7 @@ def fix_missing_ruby_gem(error, context): def fix_missing_ruby_file(error, context): paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)] - package = get_package_for_paths(paths) + package = context.apt.get_package_for_paths(paths) if package is not None: return context.add_dependency(package) paths = [ @@ -755,7 +720,7 @@ def fix_missing_ruby_file(error, context): "lib/%s.rb" % error.filename ) ] - package = get_package_for_paths(paths, regex=True) + package = context.apt.get_package_for_paths(paths, regex=True) if package is not None: return context.add_dependency(package) @@ -765,7 +730,7 @@ def fix_missing_ruby_file(error, context): def fix_missing_r_package(error, context): paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)] - package = get_package_for_paths(paths, regex=True) + package = context.apt.get_package_for_paths(paths, regex=True) if package is None: logging.warning("no package for R package %s", error.package) return False @@ -781,7 +746,7 @@ def fix_missing_java_class(error, context): logging.warning("unable to find classpath for %s", error.classname) return False logging.info("Classpath for %s: %r", error.classname, classpath) - package = get_package_for_paths(classpath) + package = context.apt.get_package_for_paths(classpath) if package is None: logging.warning("no package for files in %r", classpath) return False @@ -849,7 +814,7 @@ def fix_missing_maven_artifacts(error, context): "%s-%s.%s" % (artifact_id, version, kind), ) ] - package = get_package_for_paths(paths, regex=regex) + package = context.apt.get_package_for_paths(paths, regex=regex) if package is None: logging.warning("no package for artifact %s", artifact) return False @@ -862,7 +827,7 @@ def install_gnome_common(error, context): def install_gnome_common_dep(error, context): if error.package == "glib-gettext": - package = get_package_for_paths(["/usr/bin/glib-gettextize"]) + package = context.apt.get_package_for_paths(["/usr/bin/glib-gettextize"]) else: package = None if package is None: @@ -875,7 +840,7 @@ def install_gnome_common_dep(error, context): def install_xfce_dep(error, context): if error.package == "gtk-doc": - package = get_package_for_paths(["/usr/bin/gtkdocize"]) + package = context.apt.get_package_for_paths(["/usr/bin/gtkdocize"]) else: package = None if package is None: @@ -947,7 +912,7 @@ def fix_missing_autoconf_macro(error, context): except KeyError: logging.info("No local m4 file found defining %s", error.macro) return False - package = get_package_for_paths([path]) + package = context.apt.get_package_for_paths([path]) if package is None: logging.warning("no package for macro file %s", path) return False @@ -960,7 +925,7 @@ def fix_missing_c_sharp_compiler(error, context): def fix_missing_haskell_dependencies(error, context): path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0] - package = get_package_for_paths([path], regex=True) + package = context.apt.get_package_for_paths([path], regex=True) if package is None: logging.warning("no package for macro file %s", path) return False @@ -1033,6 +998,7 @@ def resolve_error(error, context, fixers): def build_incrementally( local_tree, + apt, suffix, build_suite, output_directory, @@ -1074,6 +1040,7 @@ def build_incrementally( if e.context[0] == "build": context = BuildDependencyContext( local_tree, + apt, subpath=subpath, committer=committer, update_changelog=update_changelog, @@ -1082,6 +1049,7 @@ def build_incrementally( context = AutopkgtestDependencyContext( e.context[1], local_tree, + apt, subpath=subpath, committer=committer, update_changelog=update_changelog, @@ -1154,9 +1122,12 @@ def main(argv=None): args = parser.parse_args() from breezy.workingtree import WorkingTree + apt = LocalAptManager() + tree = WorkingTree.open(".") build_incrementally( tree, + apt, args.suffix, args.suite, args.output_directory, diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 3b47bec..d226e0f 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -124,7 +124,7 @@ def create_dist_schroot( subdir: Optional[str] = None, ) -> str: from .buildsystem import detect_buildsystems - from .resolver import AptResolver + from .resolver.apt import AptResolver if subdir is None: subdir = "package" diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 9b02ed6..c393164 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -22,6 +22,7 @@ from buildlog_consultant.common import ( find_build_failure_description, Problem, MissingPerlModule, + MissingPythonDistribution, MissingCommand, ) @@ -69,10 +70,16 @@ def fix_npm_missing_command(error, context): return True +def fix_python_package_from_pip(error, context): + context.session.check_call(["pip", "install", error.distribution]) + return True + + GENERIC_INSTALL_FIXERS: List[ Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]] ] = [ (MissingPerlModule, fix_perl_module_from_cpan), + (MissingPythonDistribution, fix_python_package_from_pip), (MissingCommand, fix_npm_missing_command), ] @@ -84,11 +91,12 @@ def run_with_build_fixer(session: Session, args: List[str]): retcode, lines = run_with_tee(session, args) if retcode == 0: return - offset, line, error = find_build_failure_description(lines) + match, error = find_build_failure_description(lines) if error is None: logging.warning("Build failed with unidentified error. Giving up.") - if line is not None: - raise UnidentifiedError(retcode, args, lines, secondary=(offset, line)) + if match is not None: + raise UnidentifiedError( + retcode, args, lines, secondary=(match.lineno, match.line)) raise UnidentifiedError(retcode, args, lines) logging.info("Identified error: %r", error) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py new file mode 100644 index 0000000..65bf1d5 --- /dev/null +++ b/ognibuild/requirements.py @@ -0,0 +1,64 @@ +#!/usr/bin/python +# Copyright (C) 2019-2020 Jelmer Vernooij +# encoding: utf-8 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from . import UpstreamRequirement + + +class PythonPackageRequirement(UpstreamRequirement): + + package: str + + def __init__(self, package): + super(PythonPackageRequirement, self).__init__('python-package') + self.package = package + + +class BinaryRequirement(UpstreamRequirement): + + binary_name: str + + def __init__(self, binary_name): + super(BinaryRequirement, self).__init__('binary') + self.binary_name = binary_name + + +class PerlModuleRequirement(UpstreamRequirement): + + module: str + + def __init__(self, module): + super(PerlModuleRequirement, self).__init__('perl-module') + self.module = module + + +class NodePackageRequirement(UpstreamRequirement): + + package: str + + def __init__(self, package): + super(NodePackageRequirement, self).__init__('npm-package') + self.package = package + + +class CargoCrateRequirement(UpstreamRequirement): + + crate: str + + def __init__(self, crate): + super(CargoCrateRequirement, self).__init__('cargo-crate') + self.crate = crate diff --git a/ognibuild/resolver.py b/ognibuild/resolver/__init__.py similarity index 66% rename from ognibuild/resolver.py rename to ognibuild/resolver/__init__.py index 63a473a..9384482 100644 --- a/ognibuild/resolver.py +++ b/ognibuild/resolver/__init__.py @@ -17,11 +17,13 @@ class MissingDependencies(Exception): + def __init__(self, reqs): self.requirements = reqs class Resolver(object): + def install(self, requirements): raise NotImplementedError(self.install) @@ -29,43 +31,6 @@ class Resolver(object): raise NotImplementedError(self.explain) -class AptResolver(Resolver): - def __init__(self, apt): - self.apt = apt - - @classmethod - def from_session(cls, session): - from .apt import AptManager - - return cls(AptManager(session)) - - def install(self, requirements): - missing = [] - for req in requirements: - pps = list(self._possible_paths(req)) - if not pps or not any(self.apt.session.exists(p) for p in pps): - missing.append(req) - if missing: - self.apt.install(list(self.resolve(missing))) - - def explain(self, requirements): - raise NotImplementedError(self.explain) - - def _possible_paths(self, req): - if req.family == "binary": - yield "/usr/bin/%s" % req.name - else: - return - - def resolve(self, requirements): - for req in requirements: - if req.family == "python3": - yield "python3-%s" % req.name - else: - list(self._possible_paths(req)) - raise NotImplementedError - - class NativeResolver(Resolver): def __init__(self, session): self.session = session @@ -94,7 +59,8 @@ class ExplainResolver(Resolver): class AutoResolver(Resolver): - """Automatically find out the most appropriate way to instal dependencies.""" + """Automatically find out the most appropriate way to install dependencies. + """ def __init__(self, session): self.session = session diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py new file mode 100644 index 0000000..5fe42d8 --- /dev/null +++ b/ognibuild/resolver/apt.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import posixpath + +from ..apt import AptManager + +from . import Resolver +from ..requirements import ( + BinaryRequirement, + PythonPackageRequirement, + ) + + +class NoAptPackage(Exception): + """No apt package.""" + + +def resolve_binary_req(apt_mgr, req): + if posixpath.isabs(req.binary_name): + paths = [req.binary_name] + else: + paths = [ + posixpath.join(dirname, req.binary_name) + for dirname in ["/usr/bin", "/bin"] + ] + return apt_mgr.get_package_for_paths(paths) + + +APT_REQUIREMENT_RESOLVERS = [ + (BinaryRequirement, resolve_binary_req), +] + + +class AptResolver(Resolver): + + def __init__(self, apt): + self.apt = apt + + @classmethod + def from_session(cls, session): + return cls(AptManager(session)) + + def install(self, requirements): + missing = [] + for req in requirements: + try: + pps = list(req.possible_paths()) + except NotImplementedError: + missing.append(req) + else: + if not pps or not any(self.apt.session.exists(p) for p in pps): + missing.append(req) + if missing: + self.apt.install(list(self.resolve(missing))) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + def resolve(self, requirements): + for req in requirements: + for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: + if isinstance(req, rr_class): + package_name = rr_fn(self.apt, req) + if package_name is None: + raise NoAptPackage() + yield package_name + break + else: + raise NotImplementedError diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index d95bbe3..07725f3 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -31,6 +31,7 @@ from buildlog_consultant.common import ( MissingValaPackage, ) from ..debian import apt +from ..debian.apt import LocalAptManager from ..debian.fix_build import ( resolve_error, VERSIONED_PACKAGE_FIXERS, @@ -88,8 +89,10 @@ blah (0.1) UNRELEASED; urgency=medium yield pkg def resolve(self, error, context=("build",)): + apt = LocalAptManager() context = BuildDependencyContext( self.tree, + apt, subpath="", committer="Janitor ", update_changelog=True,