From fb41c93e825c36da234fbd37e0a7b45340fa9bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 28 Feb 2021 23:04:54 +0000 Subject: [PATCH 01/21] Some more fixes. --- ognibuild/buildsystem.py | 14 +++++++++++--- ognibuild/debian/fix_build.py | 8 +++++--- ognibuild/dist.py | 2 +- ognibuild/session/plain.py | 3 +++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 60309e6..75a024b 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -114,7 +114,11 @@ class SetupPy(BuildSystem): def __init__(self, path): self.path = path from distutils.core import run_setup - self.result = run_setup(os.path.abspath(path), stop_after="init") + try: + self.result = run_setup(os.path.abspath(path), stop_after="init") + except RuntimeError as e: + logging.warning('Unable to load setup.py metadata: %s', e) + self.result = None def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) @@ -137,9 +141,9 @@ class SetupPy(BuildSystem): logging.debug("Reference to setuptools-scm found, installing.") resolver.install( [ - PythonPackageRequirement("setuptools-scm"), + PythonPackageRequirement("setuptools_scm"), BinaryRequirement("git"), - BinaryRequirement("mercurial"), + BinaryRequirement("hg"), ] ) @@ -181,6 +185,8 @@ class SetupPy(BuildSystem): fixers) def get_declared_dependencies(self): + if self.result is None: + raise NotImplementedError for require in self.result.get_requires(): yield "build", PythonPackageRequirement(require) # Not present for distutils-only packages @@ -193,6 +199,8 @@ class SetupPy(BuildSystem): yield "test", PythonPackageRequirement(require) def get_declared_outputs(self): + if self.result is None: + raise NotImplementedError for script in self.result.scripts or []: yield UpstreamOutput("binary", os.path.basename(script)) entry_points = getattr(self.result, 'entry_points', None) or {} diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 1ccc9bf..01c7d36 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -48,9 +48,11 @@ from debmutate.reformatting import ( FormattingUnpreservable, GeneratedFile, ) -from lintian_brush import ( - reset_tree, -) +try: + from breezy.workspace import reset_tree +except ImportError: + from lintian_brush import reset_tree + from lintian_brush.changelog import ( add_changelog_entry, ) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 2349fe7..e507f18 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -103,7 +103,7 @@ class DistCatcher(object): logging.info("No tarballs found in dist directory.") parent_directory = os.path.dirname(self.export_directory) - diff = set(os.listdir(parent_directory)) - set([subdir]) + diff = set(os.listdir(parent_directory)) - set([self.export_directory]) if len(diff) == 1: fn = diff.pop() logging.info("Found tarball %s in parent directory.", fn) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 084fa1b..749bf49 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -44,3 +44,6 @@ class PlainSession(Session): def scandir(self, path): return os.scandir(path) + + def chdir(self, path): + os.chdir(path) From 94d98b244de1da487b08ab8c36bf37a392bc170c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 01:20:06 +0000 Subject: [PATCH 02/21] Improve deb-fix-build. --- ognibuild/debian/fix_build.py | 38 ++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 01c7d36..acbdc0e 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -498,6 +498,9 @@ class SimpleBuildFixer(BuildFixer): self._problem_cls = problem_cls self._fn = fn + def __repr__(self): + return "%s(%r, %r)" % (type(self).__name__, self._problem_cls, self._fn) + def can_fix(self, problem): return isinstance(problem, self._problem_cls) @@ -533,7 +536,7 @@ def build_incrementally( build_suite, output_directory, build_command, - build_changelog_entry="Build for debian-janitor apt repository.", + build_changelog_entry, committer=None, max_iterations=DEFAULT_MAX_ITERATIONS, subpath="", @@ -658,19 +661,30 @@ def main(argv=None): from breezy.workingtree import WorkingTree from .apt import AptManager from ..session.plain import PlainSession + import tempfile + import contextlib apt = AptManager(PlainSession()) - tree = WorkingTree.open(".") - build_incrementally( - tree, - apt, - args.suffix, - args.suite, - args.output_directory, - args.build_command, - committer=args.committer, - update_changelog=args.update_changelog, - ) + logging.basicConfig(level=logging.INFO) + + with contextlib.ExitStack() as es: + if args.output_directory is None: + output_directory = es.enter_context(tempfile.TemporaryDirectory()) + logging.info('Using output directory %s', output_directory) + else: + output_directory = args.output_directory + + tree = WorkingTree.open(".") + build_incrementally( + tree, + apt, + args.suffix, + args.suite, + output_directory, + args.build_command, + committer=args.committer, + update_changelog=args.update_changelog, + ) if __name__ == "__main__": From 88f484c58fefc0028123825fe19364cfe5fd0bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 01:21:18 +0000 Subject: [PATCH 03/21] Don't hardcode changelog entry. --- ognibuild/debian/build.py | 9 +++++---- ognibuild/debian/fix_build.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 100f56d..da42d4e 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -207,7 +207,7 @@ def attempt_build( build_suite, output_directory, build_command, - build_changelog_entry="Build for debian-janitor apt repository.", + build_changelog_entry=None, subpath="", source_date_epoch=None, ): @@ -224,9 +224,10 @@ def attempt_build( source_date_epoch: Source date epoch to set Returns: Tuple with (changes_name, cl_version) """ - add_dummy_changelog_entry( - local_tree, subpath, suffix, build_suite, build_changelog_entry - ) + if build_changelog_entry is not None: + add_dummy_changelog_entry( + local_tree, subpath, suffix, build_suite, build_changelog_entry + ) return build_once( local_tree, build_suite, diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index acbdc0e..3ece8c7 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -682,6 +682,7 @@ def main(argv=None): args.suite, output_directory, args.build_command, + None, committer=args.committer, update_changelog=args.update_changelog, ) From e103194e1a614f079e1f67e77363a5b962c501b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 01:35:33 +0000 Subject: [PATCH 04/21] Setting logging format. --- ognibuild/__init__.py | 3 +++ ognibuild/__main__.py | 4 ++-- ognibuild/debian/fix_build.py | 2 +- ognibuild/dist.py | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 9b7b07f..762114c 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -70,3 +70,6 @@ class UpstreamOutput(object): def __repr__(self): return "%s(%r, %r)" % (type(self).__name__, self.family, self.name) + + def get_declared_dependencies(self): + raise NotImplementedError(self.get_declared_dependencies) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index f253b97..143ade7 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -110,9 +110,9 @@ def main(): # noqa: C901 parser.print_usage() return 1 if args.verbose: - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.DEBUG, format='%(message)s') else: - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.INFO, format='%(message)s') if args.schroot: from .session.schroot import SchrootSession diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 3ece8c7..15335f2 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -665,7 +665,7 @@ def main(argv=None): import contextlib apt = AptManager(PlainSession()) - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.INFO, format='%(message)s') with contextlib.ExitStack() as es: if args.output_directory is None: diff --git a/ognibuild/dist.py b/ognibuild/dist.py index e507f18..87b2da0 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -204,9 +204,9 @@ if __name__ == "__main__": args = parser.parse_args() if args.verbose: - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.DEBUG, format='%(message)s') else: - logging.basicConfig(level=logging.INFO) + logging.basicConfig(level=logging.INFO, format='%(message)s') tree = WorkingTree.open(args.directory) if args.packaging_directory: From 217e87a3a8dda1ffae95adcfe564f0e434e4b6e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 01:42:44 +0000 Subject: [PATCH 05/21] Split out binaries. --- ognibuild/__init__.py | 6 +---- ognibuild/buildsystem.py | 12 ++++++---- ognibuild/outputs.py | 47 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 ognibuild/outputs.py diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 762114c..cc0310e 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -64,12 +64,8 @@ class UpstreamRequirement(object): class UpstreamOutput(object): - def __init__(self, family, name): + def __init__(self, family): self.family = family - self.name = name - - def __repr__(self): - return "%s(%r, %r)" % (type(self).__name__, self.family, self.name) def get_declared_dependencies(self): raise NotImplementedError(self.get_declared_dependencies) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 75a024b..0211613 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -23,7 +23,11 @@ import re from typing import Optional import warnings -from . import shebang_binary, UpstreamOutput, UnidentifiedError +from . import shebang_binary, UnidentifiedError +from .outputs import ( + BinaryOutput, + PythonPackageOutput, + ) from .requirements import ( BinaryRequirement, PythonPackageRequirement, @@ -202,12 +206,12 @@ class SetupPy(BuildSystem): if self.result is None: raise NotImplementedError for script in self.result.scripts or []: - yield UpstreamOutput("binary", os.path.basename(script)) + yield BinaryOutput(os.path.basename(script)) entry_points = getattr(self.result, 'entry_points', None) or {} for script in entry_points.get("console_scripts", []): - yield UpstreamOutput("binary", script.split("=")[0]) + yield BinaryOutput(script.split("=")[0]) for package in self.result.packages or []: - yield UpstreamOutput("python3", package) + yield PythonPackageOutput(package, python_version="cpython3") class PyProject(BuildSystem): diff --git a/ognibuild/outputs.py b/ognibuild/outputs.py new file mode 100644 index 0000000..bcb305b --- /dev/null +++ b/ognibuild/outputs.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# Copyright (C) 2019-2020 Jelmer Vernooij +# encoding: utf-8 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from . import UpstreamOutput + + +class BinaryOutput(UpstreamOutput): + + def __init__(self, name): + super(BinaryOutput, self).__init__('binary') + self.name = name + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.name) + + def __str__(self): + return "binary: %s" % self.name + + +class PythonPackageOutput(UpstreamOutput): + + def __init__(self, name, python_version=None): + super(PythonPackageOutput, self).__init__('python-package') + self.name = name + self.python_version = python_version + + def __str__(self): + return "python package: %s" % self.name + + def __repr__(self): + return "%s(%r, python_version=%r)" % ( + type(self).__name__, self.name, self.python_version) From 78b59759c9240b1bab9ff94929e9d6eceaf97d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 15:00:32 +0000 Subject: [PATCH 06/21] Some renames. --- ognibuild/__init__.py | 2 +- ognibuild/__main__.py | 7 +++-- ognibuild/buildlog.py | 2 +- ognibuild/buildsystem.py | 6 ++++ ognibuild/debian/apt.py | 4 +-- ognibuild/debian/fix_build.py | 4 +-- ognibuild/dist.py | 4 +-- ognibuild/fix_build.py | 11 +++---- ognibuild/requirements.py | 56 +++++++++++++++++------------------ ognibuild/resolver/apt.py | 10 ++++--- 10 files changed, 57 insertions(+), 49 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index cc0310e..684c6c1 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -50,7 +50,7 @@ def shebang_binary(p): return os.path.basename(args[0].decode()).strip() -class UpstreamRequirement(object): +class Requirement(object): # Name of the family of requirements - e.g. "python-package" family: str diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 143ade7..b657e6b 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -62,9 +62,9 @@ STAGE_MAP = { } def determine_fixers(session, resolver): - from .buildlog import UpstreamRequirementFixer + from .buildlog import RequirementFixer from .resolver.apt import AptResolver - return [UpstreamRequirementFixer(resolver)] + return [RequirementFixer(resolver)] def main(): # noqa: C901 @@ -136,6 +136,7 @@ def main(): # noqa: C901 if not args.ignore_declared_dependencies and not args.explain: stages = STAGE_MAP[args.subcommand] if stages: + logging.info('Checking that declared requirements are present') for bs in bss: install_necessary_declared_requirements(resolver, bs, stages) fixers = determine_fixers(session, resolver) @@ -166,7 +167,7 @@ def main(): # noqa: C901 if args.subcommand == "info": from .info import run_info run_info(session, buildsystems=bss) - except UnidentifiedError: + except UnidentifiedError as e: return 1 except NoBuildToolsFound: logging.info("No build tools found.") diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 2743ff9..2401ddf 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -173,7 +173,7 @@ def problem_to_upstream_requirement(problem): return None -class UpstreamRequirementFixer(BuildFixer): +class RequirementFixer(BuildFixer): def __init__(self, resolver): self.resolver = resolver diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 0211613..5ef2e17 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -118,6 +118,8 @@ class SetupPy(BuildSystem): def __init__(self, path): self.path = path from distutils.core import run_setup + # TODO(jelmer): Perhaps run this in session, so we can install + # missing dependencies? try: self.result = run_setup(os.path.abspath(path), stop_after="init") except RuntimeError as e: @@ -421,6 +423,10 @@ class Make(BuildSystem): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "all"], fixers) + def clean(self, session, resolver, fixers): + self.setup(session, resolver, fixers) + run_with_build_fixers(session, ["make", "clean"], fixers) + def test(self, session, resolver, fixers): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "check"], fixers) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 1b33327..4ff00c1 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -39,11 +39,9 @@ def run_apt(session: Session, args: List[str]) -> None: match, error = find_apt_get_failure(lines) if error is not None: raise DetailedFailure(retcode, args, error) - if match is not None: - raise UnidentifiedError(retcode, args, lines, secondary=(match.lineno, match.line)) while lines and lines[-1] == "": lines.pop(-1) - raise UnidentifiedError(retcode, args, lines) + raise UnidentifiedError(retcode, args, lines, secondary=match) class FileSearcher(object): diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 15335f2..e6e2b2e 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -81,7 +81,7 @@ from buildlog_consultant.sbuild import ( ) from ..fix_build import BuildFixer, resolve_error, DependencyContext -from ..buildlog import UpstreamRequirementFixer +from ..buildlog import RequirementFixer from ..resolver.apt import ( AptRequirement, get_package_for_python_module, @@ -525,7 +525,7 @@ def apt_fixers(apt) -> List[BuildFixer]: SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution), SimpleBuildFixer(AptFetchFailure, retry_apt_failure), - UpstreamRequirementFixer(resolver), + RequirementFixer(resolver), ] diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 87b2da0..31baf21 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -125,7 +125,7 @@ def create_dist_schroot( ) -> str: from .buildsystem import detect_buildsystems from .resolver.apt import AptResolver - from .buildlog import UpstreamRequirementFixer + from .buildlog import RequirementFixer if subdir is None: subdir = "package" @@ -151,7 +151,7 @@ def create_dist_schroot( buildsystems = list(detect_buildsystems(export_directory)) resolver = AptResolver.from_session(session) - fixers = [UpstreamRequirementFixer(resolver)] + fixers = [RequirementFixer(resolver)] with DistCatcher(export_directory) as dc: oldcwd = os.getcwd() diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index d46016d..efea8d7 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -89,11 +89,12 @@ def run_with_build_fixers( return match, error = find_build_failure_description(lines) if error is None: - logging.warning("Build failed with unidentified error. Giving up.") - if match is not None: - raise UnidentifiedError( - retcode, args, lines, secondary=(match.lineno, match.line)) - raise UnidentifiedError(retcode, args, lines) + if match: + logging.warning("Build failed with unidentified error:") + logging.warning('%s', match.line.rstrip('\n')) + else: + logging.warning("Build failed and unable to find cause. Giving up.") + raise UnidentifiedError(retcode, args, lines, secondary=match) logging.info("Identified error: %r", error) if error in fixed_errors: diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 44c9f27..a1d752d 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -19,10 +19,10 @@ import posixpath from typing import Optional, List, Tuple -from . import UpstreamRequirement +from . import Requirement -class PythonPackageRequirement(UpstreamRequirement): +class PythonPackageRequirement(Requirement): package: str @@ -41,7 +41,7 @@ class PythonPackageRequirement(UpstreamRequirement): return "python package: %s" % self.package -class BinaryRequirement(UpstreamRequirement): +class BinaryRequirement(Requirement): binary_name: str @@ -50,7 +50,7 @@ class BinaryRequirement(UpstreamRequirement): self.binary_name = binary_name -class PerlModuleRequirement(UpstreamRequirement): +class PerlModuleRequirement(Requirement): module: str filename: Optional[str] @@ -66,7 +66,7 @@ class PerlModuleRequirement(UpstreamRequirement): return self.module.replace("::", "/") + ".pm" -class NodePackageRequirement(UpstreamRequirement): +class NodePackageRequirement(Requirement): package: str @@ -75,7 +75,7 @@ class NodePackageRequirement(UpstreamRequirement): self.package = package -class CargoCrateRequirement(UpstreamRequirement): +class CargoCrateRequirement(Requirement): crate: str @@ -84,7 +84,7 @@ class CargoCrateRequirement(UpstreamRequirement): self.crate = crate -class PkgConfigRequirement(UpstreamRequirement): +class PkgConfigRequirement(Requirement): module: str @@ -94,7 +94,7 @@ class PkgConfigRequirement(UpstreamRequirement): self.minimum_version = minimum_version -class PathRequirement(UpstreamRequirement): +class PathRequirement(Requirement): path: str @@ -103,7 +103,7 @@ class PathRequirement(UpstreamRequirement): self.path = path -class CHeaderRequirement(UpstreamRequirement): +class CHeaderRequirement(Requirement): header: str @@ -112,14 +112,14 @@ class CHeaderRequirement(UpstreamRequirement): self.header = header -class JavaScriptRuntimeRequirement(UpstreamRequirement): +class JavaScriptRuntimeRequirement(Requirement): def __init__(self): super(JavaScriptRuntimeRequirement, self).__init__( 'javascript-runtime') -class ValaPackageRequirement(UpstreamRequirement): +class ValaPackageRequirement(Requirement): package: str @@ -128,7 +128,7 @@ class ValaPackageRequirement(UpstreamRequirement): self.package = package -class RubyGemRequirement(UpstreamRequirement): +class RubyGemRequirement(Requirement): gem: str minimum_version: Optional[str] @@ -139,7 +139,7 @@ class RubyGemRequirement(UpstreamRequirement): self.minimum_version = minimum_version -class GoPackageRequirement(UpstreamRequirement): +class GoPackageRequirement(Requirement): package: str @@ -148,7 +148,7 @@ class GoPackageRequirement(UpstreamRequirement): self.package = package -class DhAddonRequirement(UpstreamRequirement): +class DhAddonRequirement(Requirement): path: str @@ -157,7 +157,7 @@ class DhAddonRequirement(UpstreamRequirement): self.path = path -class PhpClassRequirement(UpstreamRequirement): +class PhpClassRequirement(Requirement): php_class: str @@ -166,7 +166,7 @@ class PhpClassRequirement(UpstreamRequirement): self.php_class = php_class -class RPackageRequirement(UpstreamRequirement): +class RPackageRequirement(Requirement): package: str minimum_version: Optional[str] @@ -177,7 +177,7 @@ class RPackageRequirement(UpstreamRequirement): self.minimum_version = minimum_version -class LibraryRequirement(UpstreamRequirement): +class LibraryRequirement(Requirement): library: str @@ -186,7 +186,7 @@ class LibraryRequirement(UpstreamRequirement): self.library = library -class RubyFileRequirement(UpstreamRequirement): +class RubyFileRequirement(Requirement): filename: str @@ -195,7 +195,7 @@ class RubyFileRequirement(UpstreamRequirement): self.filename = filename -class XmlEntityRequirement(UpstreamRequirement): +class XmlEntityRequirement(Requirement): url: str @@ -204,7 +204,7 @@ class XmlEntityRequirement(UpstreamRequirement): self.url = url -class SprocketsFileRequirement(UpstreamRequirement): +class SprocketsFileRequirement(Requirement): content_type: str name: str @@ -215,7 +215,7 @@ class SprocketsFileRequirement(UpstreamRequirement): self.name = name -class JavaClassRequirement(UpstreamRequirement): +class JavaClassRequirement(Requirement): classname: str @@ -224,7 +224,7 @@ class JavaClassRequirement(UpstreamRequirement): self.classname = classname -class HaskellPackageRequirement(UpstreamRequirement): +class HaskellPackageRequirement(Requirement): package: str @@ -233,7 +233,7 @@ class HaskellPackageRequirement(UpstreamRequirement): self.package = package -class MavenArtifactRequirement(UpstreamRequirement): +class MavenArtifactRequirement(Requirement): artifacts: List[Tuple[str, str, str]] @@ -242,13 +242,13 @@ class MavenArtifactRequirement(UpstreamRequirement): self.artifacts = artifacts -class GnomeCommonRequirement(UpstreamRequirement): +class GnomeCommonRequirement(Requirement): def __init__(self): super(GnomeCommonRequirement, self).__init__('gnome-common') -class JDKFileRequirement(UpstreamRequirement): +class JDKFileRequirement(Requirement): jdk_path: str filename: str @@ -263,7 +263,7 @@ class JDKFileRequirement(UpstreamRequirement): return posixpath.join(self.jdk_path, self.filename) -class PerlFileRequirement(UpstreamRequirement): +class PerlFileRequirement(Requirement): filename: str @@ -272,7 +272,7 @@ class PerlFileRequirement(UpstreamRequirement): self.filename = filename -class AutoconfMacroRequirement(UpstreamRequirement): +class AutoconfMacroRequirement(Requirement): macro: str @@ -281,7 +281,7 @@ class AutoconfMacroRequirement(UpstreamRequirement): self.macro = macro -class PythonModuleRequirement(UpstreamRequirement): +class PythonModuleRequirement(Requirement): module: str python_version: Optional[str] diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 4439bfa..17c2968 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -23,11 +23,12 @@ from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements from ..requirements import ( + Requirement, BinaryRequirement, CHeaderRequirement, PkgConfigRequirement, PathRequirement, - UpstreamRequirement, + Requirement, JavaScriptRuntimeRequirement, ValaPackageRequirement, RubyGemRequirement, @@ -53,9 +54,10 @@ from ..requirements import ( ) -class AptRequirement(object): +class AptRequirement(Requirement): def __init__(self, package, minimum_version=None): + super(AptRequirement, self).__init__('apt') self.package = package self.minimum_version = minimum_version @@ -493,7 +495,7 @@ APT_REQUIREMENT_RESOLVERS = [ ] -def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement) -> AptRequirement: +def resolve_requirement_apt(apt_mgr, req: Requirement) -> AptRequirement: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): return rr_fn(apt_mgr, req) @@ -538,5 +540,5 @@ class AptResolver(Resolver): def explain(self, requirements): raise NotImplementedError(self.explain) - def resolve(self, req: UpstreamRequirement): + def resolve(self, req: Requirement): return resolve_requirement_apt(self.apt, req) From a8663e0eaa42c8d44d66911d167125246cfa3d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 15:22:28 +0000 Subject: [PATCH 07/21] More complex python requirement parsing. --- ognibuild/buildsystem.py | 6 +++--- ognibuild/requirements.py | 23 ++++++++++++++++++----- setup.py | 1 + 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 5ef2e17..de153c8 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -194,15 +194,15 @@ class SetupPy(BuildSystem): if self.result is None: raise NotImplementedError for require in self.result.get_requires(): - yield "build", PythonPackageRequirement(require) + yield "build", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages if getattr(self.result, 'install_requires', []): for require in self.result.install_requires: - yield "install", PythonPackageRequirement(require) + yield "install", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages if getattr(self.result, 'tests_require', []): for require in self.result.tests_require: - yield "test", PythonPackageRequirement(require) + yield "test", PythonPackageRequirement.from_requirement_str(require) def get_declared_outputs(self): if self.result is None: diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index a1d752d..a523246 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -26,19 +26,32 @@ class PythonPackageRequirement(Requirement): package: str - def __init__(self, package, python_version=None, minimum_version=None): + def __init__(self, package, python_version=None, specs=None, + minimum_version=None): super(PythonPackageRequirement, self).__init__('python-package') self.package = package self.python_version = python_version - self.minimum_version = minimum_version + if minimum_version is not None: + specs = [('>=', minimum_version)] + self.specs = specs def __repr__(self): - return "%s(%r, python_version=%r, minimum_version=%r)" % ( + return "%s(%r, python_version=%r, specs=%r)" % ( type(self).__name__, self.package, self.python_version, - self.minimum_version) + self.specs) def __str__(self): - return "python package: %s" % self.package + if self.specs: + return "python package: %s (%r)" % (self.package, self.specs) + else: + return "python package: %s" % (self.package, ) + + + @classmethod + def from_requirement_str(cls, text): + from requirements.requirement import Requirement + req = Requirement.parse(text) + return cls(package=req.name, specs=req.specs) class BinaryRequirement(Requirement): diff --git a/setup.py b/setup.py index 26a9cce..f8d358e 100755 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ setup(name="ognibuild", install_requires=[ 'breezy', 'buildlog-consultant', + 'requirements-parser', ], extras_require={ 'debian': ['debmutate', 'python_debian', 'python_apt'], From 693b6382ae5c28a1d76f9a4593e1604631f7c146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 16:28:23 +0000 Subject: [PATCH 08/21] Python specs. --- ognibuild/debian/fix_build.py | 11 ++++++++--- ognibuild/resolver/apt.py | 28 ++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index e6e2b2e..d20302a 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -349,9 +349,14 @@ def fix_missing_python_module(error, context): targeted = set() default = not targeted - pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", None) - py2_pkg = get_package_for_python_module(context.apt, error.module, "python2", None) - py3_pkg = get_package_for_python_module(context.apt, error.module, "python3", None) + if error.minimum_version: + specs = [('>=', error.minimum_version)] + else: + specs = [] + + pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", specs) + py2_pkg = get_package_for_python_module(context.apt, error.module, "python2", specs) + py3_pkg = get_package_for_python_module(context.apt, error.module, "python3", specs) extra_build_deps = [] if error.python_version == 2: diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 17c2968..50cf9f6 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -62,7 +62,7 @@ class AptRequirement(Requirement): self.minimum_version = minimum_version -def get_package_for_python_package(apt_mgr, package, python_version, minimum_version=None): +def get_package_for_python_package(apt_mgr, package, python_version, specs=None): if python_version == "pypy": pkg_name = apt_mgr.get_package_for_paths( ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package], @@ -78,12 +78,20 @@ def get_package_for_python_package(apt_mgr, package, python_version, minimum_ver else: raise NotImplementedError # TODO(jelmer): Dealing with epoch, etc? + if not specs: + minimum_version = None + else: + for spec in specs: + if spec[0] == '>=': + minimum_version = spec[1] + else: + raise NotImplementedError(spec) if pkg_name is not None: return AptRequirement(pkg_name, minimum_version) return None -def get_package_for_python_module(apt_mgr, module, python_version, minimum_version): +def get_package_for_python_module(apt_mgr, module, python_version, specs): if python_version == "python3": paths = [ posixpath.join( @@ -136,6 +144,14 @@ def get_package_for_python_module(apt_mgr, module, python_version, minimum_versi ] else: raise AssertionError("unknown python version %r" % python_version) + if not specs: + minimum_version = None + else: + for spec in specs: + if spec[0] == '>=': + minimum_version = spec[1] + else: + raise NotImplementedError(spec) pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: return AptRequirement(pkg_name, minimum_version=minimum_version) @@ -449,18 +465,18 @@ def resolve_autoconf_macro_req(apt_mgr, req): def resolve_python_module_req(apt_mgr, req): if req.python_version == 2: - return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.minimum_version) + return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.specs) elif req.python_version in (None, 3): - return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.minimum_version) + return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.specs) else: return None def resolve_python_package_req(apt_mgr, req): if req.python_version == 2: - return get_package_for_python_package(apt_mgr, req.package, "cpython2", req.minimum_version) + return get_package_for_python_package(apt_mgr, req.package, "cpython2", req.specs) elif req.python_version in (None, 3): - return get_package_for_python_package(apt_mgr, req.package, "cpython3", req.minimum_version) + return get_package_for_python_package(apt_mgr, req.package, "cpython3", req.specs) else: return None From 8955497adf088a760b7e85a59efc966d75ed9f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 17:14:53 +0000 Subject: [PATCH 09/21] Better AptRequirement management. --- ognibuild/__main__.py | 13 +++- ognibuild/buildsystem.py | 6 +- ognibuild/debian/fix_build.py | 43 ++++---------- ognibuild/dist.py | 9 +++ ognibuild/resolver/apt.py | 108 +++++++++++++++++++--------------- 5 files changed, 94 insertions(+), 85 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index b657e6b..d5f5ef4 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -52,15 +52,22 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages): resolver.install(missing) +# Types of dependencies: +# - core: necessary to do anything with the package +# - build: necessary to build the package +# - test: necessary to run the tests +# - dev: necessary for development (e.g. linters, yacc) + STAGE_MAP = { "dist": [], "info": [], - "install": ["build"], - "test": ["test", "dev"], - "build": ["build"], + "install": ["core", "build"], + "test": ["test", "build", "core"], + "build": ["build", "core"], "clean": [], } + def determine_fixers(session, resolver): from .buildlog import RequirementFixer from .resolver.apt import AptResolver diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index de153c8..2848f18 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -194,11 +194,11 @@ class SetupPy(BuildSystem): if self.result is None: raise NotImplementedError for require in self.result.get_requires(): - yield "build", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages if getattr(self.result, 'install_requires', []): for require in self.result.install_requires: - yield "install", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages if getattr(self.result, 'tests_require', []): for require in self.result.tests_require: @@ -504,6 +504,8 @@ class Make(BuildSystem): return for require in data.get("requires", []): yield "build", PerlModuleRequirement(require) + else: + raise NotImplementedError class Cargo(BuildSystem): diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index d20302a..2cfa55d 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -34,8 +34,7 @@ from breezy.commit import PointlessCommit from breezy.mutabletree import MutableTree from breezy.tree import Tree from debmutate.control import ( - ensure_some_version, - ensure_minimum_version, + ensure_relation, ControlEditor, ) from debmutate.debhelper import ( @@ -147,24 +146,14 @@ def add_build_dependency( for binary in updater.binaries: if binary["Package"] == requirement.package: raise CircularDependency(requirement.package) - if requirement.minimum_version: - updater.source["Build-Depends"] = ensure_minimum_version( + updater.source["Build-Depends"] = ensure_relation( updater.source.get("Build-Depends", ""), - requirement.package, requirement.minimum_version - ) - else: - updater.source["Build-Depends"] = ensure_some_version( - updater.source.get("Build-Depends", ""), - requirement.package - ) + requirement.relations) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False - if requirement.minimum_version: - desc = "%s (>= %s)" % (requirement.package, requirement.minimum_version) - else: - desc = requirement.package + desc = PkgRelation.str(requirement.relations) if not updater.changed: logging.info("Giving up; dependency %s was already present.", desc) @@ -204,26 +193,16 @@ def add_test_dependency( command_counter += 1 if name != testname: continue - if requirement.minimum_version: - control["Depends"] = ensure_minimum_version( - control.get("Depends", ""), - requirement.package, requirement.minimum_version - ) - else: - control["Depends"] = ensure_some_version( - control.get("Depends", ""), requirement.package - ) + control["Depends"] = ensure_relation( + control.get("Depends", ""), + requirement.relations) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False if not updater.changed: return False - if requirement.minimum_version: - desc = "%s (>= %s)" % ( - requirement.package, requirement.minimum_version) - else: - desc = requirement.package + desc = PkgRelation.str(requirement.relations) logging.info("Adding dependency to test %s: %s", testname, desc) return commit_debian_changes( @@ -336,7 +315,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 for dep_pkg in extra_build_deps: assert dep_pkg is not None if not context.add_dependency( - AptRequirement( + AptRequirement.simple( dep_pkg.package, minimum_version=error.minimum_version)): return False return True @@ -389,7 +368,7 @@ def fix_missing_python_module(error, context): for dep_pkg in extra_build_deps: assert dep_pkg is not None if not context.add_dependency( - AptRequirement(dep_pkg.package, error.minimum_version)): + AptRequirement.simple(dep_pkg.package, error.minimum_version)): return False return True @@ -412,7 +391,7 @@ def enable_dh_autoreconf(context): return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return context.add_dependency(AptRequirement("dh-autoreconf")) + return context.add_dependency(AptRequirement.simple("dh-autoreconf")) return False diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 31baf21..9c97e0e 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -21,6 +21,7 @@ import os import shutil import sys import tempfile +import time from typing import Optional from debian.deb822 import Deb822 @@ -80,6 +81,7 @@ class DistCatcher(object): self.export_directory = directory self.files = [] self.existing_files = None + self.start_time = time.time() def __enter__(self): self.existing_files = os.listdir(self.export_directory) @@ -110,6 +112,13 @@ class DistCatcher(object): self.files.append(os.path.join(parent_directory, fn)) return fn + if "dist" in new_files: + for entry in os.scandir(os.path.join(self.export_directory, "dist")): + if is_dist_file(entry.name) and entry.stat().st_mtime > self.start_time: + logging.info("Found tarball %s in dist directory.", entry.name) + self.files.append(entry.path) + return entry.name + def __exit__(self, exc_type, exc_val, exc_tb): self.find_files() return False diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 50cf9f6..6d470cc 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -15,10 +15,14 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from itertools import chain import logging import os import posixpath +from debian.changelog import Version +from debian.deb822 import PkgRelation + from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements @@ -56,10 +60,20 @@ from ..requirements import ( class AptRequirement(Requirement): - def __init__(self, package, minimum_version=None): + def __init__(self, relations): super(AptRequirement, self).__init__('apt') - self.package = package - self.minimum_version = minimum_version + self.relations = relations + + @classmethod + def simple(cls, package, minimum_version=None): + rel = {'name': package} + if minimum_version is not None: + rel['version'] = ('>=', minimum_version) + return cls([[rel]]) + + @classmethod + def from_str(cls, text): + return cls(PkgRelation.parse_relations(text)) def get_package_for_python_package(apt_mgr, package, python_version, specs=None): @@ -77,18 +91,16 @@ def get_package_for_python_package(apt_mgr, package, python_version, specs=None) regex=True) else: raise NotImplementedError + if pkg_name is None: + return None # TODO(jelmer): Dealing with epoch, etc? if not specs: - minimum_version = None + rels = [[{'name': pkg_name}]] else: + rels = [] for spec in specs: - if spec[0] == '>=': - minimum_version = spec[1] - else: - raise NotImplementedError(spec) - if pkg_name is not None: - return AptRequirement(pkg_name, minimum_version) - return None + rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}]) + return AptRequirement(rels) def get_package_for_python_module(apt_mgr, module, python_version, specs): @@ -144,18 +156,17 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): ] else: raise AssertionError("unknown python version %r" % python_version) - if not specs: - minimum_version = None - else: - for spec in specs: - if spec[0] == '>=': - minimum_version = spec[1] - else: - raise NotImplementedError(spec) pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement(pkg_name, minimum_version=minimum_version) - return None + if pkg_name is None: + return None + rels = [] + if not specs: + rels = [[{'name': pkg_name}]] + else: + rels = [] + for spec in specs: + rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}]) + return AptRequirement(rels) def resolve_binary_req(apt_mgr, req): @@ -168,7 +179,7 @@ def resolve_binary_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -181,14 +192,14 @@ def resolve_pkg_config_req(apt_mgr, req): [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], regex=True) if package is not None: - return AptRequirement(package, minimum_version=req.minimum_version) + return AptRequirement.simple(package, minimum_version=req.minimum_version) return None def resolve_path_req(apt_mgr, req): package = apt_mgr.get_package_for_paths([req.path]) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -202,14 +213,14 @@ def resolve_c_header_req(apt_mgr, req): ) if package is None: return None - return AptRequirement(package) + return AptRequirement.simple(package) def resolve_js_runtime_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( ["/usr/bin/node", "/usr/bin/duk"], regex=False) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -217,7 +228,7 @@ def resolve_vala_package_req(apt_mgr, req): path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package package = apt_mgr.get_package_for_paths([path], regex=True) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -231,7 +242,7 @@ def resolve_ruby_gem_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( paths, regex=True) if package is not None: - return AptRequirement(package, minimum_version=req.minimum_version) + return AptRequirement.simple(package, minimum_version=req.minimum_version) return None @@ -241,7 +252,7 @@ def resolve_go_package_req(apt_mgr, req): regex=True ) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -249,7 +260,7 @@ def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] package = apt_mgr.get_package_for_paths(paths) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -257,7 +268,7 @@ def resolve_php_class_req(apt_mgr, req): path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") package = apt_mgr.get_package_for_paths([path]) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -265,7 +276,7 @@ def resolve_r_package_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)] package = apt_mgr.get_package_for_paths(paths, regex=True) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -277,7 +288,7 @@ def resolve_node_package_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -290,7 +301,7 @@ def resolve_library_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -298,7 +309,7 @@ def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] package = apt_mgr.get_package_for_paths(paths) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" @@ -307,7 +318,7 @@ def resolve_ruby_file_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -326,7 +337,7 @@ def resolve_xml_entity_req(apt_mgr, req): pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -338,7 +349,7 @@ def resolve_sprockets_file_req(apt_mgr, req): return None pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -357,14 +368,14 @@ def resolve_java_class_req(apt_mgr, req): if package is None: logging.warning("no package for files in %r", classpath) return None - return AptRequirement(package) + return AptRequirement.simple(package) def resolve_haskell_package_req(apt_mgr, req): path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0] pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -396,19 +407,19 @@ def resolve_maven_artifact_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None def resolve_gnome_common_req(apt_mgr, req): - return AptRequirement('gnome-common') + return AptRequirement.simple('gnome-common') def resolve_jdk_file_req(apt_mgr, req): path = req.jdk_path + ".*/" + req.filename pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -427,14 +438,14 @@ def resolve_perl_module_req(apt_mgr, req): paths = [posixpath.join(inc, req.filename) for inc in req.inc] pkg_name = apt_mgr.get_package_for_paths(paths, regex=False) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None def resolve_perl_file_req(apt_mgr, req): pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -459,7 +470,7 @@ def resolve_autoconf_macro_req(apt_mgr, req): return None pkg_name = apt_mgr.get_package_for_paths([path]) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -549,7 +560,8 @@ class AptResolver(Resolver): else: apt_requirements.append(apt_req) if apt_requirements: - self.apt.install([r.package for r in apt_requirements]) + self.apt.satisfy([PkgRelation.str(chain(*[ + r.relations for r in apt_requirements]))]) if still_missing: raise UnsatisfiedRequirements(still_missing) From 354001c60abc4be0933abd15128369cc2bf9c55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 17:20:26 +0000 Subject: [PATCH 10/21] egg-info doesn't need to have contents.. --- ognibuild/debian/fix_build.py | 6 +++--- ognibuild/resolver/apt.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 2cfa55d..1d4b336 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -259,7 +259,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 default = not targeted pypy_pkg = context.apt.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % error.distribution], regex=True + ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True ) if pypy_pkg is None: pypy_pkg = "pypy-%s" % error.distribution @@ -267,7 +267,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 pypy_pkg = None py2_pkg = context.apt.get_package_for_paths( - ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info/PKG-INFO" % error.distribution], + ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution], regex=True, ) if py2_pkg is None: @@ -276,7 +276,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 py2_pkg = None py3_pkg = context.apt.get_package_for_paths( - ["/usr/lib/python3/dist-packages/%s-.*.egg-info/PKG-INFO" % error.distribution], + ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution], regex=True, ) if py3_pkg is None: diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 6d470cc..8648700 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -79,15 +79,15 @@ class AptRequirement(Requirement): def get_package_for_python_package(apt_mgr, package, python_version, specs=None): if python_version == "pypy": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package], + ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % package], regex=True) elif python_version == "cpython2": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info/PKG-INFO" % package], + ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % package], regex=True) elif python_version == "cpython3": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/python3/dist-packages/%s-.*.egg-info/PKG-INFO" % package], + ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % package], regex=True) else: raise NotImplementedError From 677358e0d56d4b8da9d077c6b5196eefe63947c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 18:20:21 +0000 Subject: [PATCH 11/21] Support quiet mode for dist. --- ognibuild/buildsystem.py | 62 ++++++++++++++++++++++++++++------- ognibuild/debian/apt.py | 6 ++-- ognibuild/debian/fix_build.py | 25 +++++++------- ognibuild/dist.py | 4 +-- ognibuild/resolver/apt.py | 10 ++++++ 5 files changed, 79 insertions(+), 28 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 2848f18..dbf52d4 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -58,7 +58,7 @@ class BuildSystem(object): def __str__(self): return self.name - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): raise NotImplementedError(self.dist) def test(self, session, resolver, fixers): @@ -90,7 +90,7 @@ class Pear(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("pear")]) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) run_with_build_fixers(session, ["pear", "package"], fixers) @@ -111,13 +111,48 @@ class Pear(BuildSystem): run_with_build_fixers(session, ["pear", "install", self.path], fixers) +# run_setup, but setting __name__ +# Imported from Python's distutils.core, Copyright (C) PSF + +def run_setup(script_name, script_args=None, stop_after="run"): + from distutils import core + import sys + if stop_after not in ('init', 'config', 'commandline', 'run'): + raise ValueError("invalid value for 'stop_after': %r" % (stop_after,)) + + core._setup_stop_after = stop_after + + save_argv = sys.argv.copy() + g = {'__file__': script_name, '__name__': '__main__'} + try: + try: + sys.argv[0] = script_name + if script_args is not None: + sys.argv[1:] = script_args + with open(script_name, 'rb') as f: + exec(f.read(), g) + finally: + sys.argv = save_argv + core._setup_stop_after = None + except SystemExit: + # Hmm, should we do something if exiting with a non-zero code + # (ie. error)? + pass + + if core._setup_distribution is None: + raise RuntimeError(("'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?") % \ + script_name) + + return core._setup_distribution + + class SetupPy(BuildSystem): name = "setup.py" def __init__(self, path): self.path = path - from distutils.core import run_setup # TODO(jelmer): Perhaps run this in session, so we can install # missing dependencies? try: @@ -163,9 +198,12 @@ class SetupPy(BuildSystem): self.setup(resolver) self._run_setup(session, resolver, ["build"], fixers) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) - self._run_setup(session, resolver, ["sdist"], fixers) + preargs = [] + if quiet: + preargs.append('--quiet') + self._run_setup(session, resolver, preargs + ["sdist"], fixers) def clean(self, session, resolver, fixers): self.setup(resolver) @@ -230,7 +268,7 @@ class PyProject(BuildSystem): with open(self.path, "r") as pf: return toml.load(pf) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): if "poetry" in self.pyproject.get("tool", []): logging.debug( "Found pyproject.toml with poetry section, " @@ -261,7 +299,7 @@ class SetupCfg(BuildSystem): ] ) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) session.check_call(["python3", "-m", "pep517.build", "-s", "."]) @@ -285,7 +323,7 @@ class Npm(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("npm")]) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) run_with_build_fixers(session, ["npm", "pack"], fixers) @@ -300,7 +338,7 @@ class Waf(BuildSystem): def setup(self, session, resolver, fixers): resolver.install([BinaryRequirement("python3")]) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["./waf", "dist"], fixers) @@ -319,7 +357,7 @@ class Gem(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("gem2deb")]) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) gemfiles = [ entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") @@ -359,7 +397,7 @@ class DistInkt(BuildSystem): ] ) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) if self.name == "dist-inkt": resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) @@ -435,7 +473,7 @@ class Make(BuildSystem): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "install"], fixers) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): self.setup(session, resolver, fixers) try: run_with_build_fixers(session, ["make", "dist"], fixers) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 4ff00c1..677947a 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -62,7 +62,7 @@ class AptManager(object): def searchers(self): if self._searchers is None: self._searchers = [ - RemoteAptContentsFileSearcher.from_session(self.session), + AptContentsFileSearcher.from_session(self.session), GENERATED_FILE_SEARCHER] return self._searchers @@ -106,7 +106,7 @@ class ContentsFileNotFound(Exception): """The contents file was not found.""" -class RemoteAptContentsFileSearcher(FileSearcher): +class AptContentsFileSearcher(FileSearcher): def __init__(self): self._db = {} @@ -233,7 +233,7 @@ class RemoteAptContentsFileSearcher(FileSearcher): response = self._get(url + ext) except HTTPError as e: if e.status == 404: - continue + continue raise break else: diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 1d4b336..b0ced37 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -42,6 +42,7 @@ from debmutate.debhelper import ( ) from debmutate.deb822 import ( Deb822Editor, + PkgRelation, ) from debmutate.reformatting import ( FormattingUnpreservable, @@ -144,16 +145,17 @@ def add_build_dependency( try: with ControlEditor(path=control_path) as updater: for binary in updater.binaries: - if binary["Package"] == requirement.package: - raise CircularDependency(requirement.package) - updater.source["Build-Depends"] = ensure_relation( - updater.source.get("Build-Depends", ""), - requirement.relations) + if requirement.touches_package(binary["Package"]): + raise CircularDependency(binary["Package"]) + for rel in requirement.relations: + updater.source["Build-Depends"] = ensure_relation( + updater.source.get("Build-Depends", ""), + PkgRelation.str([rel])) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False - desc = PkgRelation.str(requirement.relations) + desc = requirement.pkg_relation_str() if not updater.changed: logging.info("Giving up; dependency %s was already present.", desc) @@ -193,16 +195,17 @@ def add_test_dependency( command_counter += 1 if name != testname: continue - control["Depends"] = ensure_relation( - control.get("Depends", ""), - requirement.relations) + for rel in requirement.relations: + control["Depends"] = ensure_relation( + control.get("Depends", ""), + PkgRelation.str([rel])) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False if not updater.changed: return False - desc = PkgRelation.str(requirement.relations) + desc = requirement.pkg_relation_str() logging.info("Adding dependency to test %s: %s", testname, desc) return commit_debian_changes( @@ -240,7 +243,7 @@ def commit_debian_changes( def targeted_python_versions(tree: Tree) -> Set[str]: with tree.get_file("debian/control") as f: control = Deb822(f) - build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) + build_depends = PkgRelation.parse(control.get("Build-Depends", "")) all_build_deps: Set[str] = set() for or_deps in build_depends: all_build_deps.update(or_dep["name"] for or_dep in or_deps) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 9c97e0e..730f38d 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -63,13 +63,13 @@ class DistNoTarball(Exception): """Dist operation did not create a tarball.""" -def run_dist(session, buildsystems, resolver, fixers): +def run_dist(session, buildsystems, resolver, fixers, quiet=False): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.dist(session, resolver, fixers) + buildsystem.dist(session, resolver, fixers, quiet=quiet) return raise NoBuildToolsFound() diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 8648700..d7d543d 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -75,6 +75,16 @@ class AptRequirement(Requirement): def from_str(cls, text): return cls(PkgRelation.parse_relations(text)) + def pkg_relation_str(self): + return PkgRelation.str(self.relations) + + def touches_package(self, package): + for rel in self.relations: + for entry in rel: + if entry['name'] == package: + return True + return False + def get_package_for_python_package(apt_mgr, package, python_version, specs=None): if python_version == "pypy": From 1741622d85900d6ce068860af937ee43f6f9375b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 18:26:53 +0000 Subject: [PATCH 12/21] Fix tests. --- ognibuild/debian/fix_build.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index b0ced37..0e1b4ef 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -37,12 +37,12 @@ from debmutate.control import ( ensure_relation, ControlEditor, ) +from debian.deb822 import PkgRelation from debmutate.debhelper import ( get_debhelper_compat_level, ) from debmutate.deb822 import ( Deb822Editor, - PkgRelation, ) from debmutate.reformatting import ( FormattingUnpreservable, @@ -243,7 +243,7 @@ def commit_debian_changes( def targeted_python_versions(tree: Tree) -> Set[str]: with tree.get_file("debian/control") as f: control = Deb822(f) - build_depends = PkgRelation.parse(control.get("Build-Depends", "")) + build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) all_build_deps: Set[str] = set() for or_deps in build_depends: all_build_deps.update(or_dep["name"] for or_dep in or_deps) @@ -317,9 +317,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency( - AptRequirement.simple( - dep_pkg.package, minimum_version=error.minimum_version)): + if not context.add_dependency(dep_pkg): return False return True @@ -370,8 +368,7 @@ def fix_missing_python_module(error, context): for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency( - AptRequirement.simple(dep_pkg.package, error.minimum_version)): + if not context.add_dependency(dep_pkg): return False return True From f8d269b6e5621b89628aaf2bb8dc42bcb83827c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 19:01:52 +0000 Subject: [PATCH 13/21] Fix style. --- ognibuild/__init__.py | 3 - ognibuild/__main__.py | 79 +++++++++---------- ognibuild/buildlog.py | 38 ++++----- ognibuild/buildsystem.py | 89 +++++++++++---------- ognibuild/debian/__init__.py | 1 + ognibuild/debian/apt.py | 77 +++++++++++-------- ognibuild/debian/build.py | 10 ++- ognibuild/debian/fix_build.py | 48 ++++++------ ognibuild/dist.py | 12 +-- ognibuild/fix_build.py | 13 +--- ognibuild/info.py | 22 +++--- ognibuild/outputs.py | 11 +-- ognibuild/requirements.py | 73 +++++++++--------- ognibuild/resolver/__init__.py | 58 +++++++------- ognibuild/resolver/apt.py | 98 +++++++++++++----------- ognibuild/session/schroot.py | 6 +- ognibuild/tests/test_debian_build.py | 5 +- ognibuild/tests/test_debian_fix_build.py | 11 ++- 18 files changed, 337 insertions(+), 317 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 684c6c1..6e210c2 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -21,7 +21,6 @@ import stat class DetailedFailure(Exception): - def __init__(self, retcode, argv, error): self.retcode = retcode self.argv = argv @@ -29,7 +28,6 @@ class DetailedFailure(Exception): class UnidentifiedError(Exception): - def __init__(self, retcode, argv, lines, secondary=None): self.retcode = retcode self.argv = argv @@ -63,7 +61,6 @@ class Requirement(object): class UpstreamOutput(object): - def __init__(self, family): self.family = family diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index d5f5ef4..f500051 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -23,7 +23,6 @@ from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, native_resolvers, - UnsatisfiedRequirements, ) from .resolver.apt import AptResolver @@ -39,15 +38,14 @@ def get_necessary_declared_requirements(resolver, requirements, stages): def install_necessary_declared_requirements(resolver, buildsystem, stages): missing = [] try: - declared_reqs = buildsystem.get_declared_dependencies() + declared_reqs = list(buildsystem.get_declared_dependencies()) except NotImplementedError: logging.warning( - 'Unable to determine declared dependencies from %s', buildsystem) + "Unable to determine declared dependencies from %s", buildsystem + ) else: missing.extend( - get_necessary_declared_requirements( - resolver, declared_reqs, stages - ) + get_necessary_declared_requirements(resolver, declared_reqs, stages) ) resolver.install(missing) @@ -70,7 +68,6 @@ STAGE_MAP = { def determine_fixers(session, resolver): from .buildlog import RequirementFixer - from .resolver.apt import AptResolver return [RequirementFixer(resolver)] @@ -90,36 +87,35 @@ def main(): # noqa: C901 ) parser.add_argument( "--explain", - action='store_true', - help="Explain what needs to be done rather than making changes") + action="store_true", + help="Explain what needs to be done rather than making changes", + ) parser.add_argument( "--ignore-declared-dependencies", "--optimistic", action="store_true", help="Ignore declared dependencies, follow build errors only", ) - parser.add_argument( - "--verbose", - action="store_true", - help="Be verbose") - subparsers = parser.add_subparsers(dest='subcommand') - subparsers.add_parser('dist') - subparsers.add_parser('build') - subparsers.add_parser('clean') - subparsers.add_parser('test') - subparsers.add_parser('info') - install_parser = subparsers.add_parser('install') + parser.add_argument("--verbose", action="store_true", help="Be verbose") + subparsers = parser.add_subparsers(dest="subcommand") + subparsers.add_parser("dist") + subparsers.add_parser("build") + subparsers.add_parser("clean") + subparsers.add_parser("test") + subparsers.add_parser("info") + install_parser = subparsers.add_parser("install") install_parser.add_argument( - '--user', action='store_true', help='Install in local-user directories.') + "--user", action="store_true", help="Install in local-user directories." + ) args = parser.parse_args() if not args.subcommand: parser.print_usage() return 1 if args.verbose: - logging.basicConfig(level=logging.DEBUG, format='%(message)s') + logging.basicConfig(level=logging.DEBUG, format="%(message)s") else: - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(level=logging.INFO, format="%(message)s") if args.schroot: from .session.schroot import SchrootSession @@ -135,46 +131,51 @@ def main(): # noqa: C901 resolver = native_resolvers(session) elif args.resolve == "auto": resolver = auto_resolver(session) - logging.info('Using requirement resolver: %s', resolver) + logging.info("Using requirement resolver: %s", resolver) os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) - logging.info('Detected buildsystems: %r', bss) + logging.info("Detected buildsystems: %r", bss) if not args.ignore_declared_dependencies and not args.explain: stages = STAGE_MAP[args.subcommand] if stages: - logging.info('Checking that declared requirements are present') + logging.info("Checking that declared requirements are present") for bs in bss: install_necessary_declared_requirements(resolver, bs, stages) fixers = determine_fixers(session, resolver) if args.subcommand == "dist": from .dist import run_dist + run_dist( - session=session, buildsystems=bss, resolver=resolver, - fixers=fixers) + session=session, buildsystems=bss, resolver=resolver, fixers=fixers + ) if args.subcommand == "build": from .build import run_build - run_build( - session, buildsystems=bss, resolver=resolver, - fixers=fixers) + + run_build(session, buildsystems=bss, resolver=resolver, fixers=fixers) if args.subcommand == "clean": from .clean import run_clean - run_clean( - session, buildsystems=bss, resolver=resolver, - fixers=fixers) + + run_clean(session, buildsystems=bss, resolver=resolver, fixers=fixers) if args.subcommand == "install": from .install import run_install + run_install( - session, buildsystems=bss, resolver=resolver, - fixers=fixers, user=args.user) + session, + buildsystems=bss, + resolver=resolver, + fixers=fixers, + user=args.user, + ) if args.subcommand == "test": from .test import run_test - run_test(session, buildsystems=bss, resolver=resolver, - fixers=fixers) + + run_test(session, buildsystems=bss, resolver=resolver, fixers=fixers) if args.subcommand == "info": from .info import run_info + run_info(session, buildsystems=bss) - except UnidentifiedError as e: + except UnidentifiedError: return 1 except NoBuildToolsFound: logging.info("No build tools found.") diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 2401ddf..3560e32 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -21,7 +21,6 @@ import logging from buildlog_consultant.common import ( - MissingConfigStatusInput, MissingPythonModule, MissingPythonDistribution, MissingCHeader, @@ -41,15 +40,12 @@ from buildlog_consultant.common import ( MissingLibrary, MissingJavaClass, MissingCSharpCompiler, - MissingConfigure, - MissingAutomakeInput, MissingRPackage, MissingRubyFile, MissingAutoconfMacro, MissingValaPackage, MissingXfceDependency, MissingHaskellDependencies, - NeedPgBuildExtUpdateControl, DhAddonLoadFailure, MissingMavenArtifacts, GnomeCommonMissing, @@ -84,17 +80,16 @@ from .requirements import ( AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, - ) +) -def problem_to_upstream_requirement(problem): +def problem_to_upstream_requirement(problem): # noqa: C901 if isinstance(problem, MissingFile): return PathRequirement(problem.path) elif isinstance(problem, MissingCommand): return BinaryRequirement(problem.command) elif isinstance(problem, MissingPkgConfig): - return PkgConfigRequirement( - problem.module, problem.minimum_version) + return PkgConfigRequirement(problem.module, problem.minimum_version) elif isinstance(problem, MissingCHeader): return CHeaderRequirement(problem.header) elif isinstance(problem, MissingJavaScriptRuntime): @@ -126,35 +121,31 @@ def problem_to_upstream_requirement(problem): elif isinstance(problem, MissingHaskellDependencies): return [HaskellPackageRequirement(dep) for dep in problem.deps] elif isinstance(problem, MissingMavenArtifacts): - return [MavenArtifactRequirement(artifact) - for artifact in problem.artifacts] + return [MavenArtifactRequirement(artifact) for artifact in problem.artifacts] elif isinstance(problem, MissingCSharpCompiler): - return BinaryRequirement('msc') + return BinaryRequirement("msc") elif isinstance(problem, GnomeCommonMissing): return GnomeCommonRequirement() elif isinstance(problem, MissingJDKFile): return JDKFileRequirement(problem.jdk_path, problem.filename) elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": - return BinaryRequirement('glib-gettextize') + return BinaryRequirement("glib-gettextize") else: logging.warning( - "No known command for gnome-common dependency %s", - problem.package) + "No known command for gnome-common dependency %s", problem.package + ) return None elif isinstance(problem, MissingXfceDependency): if problem.package == "gtk-doc": return BinaryRequirement("gtkdocize") else: - logging.warning( - "No known command for xfce dependency %s", - problem.package) + logging.warning("No known command for xfce dependency %s", problem.package) return None elif isinstance(problem, MissingPerlModule): return PerlModuleRequirement( - module=problem.module, - filename=problem.filename, - inc=problem.inc) + module=problem.module, filename=problem.filename, inc=problem.inc + ) elif isinstance(problem, MissingPerlFile): return PerlFileRequirement(filename=problem.filename) elif isinstance(problem, MissingAutoconfMacro): @@ -163,18 +154,19 @@ def problem_to_upstream_requirement(problem): return PythonModuleRequirement( problem.module, python_version=problem.python_version, - minimum_version=problem.minimum_version) + minimum_version=problem.minimum_version, + ) elif isinstance(problem, MissingPythonDistribution): return PythonPackageRequirement( problem.module, python_version=problem.python_version, - minimum_version=problem.minimum_version) + minimum_version=problem.minimum_version, + ) else: return None class RequirementFixer(BuildFixer): - def __init__(self, resolver): self.resolver = resolver diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index dbf52d4..d309f50 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -27,14 +27,14 @@ from . import shebang_binary, UnidentifiedError from .outputs import ( BinaryOutput, PythonPackageOutput, - ) +) from .requirements import ( BinaryRequirement, PythonPackageRequirement, PerlModuleRequirement, NodePackageRequirement, CargoCrateRequirement, - ) +) from .fix_build import run_with_build_fixers @@ -114,22 +114,24 @@ class Pear(BuildSystem): # run_setup, but setting __name__ # Imported from Python's distutils.core, Copyright (C) PSF + def run_setup(script_name, script_args=None, stop_after="run"): from distutils import core import sys - if stop_after not in ('init', 'config', 'commandline', 'run'): + + if stop_after not in ("init", "config", "commandline", "run"): raise ValueError("invalid value for 'stop_after': %r" % (stop_after,)) core._setup_stop_after = stop_after save_argv = sys.argv.copy() - g = {'__file__': script_name, '__name__': '__main__'} + g = {"__file__": script_name, "__name__": "__main__"} try: try: sys.argv[0] = script_name if script_args is not None: sys.argv[1:] = script_args - with open(script_name, 'rb') as f: + with open(script_name, "rb") as f: exec(f.read(), g) finally: sys.argv = save_argv @@ -140,9 +142,13 @@ def run_setup(script_name, script_args=None, stop_after="run"): pass if core._setup_distribution is None: - raise RuntimeError(("'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?") % \ - script_name) + raise RuntimeError( + ( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" + ) + % script_name + ) return core._setup_distribution @@ -158,7 +164,7 @@ class SetupPy(BuildSystem): try: self.result = run_setup(os.path.abspath(path), stop_after="init") except RuntimeError as e: - logging.warning('Unable to load setup.py metadata: %s', e) + logging.warning("Unable to load setup.py metadata: %s", e) self.result = None def __repr__(self): @@ -202,7 +208,7 @@ class SetupPy(BuildSystem): self.setup(resolver) preargs = [] if quiet: - preargs.append('--quiet') + preargs.append("--quiet") self._run_setup(session, resolver, preargs + ["sdist"], fixers) def clean(self, session, resolver, fixers): @@ -213,7 +219,7 @@ class SetupPy(BuildSystem): self.setup(resolver) extra_args = [] if install_target.user: - extra_args.append('--user') + extra_args.append("--user") self._run_setup(session, resolver, ["install"] + extra_args, fixers) def _run_setup(self, session, resolver, args, fixers): @@ -224,9 +230,7 @@ class SetupPy(BuildSystem): else: # Just assume it's Python 3 resolver.install([BinaryRequirement("python3")]) - run_with_build_fixers( - session, ["python3", "./setup.py"] + args, - fixers) + run_with_build_fixers(session, ["python3", "./setup.py"] + args, fixers) def get_declared_dependencies(self): if self.result is None: @@ -234,11 +238,11 @@ class SetupPy(BuildSystem): for require in self.result.get_requires(): yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.result, 'install_requires', []): + if getattr(self.result, "install_requires", []): for require in self.result.install_requires: yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.result, 'tests_require', []): + if getattr(self.result, "tests_require", []): for require in self.result.tests_require: yield "test", PythonPackageRequirement.from_requirement_str(require) @@ -247,7 +251,7 @@ class SetupPy(BuildSystem): raise NotImplementedError for script in self.result.scripts or []: yield BinaryOutput(os.path.basename(script)) - entry_points = getattr(self.result, 'entry_points', None) or {} + entry_points = getattr(self.result, "entry_points", None) or {} for script in entry_points.get("console_scripts", []): yield BinaryOutput(script.split("=")[0]) for package in self.result.packages or []: @@ -271,8 +275,7 @@ class PyProject(BuildSystem): def dist(self, session, resolver, fixers, quiet=False): if "poetry" in self.pyproject.get("tool", []): logging.debug( - "Found pyproject.toml with poetry section, " - "assuming poetry project." + "Found pyproject.toml with poetry section, " "assuming poetry project." ) resolver.install( [ @@ -382,8 +385,7 @@ class DistInkt(BuildSystem): continue if key.strip() == b"class" and value.strip().startswith(b"'Dist::Inkt"): logging.debug( - "Found Dist::Inkt section in dist.ini, " - "assuming distinkt." + "Found Dist::Inkt section in dist.ini, " "assuming distinkt." ) self.name = "dist-inkt" self.dist_inkt_class = value.decode().strip("'") @@ -405,8 +407,7 @@ class DistInkt(BuildSystem): else: # Default to invoking Dist::Zilla resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers( - session, ["dzil", "build", "--in", ".."], fixers) + run_with_build_fixers(session, ["dzil", "build", "--in", ".."], fixers) class Make(BuildSystem): @@ -419,27 +420,28 @@ class Make(BuildSystem): def setup(self, session, resolver, fixers): resolver.install([BinaryRequirement("make")]) - if session.exists("Makefile.PL") and not session.exists("Makefile"): + def makefile_exists(): + return any( + [session.exists(p) for p in ["Makefile", "GNUmakefile", "makefile"]] + ) + + if session.exists("Makefile.PL") and not makefile_exists(): resolver.install([BinaryRequirement("perl")]) run_with_build_fixers(session, ["perl", "Makefile.PL"], fixers) - if not session.exists("Makefile") and not session.exists("configure"): + if not makefile_exists() and not session.exists("configure"): if session.exists("autogen.sh"): if shebang_binary("autogen.sh") is None: - run_with_build_fixers( - session, ["/bin/sh", "./autogen.sh"], fixers) + run_with_build_fixers(session, ["/bin/sh", "./autogen.sh"], fixers) try: - run_with_build_fixers( - session, ["./autogen.sh"], fixers) + run_with_build_fixers(session, ["./autogen.sh"], fixers) except UnidentifiedError as e: if ( "Gnulib not yet bootstrapped; " "run ./bootstrap instead.\n" in e.lines ): - run_with_build_fixers( - session, ["./bootstrap"], fixers) - run_with_build_fixers( - session, ["./autogen.sh"], fixers) + run_with_build_fixers(session, ["./bootstrap"], fixers) + run_with_build_fixers(session, ["./autogen.sh"], fixers) else: raise @@ -454,7 +456,7 @@ class Make(BuildSystem): ) run_with_build_fixers(session, ["autoreconf", "-i"], fixers) - if not session.exists("Makefile") and session.exists("configure"): + if not makefile_exists() and session.exists("configure"): session.check_call(["./configure"]) def build(self, session, resolver, fixers): @@ -500,7 +502,8 @@ class Make(BuildSystem): elif any( [ re.match( - r"Makefile:[0-9]+: \*\*\* Missing \'Make.inc\' " + r"(Makefile|GNUmakefile|makefile):[0-9]+: " + r"\*\*\* Missing \'Make.inc\' " r"Run \'./configure \[options\]\' and retry. Stop.\n", line, ) @@ -592,20 +595,22 @@ class Cabal(BuildSystem): def _run(self, session, args, fixers): try: - run_with_build_fixers( - session, ["runhaskell", "Setup.hs"] + args, fixers) + run_with_build_fixers(session, ["runhaskell", "Setup.hs"] + args, fixers) except UnidentifiedError as e: if "Run the 'configure' command first.\n" in e.lines: run_with_build_fixers( - session, ["runhaskell", "Setup.hs", "configure"], fixers) + session, ["runhaskell", "Setup.hs", "configure"], fixers + ) run_with_build_fixers( - session, ["runhaskell", "Setup.hs"] + args, fixers) + session, ["runhaskell", "Setup.hs"] + args, fixers + ) else: raise def test(self, session, resolver, fixers): self._run(session, ["test"], fixers) + def detect_buildsystems(path, trust_package=False): # noqa: C901 """Detect build systems.""" if os.path.exists(os.path.join(path, "package.xml")): @@ -634,9 +639,9 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 logging.debug("Found Cargo.toml, assuming rust cargo package.") yield Cargo("Cargo.toml") - if os.path.exists(os.path.join(path, 'Setup.hs')): + if os.path.exists(os.path.join(path, "Setup.hs")): logging.debug("Found Setup.hs, assuming haskell package.") - yield Cabal('Setup.hs') + yield Cabal("Setup.hs") if os.path.exists(os.path.join(path, "pom.xml")): logging.debug("Found pom.xml, assuming maven package.") @@ -656,6 +661,8 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 os.path.exists(os.path.join(path, p)) for p in [ "Makefile", + "GNUmakefile", + "makefile", "Makefile.PL", "autogen.sh", "configure.ac", diff --git a/ognibuild/debian/__init__.py b/ognibuild/debian/__init__.py index 4578b6d..8879a4c 100644 --- a/ognibuild/debian/__init__.py +++ b/ognibuild/debian/__init__.py @@ -36,5 +36,6 @@ def satisfy_build_deps(session: Session, tree): pass deps = [dep.strip().strip(",") for dep in deps] from .apt import AptManager + apt = AptManager(session) apt.satisfy(deps) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 677947a..6dc89f7 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -24,7 +24,6 @@ import os from buildlog_consultant.apt import ( find_apt_get_failure, ) -from debian.deb822 import Release from .. import DetailedFailure, UnidentifiedError from ..session import Session, run_with_tee @@ -63,17 +62,19 @@ class AptManager(object): if self._searchers is None: self._searchers = [ AptContentsFileSearcher.from_session(self.session), - GENERATED_FILE_SEARCHER] + GENERATED_FILE_SEARCHER, + ] return self._searchers def package_exists(self, package): if self._apt_cache is None: import apt + self._apt_cache = apt.Cache(rootdir=self.session.location) return package in self._apt_cache def get_package_for_paths(self, paths, regex=False): - logging.debug('Searching for packages containing %r', paths) + logging.debug("Searching for packages containing %r", paths) # TODO(jelmer): Make sure we use whatever is configured in self.session return get_package_for_paths(paths, self.searchers(), regex=regex) @@ -82,6 +83,7 @@ class AptManager(object): status_path = os.path.join(root, "var/lib/dpkg/status") missing = set(packages) import apt_pkg + with apt_pkg.TagFile(status_path) as tagf: while missing: tagf.step() @@ -93,7 +95,7 @@ class AptManager(object): return list(missing) def install(self, packages: List[str]) -> None: - logging.info('Installing using apt: %r', packages) + logging.info("Installing using apt: %r", packages) packages = self.missing(packages) if packages: run_apt(self.session, ["install"] + packages) @@ -112,16 +114,19 @@ class AptContentsFileSearcher(FileSearcher): @classmethod def from_session(cls, session): - logging.info('Loading apt contents information') + logging.info("Loading apt contents information") # TODO(jelmer): what about sources.list.d? from aptsources.sourceslist import SourcesList + sl = SourcesList() - sl.load(os.path.join(session.location, 'etc/apt/sources.list')) + sl.load(os.path.join(session.location, "etc/apt/sources.list")) return cls.from_sources_list( sl, cache_dirs=[ - os.path.join(session.location, 'var/lib/apt/lists'), - '/var/lib/apt/lists']) + os.path.join(session.location, "var/lib/apt/lists"), + "/var/lib/apt/lists", + ], + ) def __setitem__(self, path, package): self._db[path] = package @@ -146,15 +151,17 @@ class AptContentsFileSearcher(FileSearcher): @classmethod def _load_cache_file(cls, url, cache_dir): from urllib.parse import urlparse + parsed = urlparse(url) p = os.path.join( - cache_dir, - parsed.hostname + parsed.path.replace('/', '_') + '.lz4') + cache_dir, parsed.hostname + parsed.path.replace("/", "_") + ".lz4" + ) if not os.path.exists(p): return None - logging.debug('Loading cached contents file %s', p) + logging.debug("Loading cached contents file %s", p) import lz4.frame - return lz4.frame.open(p, mode='rb') + + return lz4.frame.open(p, mode="rb") @classmethod def from_urls(cls, urls, cache_dirs=None): @@ -168,39 +175,39 @@ class AptContentsFileSearcher(FileSearcher): else: if not mandatory and self._db: logging.debug( - 'Not attempting to fetch optional contents ' - 'file %s', url) + "Not attempting to fetch optional contents " "file %s", url + ) else: - logging.debug('Fetching contents file %s', url) + logging.debug("Fetching contents file %s", url) try: self.load_url(url) except ContentsFileNotFound: if mandatory: - logging.warning( - 'Unable to fetch contents file %s', url) + logging.warning("Unable to fetch contents file %s", url) else: logging.debug( - 'Unable to fetch optional contents file %s', - url) + "Unable to fetch optional contents file %s", url + ) return self @classmethod def from_sources_list(cls, sl, cache_dirs=None): # TODO(jelmer): Use aptsources.sourceslist.SourcesList from .build import get_build_architecture + # TODO(jelmer): Verify signatures, etc. urls = [] arches = [(get_build_architecture(), True), ("all", False)] for source in sl.list: if source.invalid or source.disabled: continue - if source.type == 'deb-src': + if source.type == "deb-src": continue - if source.type != 'deb': + if source.type != "deb": logging.warning("Invalid line in sources: %r", source) continue - base_url = source.uri.rstrip('/') - name = source.dist.rstrip('/') + base_url = source.uri.rstrip("/") + name = source.dist.rstrip("/") components = source.comps if components: dists_url = base_url + "/dists" @@ -210,12 +217,20 @@ class AptContentsFileSearcher(FileSearcher): for component in components: for arch, mandatory in arches: urls.append( - ("%s/%s/%s/Contents-%s" % ( - dists_url, name, component, arch), mandatory)) + ( + "%s/%s/%s/Contents-%s" + % (dists_url, name, component, arch), + mandatory, + ) + ) else: for arch, mandatory in arches: urls.append( - ("%s/%s/Contents-%s" % (dists_url, name.rstrip('/'), arch), mandatory)) + ( + "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), + mandatory, + ) + ) return cls.from_urls(urls, cache_dirs=cache_dirs) @staticmethod @@ -228,7 +243,7 @@ class AptContentsFileSearcher(FileSearcher): def load_url(self, url, allow_cache=True): from urllib.error import HTTPError - for ext in ['.xz', '.gz', '']: + for ext in [".xz", ".gz", ""]: try: response = self._get(url + ext) except HTTPError as e: @@ -238,13 +253,14 @@ class AptContentsFileSearcher(FileSearcher): break else: raise ContentsFileNotFound(url) - if ext == '.gz': + if ext == ".gz": import gzip f = gzip.GzipFile(fileobj=response) - elif ext == '.xz': + elif ext == ".xz": import lzma from io import BytesIO + f = BytesIO(lzma.decompress(response.read())) elif response.headers.get_content_type() == "text/plain": f = response @@ -280,7 +296,8 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( def get_package_for_paths( - paths: List[str], searchers: List[FileSearcher], regex: bool = False) -> Optional[str]: + paths: List[str], searchers: List[FileSearcher], regex: bool = False +) -> Optional[str]: candidates: Set[str] = set() for path in paths: for searcher in searchers: diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index da42d4e..227a38b 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -62,11 +62,13 @@ def changes_filename(package, version, arch): def get_build_architecture(): try: - return subprocess.check_output( - ['dpkg-architecture', '-qDEB_BUILD_ARCH']).strip().decode() + return ( + subprocess.check_output(["dpkg-architecture", "-qDEB_BUILD_ARCH"]) + .strip() + .decode() + ) except subprocess.CalledProcessError as e: - raise Exception( - "Could not find the build architecture: %s" % e) + raise Exception("Could not find the build architecture: %s" % e) def add_dummy_changelog_entry( diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 0e1b4ef..ca8b2b9 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -22,13 +22,12 @@ __all__ = [ import logging import os import sys -from typing import List, Set, Optional +from typing import List, Set, Optional, Type from debian.deb822 import ( Deb822, PkgRelation, ) -from debian.changelog import Version from breezy.commit import PointlessCommit from breezy.mutabletree import MutableTree @@ -37,7 +36,6 @@ from debmutate.control import ( ensure_relation, ControlEditor, ) -from debian.deb822 import PkgRelation from debmutate.debhelper import ( get_debhelper_compat_level, ) @@ -48,6 +46,7 @@ from debmutate.reformatting import ( FormattingUnpreservable, GeneratedFile, ) + try: from breezy.workspace import reset_tree except ImportError: @@ -75,7 +74,7 @@ from buildlog_consultant.common import ( MissingPythonModule, MissingPythonDistribution, MissingPerlFile, - ) +) from buildlog_consultant.sbuild import ( SbuildFailure, ) @@ -85,7 +84,7 @@ from ..buildlog import RequirementFixer from ..resolver.apt import ( AptRequirement, get_package_for_python_module, - ) +) from .build import attempt_build, DEFAULT_BUILDER @@ -100,7 +99,6 @@ class CircularDependency(Exception): class BuildDependencyContext(DependencyContext): - def add_dependency(self, requirement: AptRequirement): return add_build_dependency( self.tree, @@ -149,8 +147,8 @@ def add_build_dependency( raise CircularDependency(binary["Package"]) for rel in requirement.relations: updater.source["Build-Depends"] = ensure_relation( - updater.source.get("Build-Depends", ""), - PkgRelation.str([rel])) + updater.source.get("Build-Depends", ""), PkgRelation.str([rel]) + ) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False @@ -197,8 +195,8 @@ def add_test_dependency( continue for rel in requirement.relations: control["Depends"] = ensure_relation( - control.get("Depends", ""), - PkgRelation.str([rel])) + control.get("Depends", ""), PkgRelation.str([rel]) + ) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False @@ -330,7 +328,7 @@ def fix_missing_python_module(error, context): default = not targeted if error.minimum_version: - specs = [('>=', error.minimum_version)] + specs = [(">=", error.minimum_version)] else: specs = [] @@ -397,8 +395,9 @@ def enable_dh_autoreconf(context): def fix_missing_configure(error, context): - if (not context.tree.has_filename("configure.ac") and - not context.tree.has_filename("configure.in")): + if not context.tree.has_filename("configure.ac") and not context.tree.has_filename( + "configure.in" + ): return False return enable_dh_autoreconf(context) @@ -443,16 +442,12 @@ def fix_missing_config_status_input(error, context): class PgBuildExtOutOfDateControlFixer(BuildFixer): - def __init__(self, session): self.session = session def can_fix(self, problem): return isinstance(problem, NeedPgBuildExtUpdateControl) - def _fix(self, problem, context): - return self._fn(problem, context) - def _fix(self, error, context): logging.info("Running 'pg_buildext updatecontrol'") self.session.check_call(["pg_buildext", "updatecontrol"]) @@ -477,18 +472,17 @@ def fix_missing_makefile_pl(error, context): class SimpleBuildFixer(BuildFixer): - - def __init__(self, problem_cls, fn): + def __init__(self, problem_cls: Type[Problem], fn): self._problem_cls = problem_cls self._fn = fn def __repr__(self): return "%s(%r, %r)" % (type(self).__name__, self._problem_cls, self._fn) - def can_fix(self, problem): + def can_fix(self, problem: Problem): return isinstance(problem, self._problem_cls) - def _fix(self, problem, context): + def _fix(self, problem: Problem, context): return self._fn(problem, context) @@ -504,6 +498,7 @@ def versioned_package_fixers(session): def apt_fixers(apt) -> List[BuildFixer]: from ..resolver.apt import AptResolver + resolver = AptResolver(apt) return [ SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), @@ -529,7 +524,7 @@ def build_incrementally( ): fixed_errors = [] fixers = versioned_package_fixers(apt.session) + apt_fixers(apt) - logging.info('Using fixers: %r', fixers) + logging.info("Using fixers: %r", fixers) while True: try: return attempt_build( @@ -583,7 +578,9 @@ def build_incrementally( except GeneratedFile: logging.warning( "Control file is generated, unable to edit to " - "resolver error %r.", e.error) + "resolver error %r.", + e.error, + ) raise e except CircularDependency: logging.warning( @@ -647,14 +644,15 @@ def main(argv=None): from ..session.plain import PlainSession import tempfile import contextlib + apt = AptManager(PlainSession()) - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(level=logging.INFO, format="%(message)s") with contextlib.ExitStack() as es: if args.output_directory is None: output_directory = es.enter_context(tempfile.TemporaryDirectory()) - logging.info('Using output directory %s', output_directory) + logging.info("Using output directory %s", output_directory) else: output_directory = args.output_directory diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 730f38d..551a4b0 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -49,7 +49,7 @@ SUPPORTED_DIST_EXTENSIONS = [ ".tbz2", ".tar", ".zip", - ] +] def is_dist_file(fn): @@ -76,7 +76,6 @@ def run_dist(session, buildsystems, resolver, fixers, quiet=False): class DistCatcher(object): - def __init__(self, directory): self.export_directory = directory self.files = [] @@ -205,17 +204,14 @@ if __name__ == "__main__": parser.add_argument( "--target-directory", type=str, default="..", help="Target directory" ) - parser.add_argument( - "--verbose", - action="store_true", - help="Be verbose") + parser.add_argument("--verbose", action="store_true", help="Be verbose") args = parser.parse_args() if args.verbose: - logging.basicConfig(level=logging.DEBUG, format='%(message)s') + logging.basicConfig(level=logging.DEBUG, format="%(message)s") else: - logging.basicConfig(level=logging.INFO, format='%(message)s') + logging.basicConfig(level=logging.INFO, format="%(message)s") tree = WorkingTree.open(args.directory) if args.packaging_directory: diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index efea8d7..3ffbf3f 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -16,14 +16,10 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging -from typing import List, Tuple, Callable, Type, Optional +from typing import List, Optional from buildlog_consultant.common import ( find_build_failure_description, - Problem, - MissingPerlModule, - MissingPythonDistribution, - MissingCommand, ) from breezy.mutabletree import MutableTree @@ -63,7 +59,7 @@ class DependencyContext(object): self.update_changelog = update_changelog def add_dependency( - self, package: str, minimum_version: Optional['Version'] = None + self, package: str, minimum_version=None ) -> bool: raise NotImplementedError(self.add_dependency) @@ -79,8 +75,7 @@ class SchrootDependencyContext(DependencyContext): return True -def run_with_build_fixers( - session: Session, args: List[str], fixers: List[BuildFixer]): +def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer]): logging.info("Running %r", args) fixed_errors = [] while True: @@ -91,7 +86,7 @@ def run_with_build_fixers( if error is None: if match: logging.warning("Build failed with unidentified error:") - logging.warning('%s', match.line.rstrip('\n')) + logging.warning("%s", match.line.rstrip("\n")) else: logging.warning("Build failed and unable to find cause. Giving up.") raise UnidentifiedError(retcode, args, lines, secondary=match) diff --git a/ognibuild/info.py b/ognibuild/info.py index 3848b2b..553b620 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -15,31 +15,31 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from .buildsystem import NoBuildToolsFound, InstallTarget - def run_info(session, buildsystems): for buildsystem in buildsystems: - print('%r:' % buildsystem) + print("%r:" % buildsystem) deps = {} try: for kind, dep in buildsystem.get_declared_dependencies(): deps.setdefault(kind, []).append(dep) except NotImplementedError: - print('\tUnable to detect declared dependencies for this type of build system') + print( + "\tUnable to detect declared dependencies for this type of build system" + ) if deps: - print('\tDeclared dependencies:') + print("\tDeclared dependencies:") for kind in deps: - print('\t\t%s:' % kind) + print("\t\t%s:" % kind) for dep in deps[kind]: - print('\t\t\t%s' % dep) - print('') + print("\t\t\t%s" % dep) + print("") try: outputs = list(buildsystem.get_declared_outputs()) except NotImplementedError: - print('\tUnable to detect declared outputs for this type of build system') + print("\tUnable to detect declared outputs for this type of build system") outputs = [] if outputs: - print('\tDeclared outputs:') + print("\tDeclared outputs:") for output in outputs: - print('\t\t%s' % output) + print("\t\t%s" % output) diff --git a/ognibuild/outputs.py b/ognibuild/outputs.py index bcb305b..ba1bf85 100644 --- a/ognibuild/outputs.py +++ b/ognibuild/outputs.py @@ -20,9 +20,8 @@ from . import UpstreamOutput class BinaryOutput(UpstreamOutput): - def __init__(self, name): - super(BinaryOutput, self).__init__('binary') + super(BinaryOutput, self).__init__("binary") self.name = name def __repr__(self): @@ -33,9 +32,8 @@ class BinaryOutput(UpstreamOutput): class PythonPackageOutput(UpstreamOutput): - def __init__(self, name, python_version=None): - super(PythonPackageOutput, self).__init__('python-package') + super(PythonPackageOutput, self).__init__("python-package") self.name = name self.python_version = python_version @@ -44,4 +42,7 @@ class PythonPackageOutput(UpstreamOutput): def __repr__(self): return "%s(%r, python_version=%r)" % ( - type(self).__name__, self.name, self.python_version) + type(self).__name__, + self.name, + self.python_version, + ) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index a523246..1a46852 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -26,30 +26,32 @@ class PythonPackageRequirement(Requirement): package: str - def __init__(self, package, python_version=None, specs=None, - minimum_version=None): - super(PythonPackageRequirement, self).__init__('python-package') + def __init__(self, package, python_version=None, specs=None, minimum_version=None): + super(PythonPackageRequirement, self).__init__("python-package") self.package = package self.python_version = python_version if minimum_version is not None: - specs = [('>=', minimum_version)] + specs = [(">=", minimum_version)] self.specs = specs def __repr__(self): return "%s(%r, python_version=%r, specs=%r)" % ( - type(self).__name__, self.package, self.python_version, - self.specs) + type(self).__name__, + self.package, + self.python_version, + self.specs, + ) def __str__(self): if self.specs: return "python package: %s (%r)" % (self.package, self.specs) else: - return "python package: %s" % (self.package, ) - + return "python package: %s" % (self.package,) @classmethod def from_requirement_str(cls, text): from requirements.requirement import Requirement + req = Requirement.parse(text) return cls(package=req.name, specs=req.specs) @@ -59,7 +61,7 @@ class BinaryRequirement(Requirement): binary_name: str def __init__(self, binary_name): - super(BinaryRequirement, self).__init__('binary') + super(BinaryRequirement, self).__init__("binary") self.binary_name = binary_name @@ -70,7 +72,7 @@ class PerlModuleRequirement(Requirement): inc: Optional[List[str]] def __init__(self, module, filename=None, inc=None): - super(PerlModuleRequirement, self).__init__('perl-module') + super(PerlModuleRequirement, self).__init__("perl-module") self.module = module self.filename = filename self.inc = inc @@ -84,7 +86,7 @@ class NodePackageRequirement(Requirement): package: str def __init__(self, package): - super(NodePackageRequirement, self).__init__('npm-package') + super(NodePackageRequirement, self).__init__("npm-package") self.package = package @@ -93,7 +95,7 @@ class CargoCrateRequirement(Requirement): crate: str def __init__(self, crate): - super(CargoCrateRequirement, self).__init__('cargo-crate') + super(CargoCrateRequirement, self).__init__("cargo-crate") self.crate = crate @@ -102,7 +104,7 @@ class PkgConfigRequirement(Requirement): module: str def __init__(self, module, minimum_version=None): - super(PkgConfigRequirement, self).__init__('pkg-config') + super(PkgConfigRequirement, self).__init__("pkg-config") self.module = module self.minimum_version = minimum_version @@ -112,7 +114,7 @@ class PathRequirement(Requirement): path: str def __init__(self, path): - super(PathRequirement, self).__init__('path') + super(PathRequirement, self).__init__("path") self.path = path @@ -121,15 +123,13 @@ class CHeaderRequirement(Requirement): header: str def __init__(self, header): - super(CHeaderRequirement, self).__init__('c-header') + super(CHeaderRequirement, self).__init__("c-header") self.header = header class JavaScriptRuntimeRequirement(Requirement): - def __init__(self): - super(JavaScriptRuntimeRequirement, self).__init__( - 'javascript-runtime') + super(JavaScriptRuntimeRequirement, self).__init__("javascript-runtime") class ValaPackageRequirement(Requirement): @@ -137,7 +137,7 @@ class ValaPackageRequirement(Requirement): package: str def __init__(self, package: str): - super(ValaPackageRequirement, self).__init__('vala') + super(ValaPackageRequirement, self).__init__("vala") self.package = package @@ -147,7 +147,7 @@ class RubyGemRequirement(Requirement): minimum_version: Optional[str] def __init__(self, gem: str, minimum_version: Optional[str]): - super(RubyGemRequirement, self).__init__('gem') + super(RubyGemRequirement, self).__init__("gem") self.gem = gem self.minimum_version = minimum_version @@ -157,7 +157,7 @@ class GoPackageRequirement(Requirement): package: str def __init__(self, package: str): - super(GoPackageRequirement, self).__init__('go') + super(GoPackageRequirement, self).__init__("go") self.package = package @@ -166,7 +166,7 @@ class DhAddonRequirement(Requirement): path: str def __init__(self, path: str): - super(DhAddonRequirement, self).__init__('dh-addon') + super(DhAddonRequirement, self).__init__("dh-addon") self.path = path @@ -175,7 +175,7 @@ class PhpClassRequirement(Requirement): php_class: str def __init__(self, php_class: str): - super(PhpClassRequirement, self).__init__('php-class') + super(PhpClassRequirement, self).__init__("php-class") self.php_class = php_class @@ -185,7 +185,7 @@ class RPackageRequirement(Requirement): minimum_version: Optional[str] def __init__(self, package: str, minimum_version: Optional[str] = None): - super(RPackageRequirement, self).__init__('r-package') + super(RPackageRequirement, self).__init__("r-package") self.package = package self.minimum_version = minimum_version @@ -195,7 +195,7 @@ class LibraryRequirement(Requirement): library: str def __init__(self, library: str): - super(LibraryRequirement, self).__init__('lib') + super(LibraryRequirement, self).__init__("lib") self.library = library @@ -204,7 +204,7 @@ class RubyFileRequirement(Requirement): filename: str def __init__(self, filename: str): - super(RubyFileRequirement, self).__init__('ruby-file') + super(RubyFileRequirement, self).__init__("ruby-file") self.filename = filename @@ -213,7 +213,7 @@ class XmlEntityRequirement(Requirement): url: str def __init__(self, url: str): - super(XmlEntityRequirement, self).__init__('xml-entity') + super(XmlEntityRequirement, self).__init__("xml-entity") self.url = url @@ -223,7 +223,7 @@ class SprocketsFileRequirement(Requirement): name: str def __init__(self, content_type: str, name: str): - super(SprocketsFileRequirement, self).__init__('sprockets-file') + super(SprocketsFileRequirement, self).__init__("sprockets-file") self.content_type = content_type self.name = name @@ -233,7 +233,7 @@ class JavaClassRequirement(Requirement): classname: str def __init__(self, classname: str): - super(JavaClassRequirement, self).__init__('java-class') + super(JavaClassRequirement, self).__init__("java-class") self.classname = classname @@ -242,7 +242,7 @@ class HaskellPackageRequirement(Requirement): package: str def __init__(self, package: str): - super(HaskellPackageRequirement, self).__init__('haskell-package') + super(HaskellPackageRequirement, self).__init__("haskell-package") self.package = package @@ -251,14 +251,13 @@ class MavenArtifactRequirement(Requirement): artifacts: List[Tuple[str, str, str]] def __init__(self, artifacts): - super(MavenArtifactRequirement, self).__init__('maven-artifact') + super(MavenArtifactRequirement, self).__init__("maven-artifact") self.artifacts = artifacts class GnomeCommonRequirement(Requirement): - def __init__(self): - super(GnomeCommonRequirement, self).__init__('gnome-common') + super(GnomeCommonRequirement, self).__init__("gnome-common") class JDKFileRequirement(Requirement): @@ -267,7 +266,7 @@ class JDKFileRequirement(Requirement): filename: str def __init__(self, jdk_path: str, filename: str): - super(JDKFileRequirement, self).__init__('jdk-file') + super(JDKFileRequirement, self).__init__("jdk-file") self.jdk_path = jdk_path self.filename = filename @@ -281,7 +280,7 @@ class PerlFileRequirement(Requirement): filename: str def __init__(self, filename: str): - super(PerlFileRequirement, self).__init__('perl-file') + super(PerlFileRequirement, self).__init__("perl-file") self.filename = filename @@ -290,7 +289,7 @@ class AutoconfMacroRequirement(Requirement): macro: str def __init__(self, macro: str): - super(AutoconfMacroRequirement, self).__init__('autoconf-macro') + super(AutoconfMacroRequirement, self).__init__("autoconf-macro") self.macro = macro @@ -301,6 +300,6 @@ class PythonModuleRequirement(Requirement): minimum_version: Optional[str] def __init__(self, module, python_version=None, minimum_version=None): - super(PythonModuleRequirement, self).__init__('python-module') + super(PythonModuleRequirement, self).__init__("python-module") self.python_version = python_version self.minimum_version = minimum_version diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index bd72c51..b764eda 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -17,13 +17,11 @@ class UnsatisfiedRequirements(Exception): - def __init__(self, reqs): self.requirements = reqs class Resolver(object): - def install(self, requirements): raise NotImplementedError(self.install) @@ -38,7 +36,6 @@ class Resolver(object): class CPANResolver(Resolver): - def __init__(self, session): self.session = session @@ -47,6 +44,7 @@ class CPANResolver(Resolver): def install(self, requirements): from ..requirements import PerlModuleRequirement + missing = [] for requirement in requirements: if not isinstance(requirement, PerlModuleRequirement): @@ -55,7 +53,8 @@ class CPANResolver(Resolver): # TODO(jelmer): Specify -T to skip tests? self.session.check_call( ["cpan", "-i", requirement.module], - user="root", env={"PERL_MM_USE_DEFAULT": "1"} + user="root", + env={"PERL_MM_USE_DEFAULT": "1"}, ) if missing: raise UnsatisfiedRequirements(missing) @@ -65,7 +64,6 @@ class CPANResolver(Resolver): class HackageResolver(Resolver): - def __init__(self, session): self.session = session @@ -74,14 +72,15 @@ class HackageResolver(Resolver): def install(self, requirements): from ..requirements import HaskellPackageRequirement + missing = [] for requirement in requirements: if not isinstance(requirement, HaskellPackageRequirement): missing.append(requirement) continue self.session.check_call( - ["cabal", "install", requirement.package], - user="root") + ["cabal", "install", requirement.package], user="root" + ) if missing: raise UnsatisfiedRequirements(missing) @@ -90,7 +89,6 @@ class HackageResolver(Resolver): class CargoResolver(Resolver): - def __init__(self, session): self.session = session @@ -99,14 +97,15 @@ class CargoResolver(Resolver): def install(self, requirements): from ..requirements import CargoCrateRequirement + missing = [] for requirement in requirements: if not isinstance(requirement, CargoCrateRequirement): missing.append(requirement) continue self.session.check_call( - ["cargo", "install", requirement.crate], - user="root") + ["cargo", "install", requirement.crate], user="root" + ) if missing: raise UnsatisfiedRequirements(missing) @@ -115,7 +114,6 @@ class CargoResolver(Resolver): class PypiResolver(Resolver): - def __init__(self, session): self.session = session @@ -124,6 +122,7 @@ class PypiResolver(Resolver): def install(self, requirements): from ..requirements import PythonPackageRequirement + missing = [] for requirement in requirements: if not isinstance(requirement, PythonPackageRequirement): @@ -143,7 +142,6 @@ NPM_COMMAND_PACKAGES = { class NpmResolver(Resolver): - def __init__(self, session): self.session = session @@ -152,6 +150,7 @@ class NpmResolver(Resolver): def install(self, requirements): from ..requirements import NodePackageRequirement + missing = [] for requirement in requirements: if not isinstance(requirement, NodePackageRequirement): @@ -191,12 +190,15 @@ class StackedResolver(Resolver): def native_resolvers(session): - return StackedResolver([ - CPANResolver(session), - PypiResolver(session), - NpmResolver(session), - CargoResolver(session), - HackageResolver(session)]) + return StackedResolver( + [ + CPANResolver(session), + PypiResolver(session), + NpmResolver(session), + CargoResolver(session), + HackageResolver(session), + ] + ) class ExplainResolver(Resolver): @@ -215,14 +217,18 @@ def auto_resolver(session): # TODO(jelmer): if session is SchrootSession or if we're root, use apt from .apt import AptResolver from ..session.schroot import SchrootSession - user = session.check_output(['echo', '$USER']).decode().strip() + + user = session.check_output(["echo", "$USER"]).decode().strip() resolvers = [] - if isinstance(session, SchrootSession) or user == 'root': + if isinstance(session, SchrootSession) or user == "root": resolvers.append(AptResolver.from_session(session)) - resolvers.extend([ - CPANResolver(session), - PypiResolver(session), - NpmResolver(session), - CargoResolver(session), - HackageResolver(session)]) + resolvers.extend( + [ + CPANResolver(session), + PypiResolver(session), + NpmResolver(session), + CargoResolver(session), + HackageResolver(session), + ] + ) return StackedResolver(resolvers) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index d7d543d..c6ccc12 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -32,7 +32,6 @@ from ..requirements import ( CHeaderRequirement, PkgConfigRequirement, PathRequirement, - Requirement, JavaScriptRuntimeRequirement, ValaPackageRequirement, RubyGemRequirement, @@ -55,20 +54,19 @@ from ..requirements import ( AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, - ) +) class AptRequirement(Requirement): - def __init__(self, relations): - super(AptRequirement, self).__init__('apt') + super(AptRequirement, self).__init__("apt") self.relations = relations @classmethod def simple(cls, package, minimum_version=None): - rel = {'name': package} + rel = {"name": package} if minimum_version is not None: - rel['version'] = ('>=', minimum_version) + rel["version"] = (">=", minimum_version) return cls([[rel]]) @classmethod @@ -81,35 +79,50 @@ class AptRequirement(Requirement): def touches_package(self, package): for rel in self.relations: for entry in rel: - if entry['name'] == package: + if entry["name"] == package: return True return False +def python_spec_to_apt_rels(pkg_name, specs): + # TODO(jelmer): Dealing with epoch, etc? + if not specs: + return [[{"name": pkg_name}]] + else: + rels = [] + for spec in specs: + c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "=": "="}[spec[0]] + rels.append([{"name": pkg_name, "version": (c, Version(spec[1]))}]) + return rels + + def get_package_for_python_package(apt_mgr, package, python_version, specs=None): if python_version == "pypy": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % package], - regex=True) + ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % package.replace("-", "_")], + regex=True, + ) elif python_version == "cpython2": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % package], - regex=True) + [ + "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" + % package.replace("-", "_") + ], + regex=True, + ) elif python_version == "cpython3": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % package], - regex=True) + [ + "/usr/lib/python3/dist-packages/%s-.*.egg-info" + % package.replace("-", "_") + ], + regex=True, + ) else: raise NotImplementedError if pkg_name is None: return None - # TODO(jelmer): Dealing with epoch, etc? - if not specs: - rels = [[{'name': pkg_name}]] - else: - rels = [] - for spec in specs: - rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}]) + rels = python_spec_to_apt_rels(pkg_name, specs) return AptRequirement(rels) @@ -169,13 +182,7 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is None: return None - rels = [] - if not specs: - rels = [[{'name': pkg_name}]] - else: - rels = [] - for spec in specs: - rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}]) + rels = python_spec_to_apt_rels(pkg_name, specs) return AptRequirement(rels) @@ -184,8 +191,7 @@ def resolve_binary_req(apt_mgr, req): paths = [req.binary_name] else: paths = [ - posixpath.join(dirname, req.binary_name) - for dirname in ["/usr/bin", "/bin"] + posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] ] pkg_name = apt_mgr.get_package_for_paths(paths) if pkg_name is not None: @@ -200,7 +206,8 @@ def resolve_pkg_config_req(apt_mgr, req): if package is None: package = apt_mgr.get_package_for_paths( [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], - regex=True) + regex=True, + ) if package is not None: return AptRequirement.simple(package, minimum_version=req.minimum_version) return None @@ -228,7 +235,8 @@ def resolve_c_header_req(apt_mgr, req): def resolve_js_runtime_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( - ["/usr/bin/node", "/usr/bin/duk"], regex=False) + ["/usr/bin/node", "/usr/bin/duk"], regex=False + ) if package is not None: return AptRequirement.simple(package) return None @@ -249,8 +257,7 @@ def resolve_ruby_gem_req(apt_mgr, req): "specifications/%s-.*\\.gemspec" % req.gem ) ] - package = apt_mgr.get_package_for_paths( - paths, regex=True) + package = apt_mgr.get_package_for_paths(paths, regex=True) if package is not None: return AptRequirement.simple(package, minimum_version=req.minimum_version) return None @@ -258,8 +265,7 @@ def resolve_ruby_gem_req(apt_mgr, req): def resolve_go_package_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/share/gocode/src", req.package, ".*")], - regex=True + [posixpath.join("/usr/share/gocode/src", req.package, ".*")], regex=True ) if package is not None: return AptRequirement.simple(package) @@ -368,7 +374,8 @@ def resolve_java_class_req(apt_mgr, req): # system :( # TODO(jelmer): Call in session output = apt_mgr.session.check_output( - ["java-propose-classpath", "-c" + req.classname]) + ["java-propose-classpath", "-c" + req.classname] + ) classpath = [p for p in output.decode().strip(":").strip().split(":") if p] if not classpath: logging.warning("unable to find classpath for %s", req.classname) @@ -422,7 +429,7 @@ def resolve_maven_artifact_req(apt_mgr, req): def resolve_gnome_common_req(apt_mgr, req): - return AptRequirement.simple('gnome-common') + return AptRequirement.simple("gnome-common") def resolve_jdk_file_req(apt_mgr, req): @@ -438,8 +445,7 @@ def resolve_perl_module_req(apt_mgr, req): if req.inc is None: if req.filename is None: - paths = [posixpath.join(inc, req.relfilename) - for inc in DEFAULT_PERL_PATHS] + paths = [posixpath.join(inc, req.relfilename) for inc in DEFAULT_PERL_PATHS] elif not posixpath.isabs(req.filename): return False else: @@ -495,9 +501,13 @@ def resolve_python_module_req(apt_mgr, req): def resolve_python_package_req(apt_mgr, req): if req.python_version == 2: - return get_package_for_python_package(apt_mgr, req.package, "cpython2", req.specs) + return get_package_for_python_package( + apt_mgr, req.package, "cpython2", req.specs + ) elif req.python_version in (None, 3): - return get_package_for_python_package(apt_mgr, req.package, "cpython3", req.specs) + return get_package_for_python_package( + apt_mgr, req.package, "cpython3", req.specs + ) else: return None @@ -540,7 +550,6 @@ def resolve_requirement_apt(apt_mgr, req: Requirement) -> AptRequirement: class AptResolver(Resolver): - def __init__(self, apt): self.apt = apt @@ -570,8 +579,9 @@ class AptResolver(Resolver): else: apt_requirements.append(apt_req) if apt_requirements: - self.apt.satisfy([PkgRelation.str(chain(*[ - r.relations for r in apt_requirements]))]) + self.apt.satisfy( + [PkgRelation.str(chain(*[r.relations for r in apt_requirements]))] + ) if still_missing: raise UnsatisfiedRequirements(still_missing) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 8941844..3677262 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -62,8 +62,8 @@ class SchrootSession(Session): # TODO(jelmer): Capture stderr and forward in SessionSetupFailure raise SessionSetupFailure() logging.info( - 'Opened schroot session %s (from %s)', self.session_id, - self.chroot) + "Opened schroot session %s (from %s)", self.session_id, self.chroot + ) return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -157,7 +157,7 @@ class SchrootSession(Session): def _fullpath(self, path: str) -> str: if self._cwd is None: - raise ValueError('no cwd set') + raise ValueError("no cwd set") return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/")) def exists(self, path: str) -> bool: diff --git a/ognibuild/tests/test_debian_build.py b/ognibuild/tests/test_debian_build.py index 274f5f8..6e5970a 100644 --- a/ognibuild/tests/test_debian_build.py +++ b/ognibuild/tests/test_debian_build.py @@ -158,11 +158,10 @@ janitor (0.1-1jan+some1) UNRELEASED; urgency=medium class BuildArchitectureTests(TestCase): - def setUp(self): super(BuildArchitectureTests, self).setUp() - if not os.path.exists('/usr/bin/dpkg-architecture'): - self.skipTest('not a debian system') + if not os.path.exists("/usr/bin/dpkg-architecture"): + self.skipTest("not a debian system") def test_is_str(self): self.assertIsInstance(get_build_architecture(), str) diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index 6246c03..5283a3e 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -30,7 +30,6 @@ from buildlog_consultant.common import ( MissingRubyGem, MissingValaPackage, ) -from ..debian import apt from ..debian.apt import AptManager, FileSearcher from ..debian.fix_build import ( resolve_error, @@ -42,7 +41,6 @@ from breezy.tests import TestCaseWithTransport class DummyAptSearcher(FileSearcher): - def __init__(self, files): self._apt_files = files @@ -59,8 +57,8 @@ class DummyAptSearcher(FileSearcher): class ResolveErrorTests(TestCaseWithTransport): def setUp(self): super(ResolveErrorTests, self).setUp() - if not os.path.exists('/usr/bin/dpkg-architecture'): - self.skipTest('not a debian system') + if not os.path.exists("/usr/bin/dpkg-architecture"): + self.skipTest("not a debian system") self.tree = self.make_branch_and_tree(".") self.build_tree_contents( [ @@ -95,6 +93,7 @@ blah (0.1) UNRELEASED; urgency=medium def resolve(self, error, context=("build",)): from ..session.plain import PlainSession + session = PlainSession() apt = AptManager(session) apt._searchers = [DummyAptSearcher(self._apt_files)] @@ -122,8 +121,8 @@ blah (0.1) UNRELEASED; urgency=medium "/usr/bin/brz": "brz", "/usr/bin/brzier": "bash", } - self.overrideEnv('DEBEMAIL', 'jelmer@debian.org') - self.overrideEnv('DEBFULLNAME', 'Jelmer Vernooij') + self.overrideEnv("DEBEMAIL", "jelmer@debian.org") + self.overrideEnv("DEBFULLNAME", "Jelmer Vernooij") self.assertTrue(self.resolve(MissingCommand("brz"))) self.assertEqual("libc6, brz", self.get_build_deps()) rev = self.tree.branch.repository.get_revision(self.tree.branch.last_revision()) From 10edeef330eaa90e8705d722b32b8da8ede2a59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 01:16:24 +0000 Subject: [PATCH 14/21] Add doc on concepts. --- notes/concepts.md | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 notes/concepts.md diff --git a/notes/concepts.md b/notes/concepts.md new file mode 100644 index 0000000..b0c97d8 --- /dev/null +++ b/notes/concepts.md @@ -0,0 +1,49 @@ +Requirement +=========== + +Some sort of constraint about the environment that can be specified and satisfied. + +Examples: +* a dependency on version 1.3 of the python package "foo" +* a dependency on the apt package "blah" + +Requirements can be discovered from build system metadata files and from build logs. + +Different kinds of requirements are subclassed from the main Requirement class. + +Output +====== + +A build artifact that can be produced by a build system, e.g. an +executable file or a Perl module. + +Problem +======= + +An issue found in a build log by buildlog-consultant. + +BuildFixer +========== + +Takes a build problem and tries to resolve it in some way. + +This can mean changing the project that's being built +(by modifying the source tree), or changing the environment +(e.g. by install packages from apt). + +Common fixers: + + + InstallFixer([(resolver, repository)]) + + DebianDependencyFixer(tree, resolver) + +Repository +========== + +Some sort of provider of external requirements. Can satisfy environment +requirements. + +Resolver +======== + +Can take one kind of upstream requirement and turn it into another. E.g. +converting missing Python modules to apt or pypi packages. From ea32f33f90faf49d5026f624d6002e17f4058f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 01:16:51 +0000 Subject: [PATCH 15/21] Add architecture. --- notes/architecture.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/notes/architecture.md b/notes/architecture.md index 02ee04f..7076f88 100644 --- a/notes/architecture.md +++ b/notes/architecture.md @@ -1,11 +1,11 @@ -Upstream requirements are expressed as objects derived from UpstreamRequirement. +Upstream requirements are expressed as objects derived from Requirement. They can either be: * extracted from the build system * extracted from errors in build logs -The details of UpstreamRequirements are specific to the kind of requirement, +The details of Requirements are specific to the kind of requirement, and otherwise opaque to ognibuild. When building a package, we first make sure that all declared upstream @@ -21,10 +21,10 @@ like e.g. upgrade configure.ac to a newer version, or invoke autoreconf. A list of possible fixers can be provided. Each fixer will be called (in order) until one of them claims to ahve fixed the issue. -Problems can be converted to UpstreamRequirements by UpstreamRequirementFixer +Problems can be converted to Requirements by RequirementFixer -UpstreamRequirementFixer uses a UpstreamRequirementResolver object that -can translate UpstreamRequirement objects into apt package names or +InstallFixer uses a Resolver object that +can translate Requirement objects into apt package names or e.g. cpan commands. ognibuild keeps finding problems, resolving them and rebuilding until it finds @@ -38,14 +38,14 @@ on the host machine. For e.g. PerlModuleRequirement, need to be able to: * install from apt package - + DebianInstallFixer(AptResolver()).fix(problem) + + InstallFixer(AptResolver()).fix(problem) * update debian package (source, runtime, test) deps to include apt package + DebianPackageDepFixer(AptResolver()).fix(problem, ('test', 'foo')) * suggest command to run to install from apt package - + DebianInstallFixer(AptResolver()).command(problem) + + InstallFixer(AptResolver()).command(problem) * install from cpan - + CpanInstallFixer().fix(problem) + + InstallFixer(CpanResolver()).fix(problem) * suggest command to run to install from cpan package - + CpanInstallFixer().command(problem) + + InstallFixer(CpanResolver()).command(problem) * update source package reqs to depend on perl module + PerlDepFixer().fix(problem) From 52e119022b5c89ab6387e863f901a407316b7fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 02:18:36 +0000 Subject: [PATCH 16/21] More refactoring. --- ognibuild/__main__.py | 4 +- ognibuild/buildlog.py | 16 +++--- ognibuild/buildsystem.py | 37 ++++++++++-- ognibuild/debian/fix_build.py | 71 ++++++++++++++++++++++-- ognibuild/dist.py | 4 +- ognibuild/fix_build.py | 17 +----- ognibuild/requirements.py | 9 +++ ognibuild/resolver/__init__.py | 39 ++++--------- ognibuild/session/plain.py | 3 + ognibuild/tests/test_debian_fix_build.py | 1 + 10 files changed, 135 insertions(+), 66 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index f500051..af14c31 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -67,8 +67,8 @@ STAGE_MAP = { def determine_fixers(session, resolver): - from .buildlog import RequirementFixer - return [RequirementFixer(resolver)] + from .buildlog import InstallFixer + return [InstallFixer(resolver)] def main(): # noqa: C901 diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 3560e32..8a8c1e3 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -81,6 +81,7 @@ from .requirements import ( PythonModuleRequirement, PythonPackageRequirement, ) +from .resolver import UnsatisfiedRequirements def problem_to_upstream_requirement(problem): # noqa: C901 @@ -166,7 +167,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return None -class RequirementFixer(BuildFixer): +class InstallFixer(BuildFixer): def __init__(self, resolver): self.resolver = resolver @@ -188,11 +189,8 @@ class RequirementFixer(BuildFixer): if not isinstance(reqs, list): reqs = [reqs] - changed = False - for req in reqs: - package = self.resolver.resolve(req) - if package is None: - return False - if context.add_dependency(package): - changed = True - return changed + try: + self.resolver.install(reqs) + except UnsatisfiedRequirements: + return False + return True diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index d309f50..a951ee7 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -457,7 +457,7 @@ class Make(BuildSystem): run_with_build_fixers(session, ["autoreconf", "-i"], fixers) if not makefile_exists() and session.exists("configure"): - session.check_call(["./configure"]) + run_with_build_fixers(session, ["./configure"], fixers) def build(self, session, resolver, fixers): self.setup(session, resolver, fixers) @@ -553,9 +553,14 @@ class Cargo(BuildSystem): name = "cargo" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + def __init__(self, path): from toml.decoder import load + self.path = path + with open(path, "r") as f: self.cargo = load(f) @@ -568,12 +573,30 @@ class Cargo(BuildSystem): def test(self, session, resolver, fixers): run_with_build_fixers(session, ["cargo", "test"], fixers) + def clean(self, session, resolver, fixers): + run_with_build_fixers(session, ["cargo", "clean"], fixers) + + def build(self, session, resolver, fixers): + run_with_build_fixers(session, ["cargo", "build"], fixers) + class Golang(BuildSystem): """Go builds.""" name = "golang" + def __repr__(self): + return "%s()" % (type(self).__name__) + + def test(self, session, resolver, fixers): + session.check_call(["go", "test"]) + + def build(self, session, resolver, fixers): + session.check_call(["go", "build"]) + + def clean(self, session, resolver, fixers): + session.check_call(["go", "clean"]) + class Maven(BuildSystem): @@ -664,6 +687,7 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 "GNUmakefile", "makefile", "Makefile.PL", + "CMakeLists.txt", "autogen.sh", "configure.ac", "configure.in", @@ -672,6 +696,7 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 ): yield Make() + seen_golang = False if os.path.exists(os.path.join(path, ".travis.yml")): import ruamel.yaml.reader @@ -684,11 +709,13 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 language = data.get("language") if language == "go": yield Golang() + seen_golang = True - for entry in os.scandir(path): - if entry.name.endswith(".go"): - yield Golang() - break + if not seen_golang: + for entry in os.scandir(path): + if entry.name.endswith(".go"): + yield Golang() + break def get_buildsystem(path, trust_package=False): diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index ca8b2b9..753a454 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -79,8 +79,8 @@ from buildlog_consultant.sbuild import ( SbuildFailure, ) +from ..buildlog import problem_to_upstream_requirement from ..fix_build import BuildFixer, resolve_error, DependencyContext -from ..buildlog import RequirementFixer from ..resolver.apt import ( AptRequirement, get_package_for_python_module, @@ -98,7 +98,66 @@ class CircularDependency(Exception): self.package = package +class PackageDependencyFixer(BuildFixer): + + def __init__(self, apt_resolver): + self.apt_resolver = apt_resolver + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.apt_resolver) + + def __str__(self): + return "upstream requirement fixer(%s)" % self.apt_resolver + + def can_fix(self, error): + req = problem_to_upstream_requirement(error) + return req is not None + + def fix(self, error, context): + reqs = problem_to_upstream_requirement(error) + if reqs is None: + return False + + if not isinstance(reqs, list): + reqs = [reqs] + + changed = False + for req in reqs: + package = self.apt_resolver.resolve(req) + if package is None: + return False + if context.phase[0] == "autopkgtest": + return add_test_dependency( + context.tree, + context.phase[1], + package, + committer=context.committer, + subpath=context.subpath, + update_changelog=context.update_changelog, + ) + elif context.phase[0] == "build": + return add_build_dependency( + context.tree, + package, + committer=context.committer, + subpath=context.subpath, + update_changelog=context.update_changelog, + ) + else: + logging.warning('Unknown phase %r', context.phase) + return False + return changed + + class BuildDependencyContext(DependencyContext): + def __init__( + self, phase, tree, apt, subpath="", committer=None, update_changelog=True + ): + self.phase = phase + super(BuildDependencyContext, self).__init__( + tree, apt, subpath, committer, update_changelog + ) + def add_dependency(self, requirement: AptRequirement): return add_build_dependency( self.tree, @@ -111,9 +170,9 @@ class BuildDependencyContext(DependencyContext): class AutopkgtestDependencyContext(DependencyContext): def __init__( - self, testname, tree, apt, subpath="", committer=None, update_changelog=True + self, phase, tree, apt, subpath="", committer=None, update_changelog=True ): - self.testname = testname + self.phase = phase super(AutopkgtestDependencyContext, self).__init__( tree, apt, subpath, committer, update_changelog ) @@ -498,13 +557,12 @@ def versioned_package_fixers(session): def apt_fixers(apt) -> List[BuildFixer]: from ..resolver.apt import AptResolver - resolver = AptResolver(apt) return [ SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution), SimpleBuildFixer(AptFetchFailure, retry_apt_failure), - RequirementFixer(resolver), + PackageDependencyFixer(resolver), ] @@ -553,6 +611,7 @@ def build_incrementally( reset_tree(local_tree, local_tree.basis_tree(), subpath=subpath) if e.phase[0] == "build": context = BuildDependencyContext( + e.phase, local_tree, apt, subpath=subpath, @@ -561,7 +620,7 @@ def build_incrementally( ) elif e.phase[0] == "autopkgtest": context = AutopkgtestDependencyContext( - e.phase[1], + e.phase, local_tree, apt, subpath=subpath, diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 551a4b0..cfdb5db 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -133,7 +133,7 @@ def create_dist_schroot( ) -> str: from .buildsystem import detect_buildsystems from .resolver.apt import AptResolver - from .buildlog import RequirementFixer + from .buildlog import InstallFixer if subdir is None: subdir = "package" @@ -159,7 +159,7 @@ def create_dist_schroot( buildsystems = list(detect_buildsystems(export_directory)) resolver = AptResolver.from_session(session) - fixers = [RequirementFixer(resolver)] + fixers = [InstallFixer(resolver)] with DistCatcher(export_directory) as dc: oldcwd = os.getcwd() diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 3ffbf3f..d7dca7f 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -58,23 +58,10 @@ class DependencyContext(object): self.committer = committer self.update_changelog = update_changelog - def add_dependency( - self, package: str, minimum_version=None - ) -> bool: + def add_dependency(self, package) -> bool: raise NotImplementedError(self.add_dependency) -class SchrootDependencyContext(DependencyContext): - def __init__(self, session): - self.session = session - self.apt = AptManager(session) - - def add_dependency(self, package, minimum_version=None): - # TODO(jelmer): Handle minimum_version - self.apt.install([package]) - return True - - def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer]): logging.info("Running %r", args) fixed_errors = [] @@ -99,7 +86,7 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF raise DetailedFailure(retcode, args, error) if not resolve_error( error, - SchrootDependencyContext(session), + None, fixers=fixers, ): logging.warning("Failed to find resolution for error %r. Giving up.", error) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 1a46852..560ad29 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -98,6 +98,15 @@ class CargoCrateRequirement(Requirement): super(CargoCrateRequirement, self).__init__("cargo-crate") self.crate = crate + def __repr__(self): + return "%s(%r)" % ( + type(self).__name__, + self.crate, + ) + + def __str__(self): + return "cargo crate: %s" % self.crate + class PkgConfigRequirement(Requirement): diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index b764eda..8eca42c 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -42,6 +42,9 @@ class CPANResolver(Resolver): def __str__(self): return "cpan" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + def install(self, requirements): from ..requirements import PerlModuleRequirement @@ -70,6 +73,9 @@ class HackageResolver(Resolver): def __str__(self): return "hackage" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + def install(self, requirements): from ..requirements import HaskellPackageRequirement @@ -88,31 +94,6 @@ class HackageResolver(Resolver): raise NotImplementedError(self.explain) -class CargoResolver(Resolver): - def __init__(self, session): - self.session = session - - def __str__(self): - return "cargo" - - def install(self, requirements): - from ..requirements import CargoCrateRequirement - - missing = [] - for requirement in requirements: - if not isinstance(requirement, CargoCrateRequirement): - missing.append(requirement) - continue - self.session.check_call( - ["cargo", "install", requirement.crate], user="root" - ) - if missing: - raise UnsatisfiedRequirements(missing) - - def explain(self, requirements): - raise NotImplementedError(self.explain) - - class PypiResolver(Resolver): def __init__(self, session): self.session = session @@ -120,6 +101,9 @@ class PypiResolver(Resolver): def __str__(self): return "pypi" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + def install(self, requirements): from ..requirements import PythonPackageRequirement @@ -148,6 +132,9 @@ class NpmResolver(Resolver): def __str__(self): return "npm" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + def install(self, requirements): from ..requirements import NodePackageRequirement @@ -195,7 +182,6 @@ def native_resolvers(session): CPANResolver(session), PypiResolver(session), NpmResolver(session), - CargoResolver(session), HackageResolver(session), ] ) @@ -227,7 +213,6 @@ def auto_resolver(session): CPANResolver(session), PypiResolver(session), NpmResolver(session), - CargoResolver(session), HackageResolver(session), ] ) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 749bf49..b1f237c 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -27,6 +27,9 @@ class PlainSession(Session): location = "/" + def __repr__(self): + return "%s()" % (type(self).__name__, ) + def create_home(self): pass diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index 5283a3e..a06884a 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -98,6 +98,7 @@ blah (0.1) UNRELEASED; urgency=medium apt = AptManager(session) apt._searchers = [DummyAptSearcher(self._apt_files)] context = BuildDependencyContext( + ("build", ), self.tree, apt, subpath="", From dd015abd4ad8a0c916745e5c43c4eac2f8e83ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 03:22:25 +0000 Subject: [PATCH 17/21] Check requirements before. --- ognibuild/__main__.py | 18 +++++++---- ognibuild/buildlog.py | 2 +- ognibuild/buildsystem.py | 3 ++ ognibuild/requirements.py | 55 +++++++++++++++++++++++++++++++++- ognibuild/resolver/__init__.py | 3 +- 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index af14c31..3a40ef1 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -35,8 +35,8 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(resolver, buildsystem, stages): - missing = [] +def install_necessary_declared_requirements(session, resolver, buildsystem, stages): + relevant = [] try: declared_reqs = list(buildsystem.get_declared_dependencies()) except NotImplementedError: @@ -44,10 +44,18 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages): "Unable to determine declared dependencies from %s", buildsystem ) else: - missing.extend( + relevant.extend( get_necessary_declared_requirements(resolver, declared_reqs, stages) ) - resolver.install(missing) + missing = [] + for req in relevant: + try: + if not req.met(session): + missing.append(req) + except NotImplementedError: + missing.append(req) + if missing: + resolver.install(missing) # Types of dependencies: @@ -141,7 +149,7 @@ def main(): # noqa: C901 if stages: logging.info("Checking that declared requirements are present") for bs in bss: - install_necessary_declared_requirements(resolver, bs, stages) + install_necessary_declared_requirements(session, resolver, bs, stages) fixers = determine_fixers(session, resolver) if args.subcommand == "dist": from .dist import run_dist diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 8a8c1e3..33d51ed 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -120,7 +120,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 elif isinstance(problem, MissingJavaClass): return JavaClassRequirement(problem.classname) elif isinstance(problem, MissingHaskellDependencies): - return [HaskellPackageRequirement(dep) for dep in problem.deps] + return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps] elif isinstance(problem, MissingMavenArtifacts): return [MavenArtifactRequirement(artifact) for artifact in problem.artifacts] elif isinstance(problem, MissingCSharpCompiler): diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index a951ee7..5291b4f 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -633,6 +633,9 @@ class Cabal(BuildSystem): def test(self, session, resolver, fixers): self._run(session, ["test"], fixers) + def dist(self, session, resolver, fixers, quiet=False): + self._run(session, ["sdist"], fixers) + def detect_buildsystems(path, trust_package=False): # noqa: C901 """Detect build systems.""" diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 560ad29..2f3673f 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -17,6 +17,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import posixpath +import subprocess from typing import Optional, List, Tuple from . import Requirement @@ -55,6 +56,26 @@ class PythonPackageRequirement(Requirement): req = Requirement.parse(text) return cls(package=req.name, specs=req.specs) + def met(self, session): + if self.python_version == "cpython3": + cmd = "python3" + elif self.python_version == "cpython2": + cmd = "python2" + elif self.python_version == "pypy": + cmd = "pypy" + elif self.python_version == "pypy3": + cmd = "pypy3" + elif self.python_version is None: + cmd = "python3" + else: + raise NotImplementedError + text = self.package + ','.join([''.join(spec) for spec in self.specs]) + p = session.Popen( + [cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + p.communicate() + return p.returncode == 0 + class BinaryRequirement(Requirement): @@ -64,6 +85,13 @@ class BinaryRequirement(Requirement): super(BinaryRequirement, self).__init__("binary") self.binary_name = binary_name + def met(self, session): + p = session.Popen( + ["which", self.binary_name], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + p.communicate() + return p.returncode == 0 + class PerlModuleRequirement(Requirement): @@ -250,9 +278,15 @@ class HaskellPackageRequirement(Requirement): package: str - def __init__(self, package: str): + def __init__(self, package: str, specs=None): super(HaskellPackageRequirement, self).__init__("haskell-package") self.package = package + self.specs = specs + + @classmethod + def from_string(cls, text): + parts = text.split() + return cls(parts[0], specs=parts[1:]) class MavenArtifactRequirement(Requirement): @@ -312,3 +346,22 @@ class PythonModuleRequirement(Requirement): super(PythonModuleRequirement, self).__init__("python-module") self.python_version = python_version self.minimum_version = minimum_version + + def met(self, session): + if self.python_version == "cpython3": + cmd = "python3" + elif self.python_version == "cpython2": + cmd = "python2" + elif self.python_version == "pypy": + cmd = "pypy" + elif self.python_version == "pypy3": + cmd = "pypy3" + elif self.python_version is None: + cmd = "python3" + else: + raise NotImplementedError + p = session.Popen( + [cmd, "-c", "import %s" % self.module], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + p.communicate() + return p.returncode == 0 diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 8eca42c..0bd5c24 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -56,7 +56,6 @@ class CPANResolver(Resolver): # TODO(jelmer): Specify -T to skip tests? self.session.check_call( ["cpan", "-i", requirement.module], - user="root", env={"PERL_MM_USE_DEFAULT": "1"}, ) if missing: @@ -85,7 +84,7 @@ class HackageResolver(Resolver): missing.append(requirement) continue self.session.check_call( - ["cabal", "install", requirement.package], user="root" + ["cabal", "install", requirement.package] ) if missing: raise UnsatisfiedRequirements(missing) From 0fa372afd40a362acce000d5a9652e64bdb08074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 04:19:06 +0000 Subject: [PATCH 18/21] More build systems --- ognibuild/buildsystem.py | 71 +++++++++++++++++++++++++++++++++- ognibuild/resolver/__init__.py | 54 ++++++++++++++++++-------- 2 files changed, 107 insertions(+), 18 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 5291b4f..2b47cc0 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -258,6 +258,59 @@ class SetupPy(BuildSystem): yield PythonPackageOutput(package, python_version="cpython3") +class Gradle(BuildSystem): + + name = "gradle" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def clean(self, session, resolver, fixers): + run_with_build_fixers(session, ["gradle", "clean"], fixers) + + def build(self, session, resolver, fixers): + run_with_build_fixers(session, ["gradle", "build"], fixers) + + def test(self, session, resolver, fixers): + run_with_build_fixers(session, ["gradle", "test"], fixers) + + +class Meson(BuildSystem): + + name = "meson" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def _setup(self, session, fixers): + if session.exists("build"): + return + session.mkdir("build") + run_with_build_fixers(session, ["meson", "setup", "build"], fixers) + + def clean(self, session, resolver, fixers): + self._setup(session, fixers) + run_with_build_fixers(session, ["ninja", "-C", "build", "clean"], fixers) + + def build(self, session, resolver, fixers): + self._setup(session, fixers) + run_with_build_fixers(session, ["ninja", "-C", "build"], fixers) + + def test(self, session, resolver, fixers): + self._setup(session, fixers) + run_with_build_fixers(session, ["ninja", "-C", "build", "test"], fixers) + + def install(self, session, resolver, fixers, install_target): + self._setup(session, fixers) + run_with_build_fixers(session, ["ninja", "-C", "build", "install"], fixers) + + class PyProject(BuildSystem): name = "pyproject" @@ -266,6 +319,9 @@ class PyProject(BuildSystem): self.path = path self.pyproject = self.load_toml() + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + def load_toml(self): import toml @@ -589,10 +645,13 @@ class Golang(BuildSystem): return "%s()" % (type(self).__name__) def test(self, session, resolver, fixers): - session.check_call(["go", "test"]) + run_with_build_fixers(session, ["go", "test"], fixers) def build(self, session, resolver, fixers): - session.check_call(["go", "build"]) + run_with_build_fixers(session, ["go", "build"], fixers) + + def install(self, session, resolver, fixers): + run_with_build_fixers(session, ["go", "install"], fixers) def clean(self, session, resolver, fixers): session.check_call(["go", "clean"]) @@ -665,6 +724,14 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 logging.debug("Found Cargo.toml, assuming rust cargo package.") yield Cargo("Cargo.toml") + if os.path.exists(os.path.join(path, "build.gradle")): + logging.debug("Found build.gradle, assuming gradle package.") + yield Gradle("build.gradle") + + if os.path.exists(os.path.join(path, "meson.build")): + logging.debug("Found meson.build, assuming meson package.") + yield Meson("meson.build") + if os.path.exists(os.path.join(path, "Setup.hs")): logging.debug("Found Setup.hs, assuming haskell package.") yield Cabal("Setup.hs") diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 0bd5c24..741144c 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -119,6 +119,33 @@ class PypiResolver(Resolver): raise NotImplementedError(self.explain) +class GoResolver(Resolver): + + def __init__(self, session): + self.session = session + + def __str__(self): + return "go" + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + + def install(self, requirements): + from ..requirements import GoPackageRequirement + + missing = [] + for requirement in requirements: + if not isinstance(requirement, GoPackageRequirement): + missing.append(requirement) + continue + self.session.check_call(["go", "get", requirement.package]) + if missing: + raise UnsatisfiedRequirements(missing) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + NPM_COMMAND_PACKAGES = { "del-cli": "del-cli", } @@ -175,15 +202,17 @@ class StackedResolver(Resolver): return +NATIVE_RESOLVER_CLS = [ + CPANResolver, + PypiResolver, + NpmResolver, + GoResolver, + HackageResolver, + ] + + def native_resolvers(session): - return StackedResolver( - [ - CPANResolver(session), - PypiResolver(session), - NpmResolver(session), - HackageResolver(session), - ] - ) + return StackedResolver([kls(session) for kls in NATIVE_RESOLVER_CLS]) class ExplainResolver(Resolver): @@ -207,12 +236,5 @@ def auto_resolver(session): resolvers = [] if isinstance(session, SchrootSession) or user == "root": resolvers.append(AptResolver.from_session(session)) - resolvers.extend( - [ - CPANResolver(session), - PypiResolver(session), - NpmResolver(session), - HackageResolver(session), - ] - ) + resolvers.extend([kls(session) for kls in NATIVE_RESOLVER_CLS]) return StackedResolver(resolvers) From fb91d5ca608ede5363de5a9a21785612af030455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 15:14:04 +0000 Subject: [PATCH 19/21] More work supporting --explain. --- ognibuild/__main__.py | 72 ++++++++++++++++++++---------- ognibuild/buildlog.py | 34 +++++++++++++++ ognibuild/requirements.py | 2 + ognibuild/resolver/__init__.py | 80 ++++++++++++++++++++++++++++++---- ognibuild/resolver/apt.py | 11 ++++- 5 files changed, 167 insertions(+), 32 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 3a40ef1..a889962 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -17,16 +17,28 @@ import logging import os +import shlex import sys -from . import UnidentifiedError +from . import UnidentifiedError, DetailedFailure +from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, native_resolvers, + UnsatisfiedRequirements, ) from .resolver.apt import AptResolver +def display_explain_commands(commands): + logging.info("Run one or more of the following commands:") + for command, reqs in commands: + if isinstance(command, list): + command = shlex.join(command) + logging.info( + ' %s (to install %s)', command, ', '.join(map(str, reqs))) + + def get_necessary_declared_requirements(resolver, requirements, stages): missing = [] for stage, req in requirements: @@ -35,18 +47,19 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(session, resolver, buildsystem, stages): +def install_necessary_declared_requirements(session, resolver, buildsystems, stages, explain=False): relevant = [] - try: - declared_reqs = list(buildsystem.get_declared_dependencies()) - except NotImplementedError: - logging.warning( - "Unable to determine declared dependencies from %s", buildsystem - ) - else: - relevant.extend( - get_necessary_declared_requirements(resolver, declared_reqs, stages) - ) + declared_reqs = [] + for buildsystem in buildsystems: + try: + declared_reqs.extend(buildsystem.get_declared_dependencies()) + except NotImplementedError: + logging.warning( + "Unable to determine declared dependencies from %r", buildsystem + ) + relevant.extend( + get_necessary_declared_requirements(resolver, declared_reqs, stages) + ) missing = [] for req in relevant: try: @@ -55,7 +68,13 @@ def install_necessary_declared_requirements(session, resolver, buildsystem, stag except NotImplementedError: missing.append(req) if missing: - resolver.install(missing) + if explain: + commands = resolver.explain(missing) + if not commands: + raise UnsatisfiedRequirements(missing) + raise ExplainInstall(commands) + else: + resolver.install(missing) # Types of dependencies: @@ -74,9 +93,11 @@ STAGE_MAP = { } -def determine_fixers(session, resolver): - from .buildlog import InstallFixer - return [InstallFixer(resolver)] +def determine_fixers(session, resolver, explain=False): + if explain: + return [ExplainInstallFixer(resolver)] + else: + return [InstallFixer(resolver)] def main(): # noqa: C901 @@ -143,14 +164,19 @@ def main(): # noqa: C901 os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) - logging.info("Detected buildsystems: %r", bss) - if not args.ignore_declared_dependencies and not args.explain: + logging.info( + "Detected buildsystems: %s", ', '.join(map(str, bss))) + if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: logging.info("Checking that declared requirements are present") - for bs in bss: - install_necessary_declared_requirements(session, resolver, bs, stages) - fixers = determine_fixers(session, resolver) + try: + install_necessary_declared_requirements( + session, resolver, bss, stages, explain=args.explain) + except ExplainInstall as e: + display_explain_commands(e.commands) + return 1 + fixers = determine_fixers(session, resolver, explain=args.explain) if args.subcommand == "dist": from .dist import run_dist @@ -183,7 +209,9 @@ def main(): # noqa: C901 from .info import run_info run_info(session, buildsystems=bss) - except UnidentifiedError: + except ExplainInstall as e: + display_explain_commands(e.commands) + except (UnidentifiedError, DetailedFailure): return 1 except NoBuildToolsFound: logging.info("No build tools found.") diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 33d51ed..bb596b8 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -194,3 +194,37 @@ class InstallFixer(BuildFixer): except UnsatisfiedRequirements: return False return True + + +class ExplainInstall(Exception): + + def __init__(self, commands): + self.commands = commands + + +class ExplainInstallFixer(BuildFixer): + def __init__(self, resolver): + self.resolver = resolver + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.resolver) + + def __str__(self): + return "upstream requirement install explainer(%s)" % self.resolver + + def can_fix(self, error): + req = problem_to_upstream_requirement(error) + return req is not None + + def fix(self, error, context): + reqs = problem_to_upstream_requirement(error) + if reqs is None: + return False + + if not isinstance(reqs, list): + reqs = [reqs] + + explanations = list(self.resolver.explain(reqs)) + if not explanations: + return False + raise ExplainInstall(explanations) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 2f3673f..e2e5ff6 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -33,6 +33,8 @@ class PythonPackageRequirement(Requirement): self.python_version = python_version if minimum_version is not None: specs = [(">=", minimum_version)] + if specs is None: + specs = [] self.specs = specs def __repr__(self): diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 741144c..51e0467 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -16,6 +16,9 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import subprocess + + class UnsatisfiedRequirements(Exception): def __init__(self, reqs): self.requirements = reqs @@ -45,6 +48,17 @@ class CPANResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def explain(self, requirements): + from ..requirements import PerlModuleRequirement + + perlreqs = [] + for requirement in requirements: + if not isinstance(requirement, PerlModuleRequirement): + continue + perlreqs.append(requirement) + if perlreqs: + yield (["cpan", "-i"] + [req.module for req in perlreqs], [perlreqs]) + def install(self, requirements): from ..requirements import PerlModuleRequirement @@ -61,9 +75,6 @@ class CPANResolver(Resolver): if missing: raise UnsatisfiedRequirements(missing) - def explain(self, requirements): - raise NotImplementedError(self.explain) - class HackageResolver(Resolver): def __init__(self, session): @@ -90,7 +101,16 @@ class HackageResolver(Resolver): raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import HaskellPackageRequirement + + haskellreqs = [] + for requirement in requirements: + if not isinstance(requirement, HaskellPackageRequirement): + continue + haskellreqs.append(requirement) + if haskellreqs: + yield (["cabal", "install"] + [req.package for req in haskellreqs], + haskellreqs) class PypiResolver(Resolver): @@ -111,12 +131,25 @@ class PypiResolver(Resolver): if not isinstance(requirement, PythonPackageRequirement): missing.append(requirement) continue - self.session.check_call(["pip", "install", requirement.package]) + try: + self.session.check_call( + ["pip", "install", requirement.package]) + except subprocess.CalledProcessError: + missing.append(requirement) if missing: raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import PythonPackageRequirement + + pyreqs = [] + for requirement in requirements: + if not isinstance(requirement, PythonPackageRequirement): + continue + pyreqs.append(requirement) + if pyreqs: + yield (["pip", "install"] + [req.package for req in pyreqs], + pyreqs) class GoResolver(Resolver): @@ -143,7 +176,16 @@ class GoResolver(Resolver): raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import GoPackageRequirement + + goreqs = [] + for requirement in requirements: + if not isinstance(requirement, GoPackageRequirement): + continue + goreqs.append(requirement) + if goreqs: + yield (["go", "get"] + [req.package for req in goreqs], + goreqs) NPM_COMMAND_PACKAGES = { @@ -179,7 +221,21 @@ class NpmResolver(Resolver): raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import NodePackageRequirement + + nodereqs = [] + packages = [] + for requirement in requirements: + if not isinstance(requirement, NodePackageRequirement): + continue + try: + package = NPM_COMMAND_PACKAGES[requirement.command] + except KeyError: + continue + nodereqs.append(requirement) + packages.append(package) + if nodereqs: + yield (["npm", "-g", "install"] + packages, nodereqs) class StackedResolver(Resolver): @@ -192,6 +248,10 @@ class StackedResolver(Resolver): def __str__(self): return "[" + ", ".join(map(str, self.subs)) + "]" + def explain(self, requirements): + for sub in self.subs: + yield from sub.explain(requirements) + def install(self, requirements): for sub in self.subs: try: @@ -228,12 +288,14 @@ class ExplainResolver(Resolver): def auto_resolver(session): - # TODO(jelmer): if session is SchrootSession or if we're root, use apt + # if session is SchrootSession or if we're root, use apt from .apt import AptResolver from ..session.schroot import SchrootSession user = session.check_output(["echo", "$USER"]).decode().strip() resolvers = [] + # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if + # present? if isinstance(session, SchrootSession) or user == "root": resolvers.append(AptResolver.from_session(session)) resolvers.extend([kls(session) for kls in NATIVE_RESOLVER_CLS]) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index c6ccc12..08d95a4 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -76,6 +76,9 @@ class AptRequirement(Requirement): def pkg_relation_str(self): return PkgRelation.str(self.relations) + def __str__(self): + return "apt requirement: %s" % self.pkg_relation_str() + def touches_package(self, package): for rel in self.relations: for entry in rel: @@ -586,7 +589,13 @@ class AptResolver(Resolver): raise UnsatisfiedRequirements(still_missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + apt_requirements = [] + for r in requirements: + apt_req = self.resolve(r) + if apt_req is not None: + apt_requirements.append((r, apt_req)) + if apt_requirements: + yield (["apt", "satisfy"] + [PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))], [o for o, r in apt_requirements]) def resolve(self, req: Requirement): return resolve_requirement_apt(self.apt, req) From 6d8eab054759f9c6f4d822d3794e377407fd7de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 17:54:36 +0000 Subject: [PATCH 20/21] Release 0.0.1. From 9c74e770a6e5620f9cec1058688250928c1ca8f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 17:55:13 +0000 Subject: [PATCH 21/21] Release 0.0.2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8d358e..d85eb85 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup(name="ognibuild", description="Detect and run any build system", - version="0.0.1", + version="0.0.2", maintainer="Jelmer Vernooij", maintainer_email="jelmer@jelmer.uk", license="GNU GPLv2 or later",