From d89426738fa8e0ad85aa536d5be39a89d1aa4a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 8 Feb 2021 17:47:39 +0000 Subject: [PATCH] More work on resolvers. --- notes/structure.md | 30 +++++ ognibuild/__main__.py | 36 +++--- ognibuild/build.py | 4 +- ognibuild/buildsystem.py | 243 ++++++++++++++++++++------------------- ognibuild/clean.py | 4 +- ognibuild/dist.py | 4 +- ognibuild/install.py | 4 +- ognibuild/resolver.py | 81 +++++++++++++ ognibuild/test.py | 4 +- 9 files changed, 267 insertions(+), 143 deletions(-) create mode 100644 notes/structure.md create mode 100644 ognibuild/resolver.py diff --git a/notes/structure.md b/notes/structure.md new file mode 100644 index 0000000..960892c --- /dev/null +++ b/notes/structure.md @@ -0,0 +1,30 @@ +Upstream requirements are expressed as objects derived from UpstreamRequirement. + +They can either be: + + * extracted from the build system + * extracted from errors in build logs + +The details of UpstreamRequirements are specific to the kind of requirement, +and otherwise opaque to ognibuild. + +When building a package, we first make sure that all declared upstream +requirements are met. + +Then we attempt to build. + +If any problems are found in the log, buildlog-consultant will report them. + +ognibuild can then invoke "fixers" to address Problems. + +Problems can be converted to UpstreamRequirements by UpstreamRequirementFixer + +Other Fixer can do things like e.g. upgrade configure.ac to a newer version. + +UpstreamRequirementFixer uses a UpstreamRequirementResolver object that +can translate UpstreamRequirement objects into apt package names or +e.g. cpan commands. + +ognibuild keeps finding problems, resolving them and rebuilding until it finds +a problem it can not resolve or that it thinks it has already resolved +(i.e. seen before). diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 1c82b1e..b0a330b 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -38,10 +38,9 @@ def main(): ) parser.add_argument("--schroot", type=str, help="schroot to run in.") parser.add_argument( - "--resolve", - choices=["explain", "apt", "native"], - help="What to do about missing dependencies", - ) + '--resolve', choices=['explain', 'apt', 'native'], + default='apt', + help='What to do about missing dependencies') args = parser.parse_args() if args.schroot: from .session.schroot import SchrootSession @@ -52,18 +51,27 @@ def main(): session = PlainSession() with session: + if args.resolve == 'apt': + from .resolver import AptResolver + resolver = AptResolver.from_session(session) + elif args.resolve == 'explain': + from .resolver import ExplainResolver + resolver = ExplainResolver.from_session(session) + elif args.resolve == 'native': + from .resolver import NativeResolver + resolver = NativeResolver.from_session(session) os.chdir(args.directory) try: - if args.subcommand == "dist": - run_dist(session) - if args.subcommand == "build": - run_build(session) - if args.subcommand == "clean": - run_clean(session) - if args.subcommand == "install": - run_install(session) - if args.subcommand == "test": - run_test(session) + if args.subcommand == 'dist': + run_dist(session=session, resolver=resolver) + if args.subcommand == 'build': + run_build(session, resolver=resolver) + if args.subcommand == 'clean': + run_clean(session, resolver=resolver) + if args.subcommand == 'install': + run_install(session, resolver=resolver) + if args.subcommand == 'test': + run_test(session, resolver=resolver) except NoBuildToolsFound: logging.info("No build tools found.") return 1 diff --git a/ognibuild/build.py b/ognibuild/build.py index b582b11..ea3fe03 100644 --- a/ognibuild/build.py +++ b/ognibuild/build.py @@ -18,13 +18,13 @@ from .buildsystem import detect_buildsystems, NoBuildToolsFound -def run_build(session): +def run_build(session, resolver): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in detect_buildsystems(session): - buildsystem.build() + buildsystem.build(resolver) return raise NoBuildToolsFound() diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 998803e..77fc2dd 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,8 +20,8 @@ import logging import re -from . import shebang_binary -from .apt import AptManager, UnidentifiedError +from . import shebang_binary, UpstreamPackage +from .apt import UnidentifiedError from .fix_build import run_with_build_fixer @@ -35,105 +35,106 @@ class BuildSystem(object): def __init__(self, session): self.session = session - def dist(self): + def dist(self, resolver): raise NotImplementedError(self.dist) - def test(self): + def test(self, resolver): raise NotImplementedError(self.test) - def build(self): + def build(self, resolver): raise NotImplementedError(self.build) - def clean(self): + def clean(self, resolver): raise NotImplementedError(self.clean) - def install(self): + def install(self, resolver): raise NotImplementedError(self.install) class Pear(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["php-pear"]) - def dist(self): - self.setup() - run_with_build_fixer(self.session, ["pear", "package"]) + def setup(self, resolver): + resolver.install([UpstreamPackage('binary', 'pear')]) - def test(self): + def dist(self, resolver): + self.setup(resolver) + run_with_build_fixer(self.session, ['pear', 'package']) + + def test(self, resolver): self.setup() run_with_build_fixer(self.session, ["pear", "run-tests"]) - def build(self): - self.setup() - run_with_build_fixer(self.session, ["pear", "build"]) + def build(self, resolver): + self.setup(resolver) + run_with_build_fixer(self.session, ['pear', 'build']) - def clean(self): - self.setup() + def clean(self, resolver): + self.setup(resolver) # TODO - def install(self): - self.setup() - run_with_build_fixer(self.session, ["pear", "install"]) + def install(self, resolver): + self.setup(resolver) + run_with_build_fixer(self.session, ['pear', 'install']) class SetupPy(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["python3", "python3-pip"]) - with open("setup.py", "r") as f: + + def setup(self, resolver): + resolver.install([ + UpstreamPackage('python3', 'pip'), + UpstreamPackage('binary', 'python3'), + ]) + with open('setup.py', 'r') as f: setup_py_contents = f.read() try: with open("setup.cfg", "r") as f: setup_cfg_contents = f.read() except FileNotFoundError: - setup_cfg_contents = "" - if "setuptools" in setup_py_contents: - logging.info("Reference to setuptools found, installing.") - apt.install(["python3-setuptools"]) - if ( - "setuptools_scm" in setup_py_contents - or "setuptools_scm" in setup_cfg_contents - ): - logging.info("Reference to setuptools-scm found, installing.") - apt.install(["python3-setuptools-scm", "git", "mercurial"]) + setup_cfg_contents = '' + if 'setuptools' in setup_py_contents: + logging.info('Reference to setuptools found, installing.') + resolver.install([UpstreamPackage('python3', 'setuptools')]) + if ('setuptools_scm' in setup_py_contents or + 'setuptools_scm' in setup_cfg_contents): + logging.info('Reference to setuptools-scm found, installing.') + resolver.install([ + UpstreamPackage('python3', 'setuptools-scm'), + UpstreamPackage('binary', 'git'), + UpstreamPackage('binary', 'mercurial'), + ]) # TODO(jelmer): Install setup_requires - def test(self): - self.setup() - self._run_setup(["test"]) + def test(self, resolver): + self.setup(resolver) + self._run_setup(resolver, ['test']) - def dist(self): - self.setup() - self._run_setup(["sdist"]) + def dist(self, resolver): + self.setup(resolver) + self._run_setup(resolver, ['sdist']) - def clean(self): - self.setup() - self._run_setup(["clean"]) + def clean(self, resolver): + self.setup(resolver) + self._run_setup(resolver, ['clean']) - def install(self): - self.setup() - self._run_setup(["install"]) + def install(self, resolver): + self.setup(resolver) + self._run_setup(resolver, ['install']) - def _run_setup(self, args): - apt = AptManager(self.session) - interpreter = shebang_binary("setup.py") + def _run_setup(self, resolver, args): + interpreter = shebang_binary('setup.py') if interpreter is not None: - if interpreter == "python3": - apt.install(["python3"]) - elif interpreter == "python2": - apt.install(["python2"]) - elif interpreter == "python": - apt.install(["python"]) + if interpreter in ('python3', 'python2', 'python'): + resolver.install([UpstreamPackage('binary', interpreter)]) else: - raise ValueError("Unknown interpreter %r" % interpreter) - apt.install(["python2", "python3"]) - run_with_build_fixer(self.session, ["./setup.py"] + args) + raise ValueError('Unknown interpreter %r' % interpreter) + run_with_build_fixer( + self.session, ['./setup.py'] + args) else: # Just assume it's Python 3 - apt.install(["python3"]) - run_with_build_fixer(self.session, ["python3", "./setup.py"] + args) + resolver.install([UpstreamPackage('binary', 'python3')]) + run_with_build_fixer( + self.session, ['python3', './setup.py'] + args) class PyProject(BuildSystem): @@ -143,75 +144,79 @@ class PyProject(BuildSystem): with open("pyproject.toml", "r") as pf: return toml.load(pf) - def dist(self): - apt = AptManager(self.session) + def dist(self, resolver): pyproject = self.load_toml() if "poetry" in pyproject.get("tool", []): logging.info( - "Found pyproject.toml with poetry section, " "assuming poetry project." - ) - apt.install(["python3-venv", "python3-pip"]) - self.session.check_call(["pip3", "install", "poetry"], user="root") - self.session.check_call(["poetry", "build", "-f", "sdist"]) + 'Found pyproject.toml with poetry section, ' + 'assuming poetry project.') + resolver.install([ + UpstreamPackage('python3', 'venv'), + UpstreamPackage('python3', 'pip'), + ]) + self.session.check_call(['pip3', 'install', 'poetry'], user='root') + self.session.check_call(['poetry', 'build', '-f', 'sdist']) return raise AssertionError("no supported section in pyproject.toml") class SetupCfg(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["python3-pep517", "python3-pip"]) - def dist(self): - self.session.check_call(["python3", "-m", "pep517.build", "-s", "."]) + def setup(self, resolver): + resolver.install([ + UpstreamPackage('python3', 'pep517'), + UpstreamPackage('python3', 'pip'), + ]) + + def dist(self, resolver): + self.setup(resolver) + self.session.check_call(['python3', '-m', 'pep517.build', '-s', '.']) class NpmPackage(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["npm"]) - def dist(self): - self.setup() - run_with_build_fixer(self.session, ["npm", "pack"]) + def setup(self, resolver): + resolver.install([UpstreamPackage('binary', 'npm')]) + + def dist(self, resolver): + self.setup(resolver) + run_with_build_fixer(self.session, ['npm', 'pack']) class Waf(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["python3"]) - def dist(self): - self.setup() - run_with_build_fixer(self.session, ["./waf", "dist"]) + def setup(self, resolver): + resolver.install([UpstreamPackage('binary', 'python3')]) + + def dist(self, resolver): + self.setup(resolver) + run_with_build_fixer(self.session, ['./waf', 'dist']) class Gem(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["gem2deb"]) - def dist(self): - self.setup() - gemfiles = [ - entry.name - for entry in self.session.scandir(".") - if entry.name.endswith(".gem") - ] + def setup(self, resolver): + resolver.install([UpstreamPackage('binary', 'gem2deb')]) + + def dist(self, resolver): + self.setup(resolver) + gemfiles = [entry.name for entry in self.session.scandir('.') + if entry.name.endswith('.gem')] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") run_with_build_fixer(self.session, ["gem2tgz", gemfiles[0]]) class DistInkt(BuildSystem): - def setup(self): - apt = AptManager(self.session) - apt.install(["libdist-inkt-perl"]) - def dist(self): - self.setup() - apt = AptManager(self.session) - with open("dist.ini", "rb") as f: + def setup(self, resolver): + resolver.install([ + UpstreamPackage('perl', 'Dist::Inkt'), + ]) + + def dist(self, resolver): + self.setup(resolver) + with open('dist.ini', 'rb') as f: for line in f: if not line.startswith(b";;"): continue @@ -230,22 +235,23 @@ class DistInkt(BuildSystem): run_with_build_fixer(self.session, ["distinkt-dist"]) return # Default to invoking Dist::Zilla - logging.info("Found dist.ini, assuming dist-zilla.") - apt.install(["libdist-zilla-perl"]) - run_with_build_fixer(self.session, ["dzil", "build", "--in", ".."]) + logging.info('Found dist.ini, assuming dist-zilla.') + resolver.install([UpstreamPackage('perl', 'Dist::Zilla')]) + run_with_build_fixer(self.session, ['dzil', 'build', '--in', '..']) class Make(BuildSystem): - def setup(self): - apt = AptManager(self.session) - if self.session.exists("Makefile.PL") and not self.session.exists("Makefile"): - apt.install(["perl"]) - run_with_build_fixer(self.session, ["perl", "Makefile.PL"]) - if not self.session.exists("Makefile") and not self.session.exists("configure"): - if self.session.exists("autogen.sh"): - if shebang_binary("autogen.sh") is None: - run_with_build_fixer(self.session, ["/bin/sh", "./autogen.sh"]) + def setup(self, resolver): + if self.session.exists('Makefile.PL') and not self.session.exists('Makefile'): + resolver.install([UpstreamPackage('binary', 'perl')]) + run_with_build_fixer(self.session, ['perl', 'Makefile.PL']) + + if not self.session.exists('Makefile') and not self.session.exists('configure'): + if self.session.exists('autogen.sh'): + if shebang_binary('autogen.sh') is None: + run_with_build_fixer( + self.session, ['/bin/sh', './autogen.sh']) try: run_with_build_fixer(self.session, ["./autogen.sh"]) except UnidentifiedError as e: @@ -269,10 +275,9 @@ class Make(BuildSystem): if not self.session.exists("Makefile") and self.session.exists("configure"): self.session.check_call(["./configure"]) - def dist(self): - self.setup() - apt = AptManager(self.session) - apt.install(["make"]) + def dist(self, resolver): + self.setup(resolver) + resolver.install([UpstreamPackage('binary', 'make')]) try: run_with_build_fixer(self.session, ["make", "dist"]) except UnidentifiedError as e: diff --git a/ognibuild/clean.py b/ognibuild/clean.py index 67cf27a..cabf76f 100644 --- a/ognibuild/clean.py +++ b/ognibuild/clean.py @@ -18,13 +18,13 @@ from .buildsystem import detect_buildsystems, NoBuildToolsFound -def run_clean(session): +def run_clean(session, resolver): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in detect_buildsystems(session): - buildsystem.clean() + buildsystem.clean(resolver) return raise NoBuildToolsFound() diff --git a/ognibuild/dist.py b/ognibuild/dist.py index e52dc25..1fa77cd 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -62,13 +62,13 @@ class DistNoTarball(Exception): """Dist operation did not create a tarball.""" -def run_dist(session): +def run_dist(session, resolver): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in detect_buildsystems(session): - buildsystem.dist() + buildsystem.dist(resolver) return raise NoBuildToolsFound() diff --git a/ognibuild/install.py b/ognibuild/install.py index b2c3922..5d386c0 100644 --- a/ognibuild/install.py +++ b/ognibuild/install.py @@ -18,13 +18,13 @@ from .buildsystem import detect_buildsystems, NoBuildToolsFound -def run_install(session): +def run_install(session, resolver): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in detect_buildsystems(session): - buildsystem.install() + buildsystem.install(resolver) return raise NoBuildToolsFound() diff --git a/ognibuild/resolver.py b/ognibuild/resolver.py new file mode 100644 index 0000000..288e9d7 --- /dev/null +++ b/ognibuild/resolver.py @@ -0,0 +1,81 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +class Resolver(object): + + def install(self, requirements): + raise NotImplementedError(self.install) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + +class AptResolver(Resolver): + + def __init__(self, apt): + self.apt = apt + + @classmethod + def from_session(cls, session): + from .apt import AptManager + return cls(AptManager(session)) + + def install(self, requirements): + self.apt.install(list(self.resolve(requirements))) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + def resolve(self, requirements): + for req in requirements: + if req.family == 'python3': + yield 'python3-%s' % req.name + else: + yield self.apt.find_file('/usr/bin/%s' % req.name) + + +class NativeResolver(Resolver): + + def __init__(self, session): + self.session = session + + @classmethod + def from_session(cls, session): + return cls(session) + + def install(self, requirements): + raise NotImplementedError(self.install) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + +class ExplainResolver(Resolver): + + def __init__(self, session): + self.session = session + + @classmethod + def from_session(cls, session): + return cls(session) + + def install(self, requirements): + raise NotImplementedError(self.install) + + def explain(self, requirements): + raise NotImplementedError(self.explain) diff --git a/ognibuild/test.py b/ognibuild/test.py index eb60e40..8f7ca08 100644 --- a/ognibuild/test.py +++ b/ognibuild/test.py @@ -18,13 +18,13 @@ from .buildsystem import detect_buildsystems, NoBuildToolsFound -def run_test(session): +def run_test(session, resolver): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in detect_buildsystems(session): - buildsystem.test() + buildsystem.test(resolver) return raise NoBuildToolsFound()