From fb91d5ca608ede5363de5a9a21785612af030455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 15:14:04 +0000 Subject: [PATCH] More work supporting --explain. --- ognibuild/__main__.py | 72 ++++++++++++++++++++---------- ognibuild/buildlog.py | 34 +++++++++++++++ ognibuild/requirements.py | 2 + ognibuild/resolver/__init__.py | 80 ++++++++++++++++++++++++++++++---- ognibuild/resolver/apt.py | 11 ++++- 5 files changed, 167 insertions(+), 32 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 3a40ef1..a889962 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -17,16 +17,28 @@ import logging import os +import shlex import sys -from . import UnidentifiedError +from . import UnidentifiedError, DetailedFailure +from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, native_resolvers, + UnsatisfiedRequirements, ) from .resolver.apt import AptResolver +def display_explain_commands(commands): + logging.info("Run one or more of the following commands:") + for command, reqs in commands: + if isinstance(command, list): + command = shlex.join(command) + logging.info( + ' %s (to install %s)', command, ', '.join(map(str, reqs))) + + def get_necessary_declared_requirements(resolver, requirements, stages): missing = [] for stage, req in requirements: @@ -35,18 +47,19 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(session, resolver, buildsystem, stages): +def install_necessary_declared_requirements(session, resolver, buildsystems, stages, explain=False): relevant = [] - try: - declared_reqs = list(buildsystem.get_declared_dependencies()) - except NotImplementedError: - logging.warning( - "Unable to determine declared dependencies from %s", buildsystem - ) - else: - relevant.extend( - get_necessary_declared_requirements(resolver, declared_reqs, stages) - ) + declared_reqs = [] + for buildsystem in buildsystems: + try: + declared_reqs.extend(buildsystem.get_declared_dependencies()) + except NotImplementedError: + logging.warning( + "Unable to determine declared dependencies from %r", buildsystem + ) + relevant.extend( + get_necessary_declared_requirements(resolver, declared_reqs, stages) + ) missing = [] for req in relevant: try: @@ -55,7 +68,13 @@ def install_necessary_declared_requirements(session, resolver, buildsystem, stag except NotImplementedError: missing.append(req) if missing: - resolver.install(missing) + if explain: + commands = resolver.explain(missing) + if not commands: + raise UnsatisfiedRequirements(missing) + raise ExplainInstall(commands) + else: + resolver.install(missing) # Types of dependencies: @@ -74,9 +93,11 @@ STAGE_MAP = { } -def determine_fixers(session, resolver): - from .buildlog import InstallFixer - return [InstallFixer(resolver)] +def determine_fixers(session, resolver, explain=False): + if explain: + return [ExplainInstallFixer(resolver)] + else: + return [InstallFixer(resolver)] def main(): # noqa: C901 @@ -143,14 +164,19 @@ def main(): # noqa: C901 os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) - logging.info("Detected buildsystems: %r", bss) - if not args.ignore_declared_dependencies and not args.explain: + logging.info( + "Detected buildsystems: %s", ', '.join(map(str, bss))) + if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: logging.info("Checking that declared requirements are present") - for bs in bss: - install_necessary_declared_requirements(session, resolver, bs, stages) - fixers = determine_fixers(session, resolver) + try: + install_necessary_declared_requirements( + session, resolver, bss, stages, explain=args.explain) + except ExplainInstall as e: + display_explain_commands(e.commands) + return 1 + fixers = determine_fixers(session, resolver, explain=args.explain) if args.subcommand == "dist": from .dist import run_dist @@ -183,7 +209,9 @@ def main(): # noqa: C901 from .info import run_info run_info(session, buildsystems=bss) - except UnidentifiedError: + except ExplainInstall as e: + display_explain_commands(e.commands) + except (UnidentifiedError, DetailedFailure): return 1 except NoBuildToolsFound: logging.info("No build tools found.") diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 33d51ed..bb596b8 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -194,3 +194,37 @@ class InstallFixer(BuildFixer): except UnsatisfiedRequirements: return False return True + + +class ExplainInstall(Exception): + + def __init__(self, commands): + self.commands = commands + + +class ExplainInstallFixer(BuildFixer): + def __init__(self, resolver): + self.resolver = resolver + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.resolver) + + def __str__(self): + return "upstream requirement install explainer(%s)" % self.resolver + + def can_fix(self, error): + req = problem_to_upstream_requirement(error) + return req is not None + + def fix(self, error, context): + reqs = problem_to_upstream_requirement(error) + if reqs is None: + return False + + if not isinstance(reqs, list): + reqs = [reqs] + + explanations = list(self.resolver.explain(reqs)) + if not explanations: + return False + raise ExplainInstall(explanations) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 2f3673f..e2e5ff6 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -33,6 +33,8 @@ class PythonPackageRequirement(Requirement): self.python_version = python_version if minimum_version is not None: specs = [(">=", minimum_version)] + if specs is None: + specs = [] self.specs = specs def __repr__(self): diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 741144c..51e0467 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -16,6 +16,9 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import subprocess + + class UnsatisfiedRequirements(Exception): def __init__(self, reqs): self.requirements = reqs @@ -45,6 +48,17 @@ class CPANResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def explain(self, requirements): + from ..requirements import PerlModuleRequirement + + perlreqs = [] + for requirement in requirements: + if not isinstance(requirement, PerlModuleRequirement): + continue + perlreqs.append(requirement) + if perlreqs: + yield (["cpan", "-i"] + [req.module for req in perlreqs], [perlreqs]) + def install(self, requirements): from ..requirements import PerlModuleRequirement @@ -61,9 +75,6 @@ class CPANResolver(Resolver): if missing: raise UnsatisfiedRequirements(missing) - def explain(self, requirements): - raise NotImplementedError(self.explain) - class HackageResolver(Resolver): def __init__(self, session): @@ -90,7 +101,16 @@ class HackageResolver(Resolver): raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import HaskellPackageRequirement + + haskellreqs = [] + for requirement in requirements: + if not isinstance(requirement, HaskellPackageRequirement): + continue + haskellreqs.append(requirement) + if haskellreqs: + yield (["cabal", "install"] + [req.package for req in haskellreqs], + haskellreqs) class PypiResolver(Resolver): @@ -111,12 +131,25 @@ class PypiResolver(Resolver): if not isinstance(requirement, PythonPackageRequirement): missing.append(requirement) continue - self.session.check_call(["pip", "install", requirement.package]) + try: + self.session.check_call( + ["pip", "install", requirement.package]) + except subprocess.CalledProcessError: + missing.append(requirement) if missing: raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import PythonPackageRequirement + + pyreqs = [] + for requirement in requirements: + if not isinstance(requirement, PythonPackageRequirement): + continue + pyreqs.append(requirement) + if pyreqs: + yield (["pip", "install"] + [req.package for req in pyreqs], + pyreqs) class GoResolver(Resolver): @@ -143,7 +176,16 @@ class GoResolver(Resolver): raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import GoPackageRequirement + + goreqs = [] + for requirement in requirements: + if not isinstance(requirement, GoPackageRequirement): + continue + goreqs.append(requirement) + if goreqs: + yield (["go", "get"] + [req.package for req in goreqs], + goreqs) NPM_COMMAND_PACKAGES = { @@ -179,7 +221,21 @@ class NpmResolver(Resolver): raise UnsatisfiedRequirements(missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + from ..requirements import NodePackageRequirement + + nodereqs = [] + packages = [] + for requirement in requirements: + if not isinstance(requirement, NodePackageRequirement): + continue + try: + package = NPM_COMMAND_PACKAGES[requirement.command] + except KeyError: + continue + nodereqs.append(requirement) + packages.append(package) + if nodereqs: + yield (["npm", "-g", "install"] + packages, nodereqs) class StackedResolver(Resolver): @@ -192,6 +248,10 @@ class StackedResolver(Resolver): def __str__(self): return "[" + ", ".join(map(str, self.subs)) + "]" + def explain(self, requirements): + for sub in self.subs: + yield from sub.explain(requirements) + def install(self, requirements): for sub in self.subs: try: @@ -228,12 +288,14 @@ class ExplainResolver(Resolver): def auto_resolver(session): - # TODO(jelmer): if session is SchrootSession or if we're root, use apt + # if session is SchrootSession or if we're root, use apt from .apt import AptResolver from ..session.schroot import SchrootSession user = session.check_output(["echo", "$USER"]).decode().strip() resolvers = [] + # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if + # present? if isinstance(session, SchrootSession) or user == "root": resolvers.append(AptResolver.from_session(session)) resolvers.extend([kls(session) for kls in NATIVE_RESOLVER_CLS]) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index c6ccc12..08d95a4 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -76,6 +76,9 @@ class AptRequirement(Requirement): def pkg_relation_str(self): return PkgRelation.str(self.relations) + def __str__(self): + return "apt requirement: %s" % self.pkg_relation_str() + def touches_package(self, package): for rel in self.relations: for entry in rel: @@ -586,7 +589,13 @@ class AptResolver(Resolver): raise UnsatisfiedRequirements(still_missing) def explain(self, requirements): - raise NotImplementedError(self.explain) + apt_requirements = [] + for r in requirements: + apt_req = self.resolve(r) + if apt_req is not None: + apt_requirements.append((r, apt_req)) + if apt_requirements: + yield (["apt", "satisfy"] + [PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))], [o for o, r in apt_requirements]) def resolve(self, req: Requirement): return resolve_requirement_apt(self.apt, req)