From 6b30479b97defc956a584394cff44e99a2f5436e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Feb 2021 03:19:33 +0000 Subject: [PATCH] More fixes. --- ognibuild/__main__.py | 11 +- ognibuild/buildlog.py | 192 +++++++++++++++++++ ognibuild/buildsystem.py | 8 +- ognibuild/debian/apt.py | 48 +++-- ognibuild/debian/fix_build.py | 227 ++++------------------- ognibuild/fix_build.py | 61 ++---- ognibuild/resolver/__init__.py | 101 +++++++++- ognibuild/resolver/apt.py | 163 ++++++++++++---- ognibuild/tests/test_debian_fix_build.py | 8 +- 9 files changed, 499 insertions(+), 320 deletions(-) create mode 100644 ognibuild/buildlog.py diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index b6dbde2..d1ffb02 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -23,7 +23,7 @@ from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( ExplainResolver, AutoResolver, - NativeResolver, + native_resolvers, MissingDependencies, ) from .resolver.apt import AptResolver @@ -91,6 +91,9 @@ def main(): # noqa: C901 '--user', action='store_true', help='Install in local-user directories.') args = parser.parse_args() + if not args.subcommand: + parser.print_usage() + return 1 if args.verbose: logging.basicConfig(level=logging.DEBUG) else: @@ -109,7 +112,7 @@ def main(): # noqa: C901 elif args.resolve == "explain": resolver = ExplainResolver.from_session(session) elif args.resolve == "native": - resolver = NativeResolver.from_session(session) + resolver = native_resolvers(session) elif args.resolver == "auto": resolver = AutoResolver.from_session(session) os.chdir(args.directory) @@ -149,10 +152,10 @@ def main(): # noqa: C901 except MissingDependencies as e: for req in e.requirements: logging.info("Missing dependency (%s:%s)", - req.family, req.name) + req.family, req.package) for resolver in [ AptResolver.from_session(session), - NativeResolver.from_session(session), + native_resolvers(session), ]: logging.info(" %s", resolver.explain([req])) return 2 diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py new file mode 100644 index 0000000..0ff19a8 --- /dev/null +++ b/ognibuild/buildlog.py @@ -0,0 +1,192 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +"""Convert problems found in the buildlog to upstream requirements. +""" + +import logging + +from buildlog_consultant.common import ( + MissingConfigStatusInput, + MissingPythonModule, + MissingPythonDistribution, + MissingCHeader, + MissingPkgConfig, + MissingCommand, + MissingFile, + MissingJavaScriptRuntime, + MissingSprocketsFile, + MissingGoPackage, + MissingPerlFile, + MissingPerlModule, + MissingXmlEntity, + MissingJDKFile, + MissingNodeModule, + MissingPhpClass, + MissingRubyGem, + MissingLibrary, + MissingJavaClass, + MissingCSharpCompiler, + MissingConfigure, + MissingAutomakeInput, + MissingRPackage, + MissingRubyFile, + MissingAutoconfMacro, + MissingValaPackage, + MissingXfceDependency, + MissingHaskellDependencies, + NeedPgBuildExtUpdateControl, + DhAddonLoadFailure, + MissingMavenArtifacts, + GnomeCommonMissing, + MissingGnomeCommonDependency, +) + +from .fix_build import BuildFixer +from .requirements import ( + BinaryRequirement, + PathRequirement, + PkgConfigRequirement, + CHeaderRequirement, + JavaScriptRuntimeRequirement, + ValaPackageRequirement, + RubyGemRequirement, + GoPackageRequirement, + DhAddonRequirement, + PhpClassRequirement, + RPackageRequirement, + NodePackageRequirement, + LibraryRequirement, + RubyFileRequirement, + XmlEntityRequirement, + SprocketsFileRequirement, + JavaClassRequirement, + HaskellPackageRequirement, + MavenArtifactRequirement, + GnomeCommonRequirement, + JDKFileRequirement, + PerlModuleRequirement, + PerlFileRequirement, + AutoconfMacroRequirement, + PythonModuleRequirement, + PythonPackageRequirement, + ) + + +def problem_to_upstream_requirement(problem): + if isinstance(problem, MissingFile): + return PathRequirement(problem.path) + elif isinstance(problem, MissingCommand): + return BinaryRequirement(problem.command) + elif isinstance(problem, MissingPkgConfig): + return PkgConfigRequirement( + problem.module, problem.minimum_version) + elif isinstance(problem, MissingCHeader): + return CHeaderRequirement(problem.header) + elif isinstance(problem, MissingJavaScriptRuntime): + return JavaScriptRuntimeRequirement() + elif isinstance(problem, MissingRubyGem): + return RubyGemRequirement(problem.gem, problem.version) + elif isinstance(problem, MissingValaPackage): + return ValaPackageRequirement(problem.package) + elif isinstance(problem, MissingGoPackage): + return GoPackageRequirement(problem.package) + elif isinstance(problem, DhAddonLoadFailure): + return DhAddonRequirement(problem.path) + elif isinstance(problem, MissingPhpClass): + return PhpClassRequirement(problem.php_class) + elif isinstance(problem, MissingRPackage): + return RPackageRequirement(problem.package, problem.minimum_version) + elif isinstance(problem, MissingNodeModule): + return NodePackageRequirement(problem.module) + elif isinstance(problem, MissingLibrary): + return LibraryRequirement(problem.library) + elif isinstance(problem, MissingRubyFile): + return RubyFileRequirement(problem.filename) + elif isinstance(problem, MissingXmlEntity): + return XmlEntityRequirement(problem.url) + elif isinstance(problem, MissingSprocketsFile): + return SprocketsFileRequirement(problem.content_type, problem.name) + elif isinstance(problem, MissingJavaClass): + return JavaClassRequirement(problem.classname) + elif isinstance(problem, MissingHaskellDependencies): + # TODO(jelmer): Create multiple HaskellPackageRequirement objects? + return HaskellPackageRequirement(problem.package) + elif isinstance(problem, MissingMavenArtifacts): + # TODO(jelmer): Create multiple MavenArtifactRequirement objects? + return MavenArtifactRequirement(problem.artifacts) + elif isinstance(problem, MissingCSharpCompiler): + return BinaryRequirement('msc') + elif isinstance(problem, GnomeCommonMissing): + return GnomeCommonRequirement() + elif isinstance(problem, MissingJDKFile): + return JDKFileRequirement(problem.jdk_path, problem.filename) + elif isinstance(problem, MissingGnomeCommonDependency): + if problem.package == "glib-gettext": + return BinaryRequirement('glib-gettextize') + else: + logging.warning( + "No known command for gnome-common dependency %s", + problem.package) + return None + elif isinstance(problem, MissingXfceDependency): + if problem.package == "gtk-doc": + return BinaryRequirement("gtkdocize") + else: + logging.warning( + "No known command for xfce dependency %s", + problem.package) + return None + elif isinstance(problem, MissingPerlModule): + return PerlModuleRequirement( + module=problem.module, + filename=problem.filename, + inc=problem.inc) + elif isinstance(problem, MissingPerlFile): + return PerlFileRequirement(filename=problem.filename) + elif isinstance(problem, MissingAutoconfMacro): + return AutoconfMacroRequirement(problem.macro) + elif isinstance(problem, MissingPythonModule): + return PythonModuleRequirement( + problem.module, + python_version=problem.python_version, + minimum_version=problem.minimum_version) + elif isinstance(problem, MissingPythonDistribution): + return PythonPackageRequirement( + problem.module, + python_version=problem.python_version, + minimum_version=problem.minimum_version) + else: + return None + + +class UpstreamRequirementFixer(BuildFixer): + + def __init__(self, resolver): + self.resolver = resolver + + def can_fix(self, error): + req = problem_to_upstream_requirement(error) + return req is not None + + def fix(self, error, context): + req = problem_to_upstream_requirement(error) + if req is None: + return False + + package = self.resolver.resolve(req) + return context.add_dependency(package) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 8f9f6ba..cc8b30e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -182,17 +182,19 @@ class SetupPy(BuildSystem): def get_declared_dependencies(self): for require in self.result.get_requires(): yield "build", PythonPackageRequirement(require) - if self.result.install_requires: + # Not present for distutils-only packages + if getattr(self.result, 'install_requires', []): for require in self.result.install_requires: yield "install", PythonPackageRequirement(require) - if self.result.tests_require: + # Not present for distutils-only packages + if getattr(self.result, 'tests_require', []): for require in self.result.tests_require: yield "test", PythonPackageRequirement(require) def get_declared_outputs(self): for script in self.result.scripts or []: yield UpstreamOutput("binary", os.path.basename(script)) - entry_points = self.result.entry_points or {} + entry_points = getattr(self.result, 'entry_points', None) or {} for script in entry_points.get("console_scripts", []): yield UpstreamOutput("binary", script.split("=")[0]) for package in self.result.packages or []: diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index ab2ff16..c4e2938 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -70,14 +70,9 @@ class AptManager(object): def package_exists(self, package): if self._apt_cache is None: - import apt_pkg - - # TODO(jelmer): Load from self.session - self._apt_cache = apt_pkg.Cache() - for p in self._apt_cache.packages: - if p.name == package: - return True - return False + import apt + self._apt_cache = apt.Cache(rootdir=self.session.location) + return package in self._apt_cache def get_package_for_paths(self, paths, regex=False): logging.debug('Searching for packages containing %r', paths) @@ -121,10 +116,12 @@ class RemoteAptContentsFileSearcher(FileSearcher): def from_session(cls, session): logging.info('Loading apt contents information') # TODO(jelmer): what about sources.list.d? - with open(os.path.join(session.location, 'etc/apt/sources.list'), 'r') as f: - return cls.from_repositories( - f.readlines(), - cache_dir=os.path.join(session.location, 'var/lib/apt/lists')) + from aptsources.sourceslist import SourcesList + sl = SourcesList() + sl.load(os.path.join(session.location, 'etc/apt/sources.list')) + return cls.from_sources_list( + sl, + cache_dir=os.path.join(session.location, 'var/lib/apt/lists')) def __setitem__(self, path, package): self._db[path] = package @@ -174,32 +171,31 @@ class RemoteAptContentsFileSearcher(FileSearcher): self.load_url(url) except ContentsFileNotFound: if mandatory: - raise - logging.debug( - 'Unable to fetch optional contents file %s', url) + logging.warning( + 'Unable to fetch contents file %s', url) + else: + logging.debug( + 'Unable to fetch optional contents file %s', url) return self @classmethod - def from_repositories(cls, sources, cache_dir=None): + def from_sources_list(cls, sl, cache_dir=None): # TODO(jelmer): Use aptsources.sourceslist.SourcesList from .build import get_build_architecture # TODO(jelmer): Verify signatures, etc. urls = [] arches = [(get_build_architecture(), True), ("all", False)] - for source in sources: - if not source.strip(): + for source in sl.list: + if source.invalid or source.disabled: continue - if source.strip().startswith('#'): + if source.type == 'deb-src': continue - parts = source.split(" ") - if parts[0] == "deb-src": - continue - if parts[0] != "deb": + if source.type != 'deb': logging.warning("Invalid line in sources: %r", source) continue - base_url = parts[1].strip().rstrip("/") - name = parts[2].strip() - components = [c.strip() for c in parts[3:]] + base_url = source.uri.rstrip('/') + name = source.dist.rstrip('/') + components = source.comps if components: dists_url = base_url + "/dists" else: diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 36f6139..a32219b 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -63,81 +63,28 @@ from debmutate._rules import ( from breezy.plugins.debian.changelog import debcommit from buildlog_consultant import Problem -from buildlog_consultant.common import ( - MissingConfigStatusInput, - MissingPythonModule, - MissingPythonDistribution, - MissingCHeader, - MissingPkgConfig, - MissingCommand, - MissingFile, - MissingJavaScriptRuntime, - MissingSprocketsFile, - MissingGoPackage, - MissingPerlFile, - MissingPerlModule, - MissingXmlEntity, - MissingJDKFile, - MissingNodeModule, - MissingPhpClass, - MissingRubyGem, - MissingLibrary, - MissingJavaClass, - MissingCSharpCompiler, - MissingConfigure, - MissingAutomakeInput, - MissingRPackage, - MissingRubyFile, - MissingAutoconfMacro, - MissingValaPackage, - MissingXfceDependency, - MissingHaskellDependencies, - NeedPgBuildExtUpdateControl, - DhAddonLoadFailure, - MissingMavenArtifacts, - GnomeCommonMissing, - MissingGnomeCommonDependency, -) from buildlog_consultant.apt import ( AptFetchFailure, ) +from buildlog_consultant.common import ( + MissingConfigStatusInput, + MissingAutomakeInput, + MissingConfigure, + NeedPgBuildExtUpdateControl, + MissingPythonModule, + MissingPythonDistribution, + MissingPerlFile, + ) from buildlog_consultant.sbuild import ( SbuildFailure, ) -from ..fix_build import BuildFixer, SimpleBuildFixer, resolve_error, DependencyContext +from ..fix_build import BuildFixer, resolve_error, DependencyContext +from ..buildlog import UpstreamRequirementFixer from ..resolver.apt import ( NoAptPackage, get_package_for_python_module, ) -from ..requirements import ( - BinaryRequirement, - PathRequirement, - PkgConfigRequirement, - CHeaderRequirement, - JavaScriptRuntimeRequirement, - ValaPackageRequirement, - RubyGemRequirement, - GoPackageRequirement, - DhAddonRequirement, - PhpClassRequirement, - RPackageRequirement, - NodePackageRequirement, - LibraryRequirement, - RubyFileRequirement, - XmlEntityRequirement, - SprocketsFileRequirement, - JavaClassRequirement, - HaskellPackageRequirement, - MavenArtifactRequirement, - GnomeCommonRequirement, - JDKFileRequirement, - PerlModuleRequirement, - PerlFileRequirement, - AutoconfMacroRequirement, - PythonModuleRequirement, - PythonPackageRequirement, - ) from .build import attempt_build, DEFAULT_BUILDER @@ -437,111 +384,6 @@ def fix_missing_python_module(error, context): return True -def problem_to_upstream_requirement(problem): - if isinstance(problem, MissingFile): - return PathRequirement(problem.path) - elif isinstance(problem, MissingCommand): - return BinaryRequirement(problem.command) - elif isinstance(problem, MissingPkgConfig): - return PkgConfigRequirement( - problem.module, problem.minimum_version) - elif isinstance(problem, MissingCHeader): - return CHeaderRequirement(problem.header) - elif isinstance(problem, MissingJavaScriptRuntime): - return JavaScriptRuntimeRequirement() - elif isinstance(problem, MissingRubyGem): - return RubyGemRequirement(problem.gem, problem.version) - elif isinstance(problem, MissingValaPackage): - return ValaPackageRequirement(problem.package) - elif isinstance(problem, MissingGoPackage): - return GoPackageRequirement(problem.package) - elif isinstance(problem, DhAddonLoadFailure): - return DhAddonRequirement(problem.path) - elif isinstance(problem, MissingPhpClass): - return PhpClassRequirement(problem.php_class) - elif isinstance(problem, MissingRPackage): - return RPackageRequirement(problem.package, problem.minimum_version) - elif isinstance(problem, MissingNodeModule): - return NodePackageRequirement(problem.module) - elif isinstance(problem, MissingLibrary): - return LibraryRequirement(problem.library) - elif isinstance(problem, MissingRubyFile): - return RubyFileRequirement(problem.filename) - elif isinstance(problem, MissingXmlEntity): - return XmlEntityRequirement(problem.url) - elif isinstance(problem, MissingSprocketsFile): - return SprocketsFileRequirement(problem.content_type, problem.name) - elif isinstance(problem, MissingJavaClass): - return JavaClassRequirement(problem.classname) - elif isinstance(problem, MissingHaskellDependencies): - # TODO(jelmer): Create multiple HaskellPackageRequirement objects? - return HaskellPackageRequirement(problem.package) - elif isinstance(problem, MissingMavenArtifacts): - # TODO(jelmer): Create multiple MavenArtifactRequirement objects? - return MavenArtifactRequirement(problem.artifacts) - elif isinstance(problem, MissingCSharpCompiler): - return BinaryRequirement('msc') - elif isinstance(problem, GnomeCommonMissing): - return GnomeCommonRequirement() - elif isinstance(problem, MissingJDKFile): - return JDKFileRequirement(problem.jdk_path, problem.filename) - elif isinstance(problem, MissingGnomeCommonDependency): - if problem.package == "glib-gettext": - return BinaryRequirement('glib-gettextize') - else: - logging.warning( - "No known command for gnome-common dependency %s", - problem.package) - return None - elif isinstance(problem, MissingXfceDependency): - if problem.package == "gtk-doc": - return BinaryRequirement("gtkdocize") - else: - logging.warning( - "No known command for xfce dependency %s", - problem.package) - return None - elif isinstance(problem, MissingPerlModule): - return PerlModuleRequirement( - module=problem.module, - filename=problem.filename, - inc=problem.inc) - elif isinstance(problem, MissingPerlFile): - return PerlFileRequirement(filename=problem.filename) - elif isinstance(problem, MissingAutoconfMacro): - return AutoconfMacroRequirement(problem.macro) - elif isinstance(problem, MissingPythonModule): - return PythonModuleRequirement( - problem.module, - python_version=problem.python_version, - minimum_version=problem.minimum_version) - elif isinstance(problem, MissingPythonDistribution): - return PythonPackageRequirement( - problem.module, - python_version=problem.python_version, - minimum_version=problem.minimum_version) - else: - return None - - -class UpstreamRequirementFixer(BuildFixer): - - def can_fix(self, error): - req = problem_to_upstream_requirement(error) - return req is not None - - def fix(self, error, context): - req = problem_to_upstream_requirement(error) - if req is None: - return False - - try: - package = context.resolver.resolve(req) - except NoAptPackage: - return False - return context.add_dependency(package) - - def retry_apt_failure(error, context): return True @@ -635,26 +477,39 @@ def fix_missing_makefile_pl(error, context): return False -VERSIONED_PACKAGE_FIXERS: List[BuildFixer] = [ - SimpleBuildFixer( - NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol), - SimpleBuildFixer(MissingConfigure, fix_missing_configure), - SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input), - SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input), -] +class SimpleBuildFixer(BuildFixer): + + def __init__(self, problem_cls, fn): + self._problem_cls = problem_cls + self._fn = fn + + def can_fix(self, problem): + return isinstance(problem, self._problem_cls) + + def _fix(self, problem, context): + return self._fn(problem, context) -APT_FIXERS: List[BuildFixer] = [ - SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), - SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution), - SimpleBuildFixer(AptFetchFailure, retry_apt_failure), - UpstreamRequirementFixer(), -] +def versioned_package_fixers(): + return [ + SimpleBuildFixer( + NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol), + SimpleBuildFixer(MissingConfigure, fix_missing_configure), + SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input), + SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input), + SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl), + ] -GENERIC_FIXERS: List[BuildFixer] = [ - SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl), -] +def apt_fixers(apt) -> List[BuildFixer]: + from ..resolver.apt import AptResolver + resolver = AptResolver(apt) + return [ + SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), + SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution), + SimpleBuildFixer(AptFetchFailure, retry_apt_failure), + UpstreamRequirementFixer(resolver), + ] def build_incrementally( @@ -720,7 +575,7 @@ def build_incrementally( raise try: if not resolve_error( - e.error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS + GENERIC_FIXERS + e.error, context, versioned_package_fixers() + apt_fixers(apt) ): logging.warning("Failed to resolve error %r. Giving up.", e.error) raise diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index d2d6a9f..c6d25a1 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -47,19 +47,6 @@ class BuildFixer(object): return self._fix(problem, context) -class SimpleBuildFixer(BuildFixer): - - def __init__(self, problem_cls, fn): - self._problem_cls = problem_cls - self._fn = fn - - def can_fix(self, problem): - return isinstance(problem, self._problem_cls) - - def _fix(self, problem, context): - return self._fn(problem, context) - - class DependencyContext(object): def __init__( self, @@ -71,8 +58,6 @@ class DependencyContext(object): ): self.tree = tree self.apt = apt - from .resolver.apt import AptResolver - self.resolver = AptResolver(apt) self.subpath = subpath self.committer = committer self.update_changelog = update_changelog @@ -94,47 +79,23 @@ class SchrootDependencyContext(DependencyContext): return True -def fix_perl_module_from_cpan(error, context): - # TODO(jelmer): Specify -T to skip tests? - context.session.check_call( - ["cpan", "-i", error.module], user="root", env={"PERL_MM_USE_DEFAULT": "1"} - ) - return True - - -NPM_COMMAND_PACKAGES = { - "del-cli": "del-cli", -} - - -def fix_npm_missing_command(error, context): - try: - package = NPM_COMMAND_PACKAGES[error.command] - except KeyError: - return False - - context.session.check_call(["npm", "-g", "install", package]) - return True - - -def fix_python_package_from_pip(error, context): - context.session.check_call(["pip", "install", error.distribution]) - return True - - -GENERIC_INSTALL_FIXERS: List[BuildFixer] = [ - SimpleBuildFixer(MissingPerlModule, fix_perl_module_from_cpan), - SimpleBuildFixer(MissingPythonDistribution, fix_python_package_from_pip), - SimpleBuildFixer(MissingCommand, fix_npm_missing_command), -] +def generic_install_fixers(session): + from .buildlog import UpstreamRequirementFixer + from .resolver import CPANResolver, PypiResolver, NpmResolver + return [ + UpstreamRequirementFixer(CPANResolver(session)), + UpstreamRequirementFixer(PypiResolver(session)), + UpstreamRequirementFixer(NpmResolver(session)), + ] def run_with_build_fixer( session: Session, args: List[str], fixers: Optional[List[BuildFixer]] = None): if fixers is None: - from .debian.fix_build import APT_FIXERS - fixers = GENERIC_INSTALL_FIXERS + APT_FIXERS + from .debian.fix_build import apt_fixers + from .resolver.apt import AptResolver + fixers = generic_install_fixers(session) + apt_fixers(AptResolver.from_session(session)) logging.info("Running %r", args) fixed_errors = [] while True: diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 18bbd98..90e40c7 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -34,20 +34,109 @@ class Resolver(object): raise NotImplementedError(self.met) -class NativeResolver(Resolver): +class CPANResolver(object): + def __init__(self, session): self.session = session - @classmethod - def from_session(cls, session): - return cls(session) - def install(self, requirements): - raise NotImplementedError(self.install) + from ..requirements import PerlModuleRequirement + missing = [] + for requirement in requirements: + if not isinstance(requirement, PerlModuleRequirement): + missing.append(requirement) + continue + # TODO(jelmer): Specify -T to skip tests? + self.session.check_call( + ["cpan", "-i", requirement.module], + user="root", env={"PERL_MM_USE_DEFAULT": "1"} + ) + if missing: + raise MissingDependencies(missing) def explain(self, requirements): raise NotImplementedError(self.explain) + def met(self, requirement): + raise NotImplementedError(self.met) + + +class PypiResolver(object): + + def __init__(self, session): + self.session = session + + def install(self, requirements): + from ..requirements import PythonPackageRequirement + missing = [] + for requirement in requirements: + if not isinstance(requirement, PythonPackageRequirement): + missing.append(requirement) + continue + self.session.check_call(["pip", "install", requirement.package]) + if missing: + raise MissingDependencies(missing) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + def met(self, requirement): + raise NotImplementedError(self.met) + + +NPM_COMMAND_PACKAGES = { + "del-cli": "del-cli", +} + + +class NpmResolver(object): + + def __init__(self, session): + self.session = session + + def install(self, requirements): + from ..requirements import NodePackageRequirement + missing = [] + for requirement in requirements: + if not isinstance(requirement, NodePackageRequirement): + missing.append(requirement) + continue + try: + package = NPM_COMMAND_PACKAGES[requirement.command] + except KeyError: + missing.append(requirement) + continue + self.session.check_call(["npm", "-g", "install", package]) + if missing: + raise MissingDependencies(missing) + + def explain(self, requirements): + raise NotImplementedError(self.explain) + + def met(self, requirement): + raise NotImplementedError(self.met) + + +class StackedResolver(Resolver): + def __init__(self, subs): + self.subs = subs + + def install(self, requirements): + for sub in self.subs: + try: + sub.install(requirements) + except MissingDependencies as e: + requirements = e.requirements + else: + return + + +def native_resolvers(session): + return StackedResolver([ + CPANResolver(session), + PypiResolver(session), + NpmResolver(session)]) + class ExplainResolver(Resolver): def __init__(self, session): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 0c6a783..e34be07 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -21,7 +21,7 @@ import posixpath from ..debian.apt import AptManager -from . import Resolver +from . import Resolver, MissingDependencies from ..requirements import ( BinaryRequirement, CHeaderRequirement, @@ -57,24 +57,35 @@ class NoAptPackage(Exception): """No apt package.""" +class AptRequirement(object): + + def __init__(self, package, minimum_version=None): + self.package = package + self.minimum_version = minimum_version + + def get_package_for_python_package(apt_mgr, package, python_version, minimum_version=None): if python_version == "pypy": - return apt_mgr.get_package_for_paths( + pkg_name = apt_mgr.get_package_for_paths( ["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package], regex=True) elif python_version == "cpython2": - return apt_mgr.get_package_for_paths( + pkg_name = apt_mgr.get_package_for_paths( ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info/PKG-INFO" % package], regex=True) elif python_version == "cpython3": - return apt_mgr.get_package_for_paths( + pkg_name = apt_mgr.get_package_for_paths( ["/usr/lib/python3/dist-packages/%s-.*.egg-info/PKG-INFO" % package], regex=True) else: raise NotImplementedError + # TODO(jelmer): Dealing with epoch, etc? + if pkg_name is not None: + return AptRequirement(pkg_name, minimum_version) + return None -def get_package_for_python_module(apt_mgr, module, python_version): +def get_package_for_python_module(apt_mgr, module, python_version, minimum_version): if python_version == "python3": paths = [ posixpath.join( @@ -127,7 +138,10 @@ def get_package_for_python_module(apt_mgr, module, python_version): ] else: raise AssertionError("unknown python version %r" % python_version) - return apt_mgr.get_package_for_paths(paths, regex=True) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name, minimum_version=minimum_version) + return None def resolve_binary_req(apt_mgr, req): @@ -138,7 +152,10 @@ def resolve_binary_req(apt_mgr, req): posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] ] - return apt_mgr.get_package_for_paths(paths) + pkg_name = apt_mgr.get_package_for_paths(paths) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_pkg_config_req(apt_mgr, req): @@ -151,11 +168,16 @@ def resolve_pkg_config_req(apt_mgr, req): [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], regex=True, minimum_version=req.minimum_version) - return package + if package is not None: + return AptRequirement(package) + return None def resolve_path_req(apt_mgr, req): - return apt_mgr.get_package_for_paths([req.path]) + package = apt_mgr.get_package_for_paths([req.path]) + if package is not None: + return AptRequirement(package) + return None def resolve_c_header_req(apt_mgr, req): @@ -166,17 +188,25 @@ def resolve_c_header_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( [posixpath.join("/usr/include", ".*", req.header)], regex=True ) - return package + if package is None: + return None + return AptRequirement(package) def resolve_js_runtime_req(apt_mgr, req): - return apt_mgr.get_package_for_paths( + package = apt_mgr.get_package_for_paths( ["/usr/bin/node", "/usr/bin/duk"], regex=False) + if package is not None: + return AptRequirement(package) + return None def resolve_vala_package_req(apt_mgr, req): path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package - return apt_mgr.get_package_for_paths([path], regex=True) + package = apt_mgr.get_package_for_paths([path], regex=True) + if package is not None: + return AptRequirement(package) + return None def resolve_ruby_gem_req(apt_mgr, req): @@ -186,30 +216,45 @@ def resolve_ruby_gem_req(apt_mgr, req): "specifications/%s-.*\\.gemspec" % req.gem ) ] - return apt_mgr.get_package_for_paths( - paths, regex=True, minimum_version=req.minimum_version) + package = apt_mgr.get_package_for_paths( + paths, regex=True) + if package is not None: + return AptRequirement(package, minimum_version=req.minimum_version) + return None def resolve_go_package_req(apt_mgr, req): - return apt_mgr.get_package_for_paths( + package = apt_mgr.get_package_for_paths( [posixpath.join("/usr/share/gocode/src", req.package, ".*")], regex=True ) + if package is not None: + return AptRequirement(package) + return None def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] - return apt_mgr.get_package_for_paths(paths) + package = apt_mgr.get_package_for_paths(paths) + if package is not None: + return AptRequirement(package) + return None def resolve_php_class_req(apt_mgr, req): path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") - return apt_mgr.get_package_for_paths([path]) + package = apt_mgr.get_package_for_paths([path]) + if package is not None: + return AptRequirement(package) + return None def resolve_r_package_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)] - return apt_mgr.get_package_for_paths(paths, regex=True) + package = apt_mgr.get_package_for_paths(paths, regex=True) + if package is not None: + return AptRequirement(package) + return None def resolve_node_package_req(apt_mgr, req): @@ -218,7 +263,10 @@ def resolve_node_package_req(apt_mgr, req): "/usr/lib/nodejs/%s/package.json" % req.package, "/usr/share/nodejs/%s/package.json" % req.package, ] - return apt_mgr.get_package_for_paths(paths, regex=True) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_library_req(apt_mgr, req): @@ -228,21 +276,27 @@ def resolve_library_req(apt_mgr, req): posixpath.join("/usr/lib/lib%s.a$" % req.library), posixpath.join("/usr/lib/.*/lib%s.a$" % req.library), ] - return apt_mgr.get_package_for_paths(paths, regex=True) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] package = apt_mgr.get_package_for_paths(paths) if package is not None: - return package + return AptRequirement(package) paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" "lib/%s.rb" % req.filename ) ] - return apt_mgr.get_package_for_paths(paths, regex=True) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_xml_entity_req(apt_mgr, req): @@ -258,7 +312,10 @@ def resolve_xml_entity_req(apt_mgr, req): else: return None - return apt_mgr.get_package_for_paths([search_path], regex=False) + pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_sprockets_file_req(apt_mgr, req): @@ -267,7 +324,10 @@ def resolve_sprockets_file_req(apt_mgr, req): else: logging.warning("unable to handle content type %s", req.content_type) return None - return apt_mgr.get_package_for_paths([path], regex=True) + pkg_name = apt_mgr.get_package_for_paths([path], regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_java_class_req(apt_mgr, req): @@ -285,12 +345,15 @@ def resolve_java_class_req(apt_mgr, req): if package is None: logging.warning("no package for files in %r", classpath) return None - return package + return AptRequirement(package) def resolve_haskell_package_req(apt_mgr, req): path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0] - return apt_mgr.get_package_for_paths([path], regex=True) + pkg_name = apt_mgr.get_package_for_paths([path], regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_maven_artifact_req(apt_mgr, req): @@ -319,16 +382,22 @@ def resolve_maven_artifact_req(apt_mgr, req): "%s-%s.%s" % (artifact_id, version, kind), ) ] - return apt_mgr.get_package_for_paths(paths, regex=regex) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_gnome_common_req(apt_mgr, req): - return 'gnome-common' + return AptRequirement('gnome-common') def resolve_jdk_file_req(apt_mgr, req): path = req.jdk_path + ".*/" + req.filename - return apt_mgr.get_package_for_paths([path], regex=True) + pkg_name = apt_mgr.get_package_for_paths([path], regex=True) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_perl_module_req(apt_mgr, req): @@ -344,11 +413,17 @@ def resolve_perl_module_req(apt_mgr, req): paths = [req.filename] else: paths = [posixpath.join(inc, req.filename) for inc in req.inc] - return apt_mgr.get_package_for_paths(paths, regex=False) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=False) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_perl_file_req(apt_mgr, req): - return apt_mgr.get_package_for_paths([req.filename], regex=False) + pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def _find_aclocal_fun(macro): @@ -370,7 +445,10 @@ def resolve_autoconf_macro_req(apt_mgr, req): except KeyError: logging.info("No local m4 file found defining %s", req.macro) return None - return apt_mgr.get_package_for_paths([path]) + pkg_name = apt_mgr.get_package_for_paths([path]) + if pkg_name is not None: + return AptRequirement(pkg_name) + return None def resolve_python_module_req(apt_mgr, req): @@ -421,14 +499,7 @@ APT_REQUIREMENT_RESOLVERS = [ ] -class AptRequirement(object): - - def __init__(self, package, minimum_version=None): - self.package = package - self.minimum_version = minimum_version - - -def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement): +def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement) -> AptRequirement: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): deb_req = rr_fn(apt_mgr, req) @@ -456,7 +527,17 @@ class AptResolver(Resolver): except NotImplementedError: missing.append(req) if missing: - self.apt.install([self.resolve(m) for m in missing]) + still_missing = [] + apt_requirements = [] + for m in missing: + try: + apt_requirements.append(self.resolve(m)) + except NoAptPackage: + still_missing.append(m) + self.apt.install( + [req.package for req in apt_requirements]) + if still_missing: + raise MissingDependencies(still_missing) def explain(self, requirements): raise NotImplementedError(self.explain) diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index 0cd80ec..c978008 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -34,8 +34,8 @@ from ..debian import apt from ..debian.apt import AptManager from ..debian.fix_build import ( resolve_error, - VERSIONED_PACKAGE_FIXERS, - APT_FIXERS, + versioned_package_fixers, + apt_fixers, BuildDependencyContext, ) from breezy.tests import TestCaseWithTransport @@ -95,10 +95,10 @@ blah (0.1) UNRELEASED; urgency=medium self.tree, apt, subpath="", - committer="Janitor ", + committer="ognibuild ", update_changelog=True, ) - return resolve_error(error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS) + return resolve_error(error, context, versioned_package_fixers() + apt_fixers(apt)) def get_build_deps(self): with open(self.tree.abspath("debian/control"), "r") as f: