From 8955497adf088a760b7e85a59efc966d75ed9f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 1 Mar 2021 17:14:53 +0000 Subject: [PATCH] Better AptRequirement management. --- ognibuild/__main__.py | 13 +++- ognibuild/buildsystem.py | 6 +- ognibuild/debian/fix_build.py | 43 ++++---------- ognibuild/dist.py | 9 +++ ognibuild/resolver/apt.py | 108 +++++++++++++++++++--------------- 5 files changed, 94 insertions(+), 85 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index b657e6b..d5f5ef4 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -52,15 +52,22 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages): resolver.install(missing) +# Types of dependencies: +# - core: necessary to do anything with the package +# - build: necessary to build the package +# - test: necessary to run the tests +# - dev: necessary for development (e.g. linters, yacc) + STAGE_MAP = { "dist": [], "info": [], - "install": ["build"], - "test": ["test", "dev"], - "build": ["build"], + "install": ["core", "build"], + "test": ["test", "build", "core"], + "build": ["build", "core"], "clean": [], } + def determine_fixers(session, resolver): from .buildlog import RequirementFixer from .resolver.apt import AptResolver diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index de153c8..2848f18 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -194,11 +194,11 @@ class SetupPy(BuildSystem): if self.result is None: raise NotImplementedError for require in self.result.get_requires(): - yield "build", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages if getattr(self.result, 'install_requires', []): for require in self.result.install_requires: - yield "install", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages if getattr(self.result, 'tests_require', []): for require in self.result.tests_require: @@ -504,6 +504,8 @@ class Make(BuildSystem): return for require in data.get("requires", []): yield "build", PerlModuleRequirement(require) + else: + raise NotImplementedError class Cargo(BuildSystem): diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index d20302a..2cfa55d 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -34,8 +34,7 @@ from breezy.commit import PointlessCommit from breezy.mutabletree import MutableTree from breezy.tree import Tree from debmutate.control import ( - ensure_some_version, - ensure_minimum_version, + ensure_relation, ControlEditor, ) from debmutate.debhelper import ( @@ -147,24 +146,14 @@ def add_build_dependency( for binary in updater.binaries: if binary["Package"] == requirement.package: raise CircularDependency(requirement.package) - if requirement.minimum_version: - updater.source["Build-Depends"] = ensure_minimum_version( + updater.source["Build-Depends"] = ensure_relation( updater.source.get("Build-Depends", ""), - requirement.package, requirement.minimum_version - ) - else: - updater.source["Build-Depends"] = ensure_some_version( - updater.source.get("Build-Depends", ""), - requirement.package - ) + requirement.relations) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False - if requirement.minimum_version: - desc = "%s (>= %s)" % (requirement.package, requirement.minimum_version) - else: - desc = requirement.package + desc = PkgRelation.str(requirement.relations) if not updater.changed: logging.info("Giving up; dependency %s was already present.", desc) @@ -204,26 +193,16 @@ def add_test_dependency( command_counter += 1 if name != testname: continue - if requirement.minimum_version: - control["Depends"] = ensure_minimum_version( - control.get("Depends", ""), - requirement.package, requirement.minimum_version - ) - else: - control["Depends"] = ensure_some_version( - control.get("Depends", ""), requirement.package - ) + control["Depends"] = ensure_relation( + control.get("Depends", ""), + requirement.relations) except FormattingUnpreservable as e: logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False if not updater.changed: return False - if requirement.minimum_version: - desc = "%s (>= %s)" % ( - requirement.package, requirement.minimum_version) - else: - desc = requirement.package + desc = PkgRelation.str(requirement.relations) logging.info("Adding dependency to test %s: %s", testname, desc) return commit_debian_changes( @@ -336,7 +315,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 for dep_pkg in extra_build_deps: assert dep_pkg is not None if not context.add_dependency( - AptRequirement( + AptRequirement.simple( dep_pkg.package, minimum_version=error.minimum_version)): return False return True @@ -389,7 +368,7 @@ def fix_missing_python_module(error, context): for dep_pkg in extra_build_deps: assert dep_pkg is not None if not context.add_dependency( - AptRequirement(dep_pkg.package, error.minimum_version)): + AptRequirement.simple(dep_pkg.package, error.minimum_version)): return False return True @@ -412,7 +391,7 @@ def enable_dh_autoreconf(context): return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return context.add_dependency(AptRequirement("dh-autoreconf")) + return context.add_dependency(AptRequirement.simple("dh-autoreconf")) return False diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 31baf21..9c97e0e 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -21,6 +21,7 @@ import os import shutil import sys import tempfile +import time from typing import Optional from debian.deb822 import Deb822 @@ -80,6 +81,7 @@ class DistCatcher(object): self.export_directory = directory self.files = [] self.existing_files = None + self.start_time = time.time() def __enter__(self): self.existing_files = os.listdir(self.export_directory) @@ -110,6 +112,13 @@ class DistCatcher(object): self.files.append(os.path.join(parent_directory, fn)) return fn + if "dist" in new_files: + for entry in os.scandir(os.path.join(self.export_directory, "dist")): + if is_dist_file(entry.name) and entry.stat().st_mtime > self.start_time: + logging.info("Found tarball %s in dist directory.", entry.name) + self.files.append(entry.path) + return entry.name + def __exit__(self, exc_type, exc_val, exc_tb): self.find_files() return False diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 50cf9f6..6d470cc 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -15,10 +15,14 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from itertools import chain import logging import os import posixpath +from debian.changelog import Version +from debian.deb822 import PkgRelation + from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements @@ -56,10 +60,20 @@ from ..requirements import ( class AptRequirement(Requirement): - def __init__(self, package, minimum_version=None): + def __init__(self, relations): super(AptRequirement, self).__init__('apt') - self.package = package - self.minimum_version = minimum_version + self.relations = relations + + @classmethod + def simple(cls, package, minimum_version=None): + rel = {'name': package} + if minimum_version is not None: + rel['version'] = ('>=', minimum_version) + return cls([[rel]]) + + @classmethod + def from_str(cls, text): + return cls(PkgRelation.parse_relations(text)) def get_package_for_python_package(apt_mgr, package, python_version, specs=None): @@ -77,18 +91,16 @@ def get_package_for_python_package(apt_mgr, package, python_version, specs=None) regex=True) else: raise NotImplementedError + if pkg_name is None: + return None # TODO(jelmer): Dealing with epoch, etc? if not specs: - minimum_version = None + rels = [[{'name': pkg_name}]] else: + rels = [] for spec in specs: - if spec[0] == '>=': - minimum_version = spec[1] - else: - raise NotImplementedError(spec) - if pkg_name is not None: - return AptRequirement(pkg_name, minimum_version) - return None + rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}]) + return AptRequirement(rels) def get_package_for_python_module(apt_mgr, module, python_version, specs): @@ -144,18 +156,17 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): ] else: raise AssertionError("unknown python version %r" % python_version) - if not specs: - minimum_version = None - else: - for spec in specs: - if spec[0] == '>=': - minimum_version = spec[1] - else: - raise NotImplementedError(spec) pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement(pkg_name, minimum_version=minimum_version) - return None + if pkg_name is None: + return None + rels = [] + if not specs: + rels = [[{'name': pkg_name}]] + else: + rels = [] + for spec in specs: + rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}]) + return AptRequirement(rels) def resolve_binary_req(apt_mgr, req): @@ -168,7 +179,7 @@ def resolve_binary_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -181,14 +192,14 @@ def resolve_pkg_config_req(apt_mgr, req): [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], regex=True) if package is not None: - return AptRequirement(package, minimum_version=req.minimum_version) + return AptRequirement.simple(package, minimum_version=req.minimum_version) return None def resolve_path_req(apt_mgr, req): package = apt_mgr.get_package_for_paths([req.path]) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -202,14 +213,14 @@ def resolve_c_header_req(apt_mgr, req): ) if package is None: return None - return AptRequirement(package) + return AptRequirement.simple(package) def resolve_js_runtime_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( ["/usr/bin/node", "/usr/bin/duk"], regex=False) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -217,7 +228,7 @@ def resolve_vala_package_req(apt_mgr, req): path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package package = apt_mgr.get_package_for_paths([path], regex=True) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -231,7 +242,7 @@ def resolve_ruby_gem_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( paths, regex=True) if package is not None: - return AptRequirement(package, minimum_version=req.minimum_version) + return AptRequirement.simple(package, minimum_version=req.minimum_version) return None @@ -241,7 +252,7 @@ def resolve_go_package_req(apt_mgr, req): regex=True ) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -249,7 +260,7 @@ def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] package = apt_mgr.get_package_for_paths(paths) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -257,7 +268,7 @@ def resolve_php_class_req(apt_mgr, req): path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") package = apt_mgr.get_package_for_paths([path]) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -265,7 +276,7 @@ def resolve_r_package_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)] package = apt_mgr.get_package_for_paths(paths, regex=True) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) return None @@ -277,7 +288,7 @@ def resolve_node_package_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -290,7 +301,7 @@ def resolve_library_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -298,7 +309,7 @@ def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] package = apt_mgr.get_package_for_paths(paths) if package is not None: - return AptRequirement(package) + return AptRequirement.simple(package) paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" @@ -307,7 +318,7 @@ def resolve_ruby_file_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -326,7 +337,7 @@ def resolve_xml_entity_req(apt_mgr, req): pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -338,7 +349,7 @@ def resolve_sprockets_file_req(apt_mgr, req): return None pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -357,14 +368,14 @@ def resolve_java_class_req(apt_mgr, req): if package is None: logging.warning("no package for files in %r", classpath) return None - return AptRequirement(package) + return AptRequirement.simple(package) def resolve_haskell_package_req(apt_mgr, req): path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0] pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -396,19 +407,19 @@ def resolve_maven_artifact_req(apt_mgr, req): ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None def resolve_gnome_common_req(apt_mgr, req): - return AptRequirement('gnome-common') + return AptRequirement.simple('gnome-common') def resolve_jdk_file_req(apt_mgr, req): path = req.jdk_path + ".*/" + req.filename pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -427,14 +438,14 @@ def resolve_perl_module_req(apt_mgr, req): paths = [posixpath.join(inc, req.filename) for inc in req.inc] pkg_name = apt_mgr.get_package_for_paths(paths, regex=False) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None def resolve_perl_file_req(apt_mgr, req): pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -459,7 +470,7 @@ def resolve_autoconf_macro_req(apt_mgr, req): return None pkg_name = apt_mgr.get_package_for_paths([path]) if pkg_name is not None: - return AptRequirement(pkg_name) + return AptRequirement.simple(pkg_name) return None @@ -549,7 +560,8 @@ class AptResolver(Resolver): else: apt_requirements.append(apt_req) if apt_requirements: - self.apt.install([r.package for r in apt_requirements]) + self.apt.satisfy([PkgRelation.str(chain(*[ + r.relations for r in apt_requirements]))]) if still_missing: raise UnsatisfiedRequirements(still_missing)