Better AptRequirement management.

ognibuild/__main__.py

@@ -52,15 +52,22 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages):
     resolver.install(missing)
 
 
+# Types of dependencies:
+# - core: necessary to do anything with the package
+# - build: necessary to build the package
+# - test: necessary to run the tests
+# - dev: necessary for development (e.g. linters, yacc)
+
 STAGE_MAP = {
     "dist": [],
     "info": [],
-    "install": ["build"],
+    "install": ["core", "build"],
-    "test": ["test", "dev"],
+    "test": ["test", "build", "core"],
-    "build": ["build"],
+    "build": ["build", "core"],
     "clean": [],
 }
 
+
 def determine_fixers(session, resolver):
     from .buildlog import RequirementFixer
     from .resolver.apt import AptResolver

ognibuild/buildsystem.py

@@ -194,11 +194,11 @@ class SetupPy(BuildSystem):
         if self.result is None:
             raise NotImplementedError
         for require in self.result.get_requires():
-            yield "build", PythonPackageRequirement.from_requirement_str(require)
+            yield "core", PythonPackageRequirement.from_requirement_str(require)
         # Not present for distutils-only packages
         if getattr(self.result, 'install_requires', []):
             for require in self.result.install_requires:
-                yield "install", PythonPackageRequirement.from_requirement_str(require)
+                yield "core", PythonPackageRequirement.from_requirement_str(require)
         # Not present for distutils-only packages
         if getattr(self.result, 'tests_require', []):
             for require in self.result.tests_require:
@@ -504,6 +504,8 @@ class Make(BuildSystem):
                     return
                 for require in data.get("requires", []):
                     yield "build", PerlModuleRequirement(require)
+        else:
+            raise NotImplementedError
 
 
 class Cargo(BuildSystem):

ognibuild/debian/fix_build.py

@@ -34,8 +34,7 @@ from breezy.commit import PointlessCommit
 from breezy.mutabletree import MutableTree
 from breezy.tree import Tree
 from debmutate.control import (
-    ensure_some_version,
+    ensure_relation,
-    ensure_minimum_version,
     ControlEditor,
 )
 from debmutate.debhelper import (
@@ -147,24 +146,14 @@ def add_build_dependency(
             for binary in updater.binaries:
                 if binary["Package"] == requirement.package:
                     raise CircularDependency(requirement.package)
-            if requirement.minimum_version:
+            updater.source["Build-Depends"] = ensure_relation(
-                updater.source["Build-Depends"] = ensure_minimum_version(
                     updater.source.get("Build-Depends", ""),
-                    requirement.package, requirement.minimum_version
+                    requirement.relations)
-                )
-            else:
-                updater.source["Build-Depends"] = ensure_some_version(
-                    updater.source.get("Build-Depends", ""),
-                    requirement.package
-                )
     except FormattingUnpreservable as e:
         logging.info("Unable to edit %s in a way that preserves formatting.", e.path)
         return False
 
-    if requirement.minimum_version:
+    desc = PkgRelation.str(requirement.relations)
-        desc = "%s (>= %s)" % (requirement.package, requirement.minimum_version)
-    else:
-        desc = requirement.package
 
     if not updater.changed:
         logging.info("Giving up; dependency %s was already present.", desc)
@@ -204,26 +193,16 @@ def add_test_dependency(
                     command_counter += 1
                 if name != testname:
                     continue
-                if requirement.minimum_version:
+                control["Depends"] = ensure_relation(
-                    control["Depends"] = ensure_minimum_version(
+                    control.get("Depends", ""),
-                        control.get("Depends", ""),
+                    requirement.relations)
-                        requirement.package, requirement.minimum_version
-                    )
-                else:
-                    control["Depends"] = ensure_some_version(
-                        control.get("Depends", ""), requirement.package
-                    )
     except FormattingUnpreservable as e:
         logging.info("Unable to edit %s in a way that preserves formatting.", e.path)
         return False
     if not updater.changed:
         return False
 
-    if requirement.minimum_version:
+    desc = PkgRelation.str(requirement.relations)
-        desc = "%s (>= %s)" % (
-            requirement.package, requirement.minimum_version)
-    else:
-        desc = requirement.package
 
     logging.info("Adding dependency to test %s: %s", testname, desc)
     return commit_debian_changes(
@@ -336,7 +315,7 @@ def fix_missing_python_distribution(error, context):  # noqa: C901
     for dep_pkg in extra_build_deps:
         assert dep_pkg is not None
         if not context.add_dependency(
-                AptRequirement(
+                AptRequirement.simple(
                     dep_pkg.package, minimum_version=error.minimum_version)):
             return False
     return True
@@ -389,7 +368,7 @@ def fix_missing_python_module(error, context):
     for dep_pkg in extra_build_deps:
         assert dep_pkg is not None
         if not context.add_dependency(
-                AptRequirement(dep_pkg.package, error.minimum_version)):
+                AptRequirement.simple(dep_pkg.package, error.minimum_version)):
             return False
     return True
 
@@ -412,7 +391,7 @@ def enable_dh_autoreconf(context):
             return dh_invoke_add_with(line, b"autoreconf")
 
         if update_rules(command_line_cb=add_with_autoreconf):
-            return context.add_dependency(AptRequirement("dh-autoreconf"))
+            return context.add_dependency(AptRequirement.simple("dh-autoreconf"))
 
     return False
 

ognibuild/dist.py

@@ -21,6 +21,7 @@ import os
 import shutil
 import sys
 import tempfile
+import time
 from typing import Optional
 
 from debian.deb822 import Deb822
@@ -80,6 +81,7 @@ class DistCatcher(object):
         self.export_directory = directory
         self.files = []
         self.existing_files = None
+        self.start_time = time.time()
 
     def __enter__(self):
         self.existing_files = os.listdir(self.export_directory)
@@ -110,6 +112,13 @@ class DistCatcher(object):
             self.files.append(os.path.join(parent_directory, fn))
             return fn
 
+        if "dist" in new_files:
+            for entry in os.scandir(os.path.join(self.export_directory, "dist")):
+                if is_dist_file(entry.name) and entry.stat().st_mtime > self.start_time:
+                    logging.info("Found tarball %s in dist directory.", entry.name)
+                    self.files.append(entry.path)
+                    return entry.name
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.find_files()
         return False

ognibuild/resolver/apt.py

@@ -15,10 +15,14 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
+from itertools import chain
 import logging
 import os
 import posixpath
 
+from debian.changelog import Version
+from debian.deb822 import PkgRelation
+
 from ..debian.apt import AptManager
 
 from . import Resolver, UnsatisfiedRequirements
@@ -56,10 +60,20 @@ from ..requirements import (
 
 class AptRequirement(Requirement):
 
-    def __init__(self, package, minimum_version=None):
+    def __init__(self, relations):
         super(AptRequirement, self).__init__('apt')
-        self.package = package
+        self.relations = relations
-        self.minimum_version = minimum_version
+
+    @classmethod
+    def simple(cls, package, minimum_version=None):
+        rel = {'name': package}
+        if minimum_version is not None:
+            rel['version'] = ('>=', minimum_version)
+        return cls([[rel]])
+
+    @classmethod
+    def from_str(cls, text):
+        return cls(PkgRelation.parse_relations(text))
 
 
 def get_package_for_python_package(apt_mgr, package, python_version, specs=None):
@@ -77,18 +91,16 @@ def get_package_for_python_package(apt_mgr, package, python_version, specs=None)
             regex=True)
     else:
         raise NotImplementedError
+    if pkg_name is None:
+        return None
     # TODO(jelmer): Dealing with epoch, etc?
     if not specs:
-        minimum_version = None
+        rels = [[{'name': pkg_name}]]
     else:
+        rels = []
         for spec in specs:
-            if spec[0] == '>=':
+            rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}])
-                minimum_version = spec[1]
+    return AptRequirement(rels)
-            else:
-                raise NotImplementedError(spec)
-    if pkg_name is not None:
-        return AptRequirement(pkg_name, minimum_version)
-    return None
 
 
 def get_package_for_python_module(apt_mgr, module, python_version, specs):
@@ -144,18 +156,17 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
         ]
     else:
         raise AssertionError("unknown python version %r" % python_version)
-    if not specs:
-        minimum_version = None
-    else:
-        for spec in specs:
-            if spec[0] == '>=':
-                minimum_version = spec[1]
-            else:
-                raise NotImplementedError(spec)
     pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
-    if pkg_name is not None:
+    if pkg_name is None:
-        return AptRequirement(pkg_name, minimum_version=minimum_version)
+        return None
-    return None
+    rels = []
+    if not specs:
+        rels = [[{'name': pkg_name}]]
+    else:
+        rels = []
+        for spec in specs:
+            rels.append([{'name': pkg_name, 'version': (spec[0], Version(spec[1]))}])
+    return AptRequirement(rels)
 
 
 def resolve_binary_req(apt_mgr, req):
@@ -168,7 +179,7 @@ def resolve_binary_req(apt_mgr, req):
         ]
     pkg_name = apt_mgr.get_package_for_paths(paths)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -181,14 +192,14 @@ def resolve_pkg_config_req(apt_mgr, req):
             [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")],
             regex=True)
     if package is not None:
-        return AptRequirement(package, minimum_version=req.minimum_version)
+        return AptRequirement.simple(package, minimum_version=req.minimum_version)
     return None
 
 
 def resolve_path_req(apt_mgr, req):
     package = apt_mgr.get_package_for_paths([req.path])
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -202,14 +213,14 @@ def resolve_c_header_req(apt_mgr, req):
         )
     if package is None:
         return None
-    return AptRequirement(package)
+    return AptRequirement.simple(package)
 
 
 def resolve_js_runtime_req(apt_mgr, req):
     package = apt_mgr.get_package_for_paths(
         ["/usr/bin/node", "/usr/bin/duk"], regex=False)
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -217,7 +228,7 @@ def resolve_vala_package_req(apt_mgr, req):
     path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package
     package = apt_mgr.get_package_for_paths([path], regex=True)
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -231,7 +242,7 @@ def resolve_ruby_gem_req(apt_mgr, req):
     package = apt_mgr.get_package_for_paths(
         paths, regex=True)
     if package is not None:
-        return AptRequirement(package, minimum_version=req.minimum_version)
+        return AptRequirement.simple(package, minimum_version=req.minimum_version)
     return None
 
 
@@ -241,7 +252,7 @@ def resolve_go_package_req(apt_mgr, req):
         regex=True
     )
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -249,7 +260,7 @@ def resolve_dh_addon_req(apt_mgr, req):
     paths = [posixpath.join("/usr/share/perl5", req.path)]
     package = apt_mgr.get_package_for_paths(paths)
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -257,7 +268,7 @@ def resolve_php_class_req(apt_mgr, req):
     path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/")
     package = apt_mgr.get_package_for_paths([path])
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -265,7 +276,7 @@ def resolve_r_package_req(apt_mgr, req):
     paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)]
     package = apt_mgr.get_package_for_paths(paths, regex=True)
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     return None
 
 
@@ -277,7 +288,7 @@ def resolve_node_package_req(apt_mgr, req):
     ]
     pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -290,7 +301,7 @@ def resolve_library_req(apt_mgr, req):
     ]
     pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -298,7 +309,7 @@ def resolve_ruby_file_req(apt_mgr, req):
     paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)]
     package = apt_mgr.get_package_for_paths(paths)
     if package is not None:
-        return AptRequirement(package)
+        return AptRequirement.simple(package)
     paths = [
         posixpath.join(
             r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
@@ -307,7 +318,7 @@ def resolve_ruby_file_req(apt_mgr, req):
     ]
     pkg_name = apt_mgr.get_package_for_paths(paths, regex=True)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -326,7 +337,7 @@ def resolve_xml_entity_req(apt_mgr, req):
 
     pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -338,7 +349,7 @@ def resolve_sprockets_file_req(apt_mgr, req):
         return None
     pkg_name = apt_mgr.get_package_for_paths([path], regex=True)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -357,14 +368,14 @@ def resolve_java_class_req(apt_mgr, req):
     if package is None:
         logging.warning("no package for files in %r", classpath)
         return None
-    return AptRequirement(package)
+    return AptRequirement.simple(package)
 
 
 def resolve_haskell_package_req(apt_mgr, req):
     path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0]
     pkg_name = apt_mgr.get_package_for_paths([path], regex=True)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -396,19 +407,19 @@ def resolve_maven_artifact_req(apt_mgr, req):
     ]
     pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
 def resolve_gnome_common_req(apt_mgr, req):
-    return AptRequirement('gnome-common')
+    return AptRequirement.simple('gnome-common')
 
 
 def resolve_jdk_file_req(apt_mgr, req):
     path = req.jdk_path + ".*/" + req.filename
     pkg_name = apt_mgr.get_package_for_paths([path], regex=True)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -427,14 +438,14 @@ def resolve_perl_module_req(apt_mgr, req):
         paths = [posixpath.join(inc, req.filename) for inc in req.inc]
     pkg_name = apt_mgr.get_package_for_paths(paths, regex=False)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
 def resolve_perl_file_req(apt_mgr, req):
     pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False)
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -459,7 +470,7 @@ def resolve_autoconf_macro_req(apt_mgr, req):
         return None
     pkg_name = apt_mgr.get_package_for_paths([path])
     if pkg_name is not None:
-        return AptRequirement(pkg_name)
+        return AptRequirement.simple(pkg_name)
     return None
 
 
@@ -549,7 +560,8 @@ class AptResolver(Resolver):
             else:
                 apt_requirements.append(apt_req)
         if apt_requirements:
-            self.apt.install([r.package for r in apt_requirements])
+            self.apt.satisfy([PkgRelation.str(chain(*[
+                r.relations for r in apt_requirements]))])
         if still_missing:
             raise UnsatisfiedRequirements(still_missing)