Import some fixes from resolver branch.

.github/workflows/pythonpackage.yml

@@ -20,12 +20,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip flake8 cython \
+        python -m pip install --upgrade pip flake8 cython
-            git+https://salsa.debian.org/python-debian-team/python-debian \
+        python -m pip install git+https://github.com/jelmer/buildlog-consultant
-            "git+https://salsa.debian.org/debian/distro-info#subdirectory=python&egg=distro-info" \
-            git+https://salsa.debian.org/jelmer/lintian-brush \
-            breezy \
-            testtools
         python setup.py develop
         mkdir -p ~/.config/breezy/plugins
         brz branch lp:brz-debian ~/.config/breezy/plugins/debian

AUTHORS

@@ -0,0 +1 @@
+Jelmer Vernooij <jelmer@jelmer.uk>

ROADMAP

@@ -0,0 +1,44 @@
+class UpstreamRequirement(object):
+
+    family: str
+
+
+class PythonPackageRequirement(UpstreamRequirement):
+
+    package: str
+
+
+SetupPy.get_build_requirements() yields some PythonPackageRequirement objects
+
+apt_resolver.install([PythonPackageRequirement(...)]) then:
+
+ * needs to translate to apt package name
+
+
+Once we find errors during build, buildlog consultant extracts them ("MissingPythonPackage", "configure.ac needs updating").
+
+fix_build then takes the problem found and converts it to an action:
+
+ * modifying some of the source files
+ * resolving requirements
+
+Resolving requirements dependencies means creating e.g. a PythonPackageRequirement() object and feeding it to resolver.install()
+
+we have specific handlers for each kind of thingy
+
+resolver.install() needs to translate the upstream information to an apt name or a cpan name or update dependencies or raise an exception or..
+
+MissingPythonPackage() -> PythonPackageRequirement()
+
+PythonPackageRequirement() can either:
+
+ * directly provide apt names, if they are known
+ * look up apt names
+
+We specifically want to support multiple resolvers. In some cases a resolver can't deal with a particular kind of requirement.
+
+Who is responsible for taking a PythonPackageRequirement and translating it to an apt package name?
+
+ 1) PythonPackageRequirement itself? That would mean knowledge about package naming etc, is with the requirement object, which seems wrong.
+ 2) PythonPackageRequirement.apt_name(apt_archive) - i.e. find the package name given an archive object of some sort
+ 3) The apt resolver has a list of callbacks to map requirements to apt package names

notes/architecture.md

@@ -0,0 +1,30 @@
+Upstream requirements are expressed as objects derived from UpstreamRequirement.
+
+They can either be:
+
+ * extracted from the build system
+ * extracted from errors in build logs
+
+The details of UpstreamRequirements are specific to the kind of requirement,
+and otherwise opaque to ognibuild.
+
+When building a package, we first make sure that all declared upstream
+requirements are met.
+
+Then we attempt to build.
+
+If any problems are found in the log, buildlog-consultant will report them.
+
+ognibuild can then invoke "fixers" to address Problems.
+
+Problems can be converted to UpstreamRequirements by UpstreamRequirementFixer
+
+Other Fixer can do things like e.g. upgrade configure.ac to a newer version.
+
+UpstreamRequirementFixer uses a UpstreamRequirementResolver object that
+can translate UpstreamRequirement objects into apt package names or
+e.g. cpan commands.
+
+ognibuild keeps finding problems, resolving them and rebuilding until it finds
+a problem it can not resolve or that it thinks it has already resolved
+(i.e. seen before).

ognibuild/apt.py

@@ -16,19 +16,22 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 
-
+import logging
-from typing import List
+import re
+from typing import List, Iterator, Optional, Set
 
 import os
 from buildlog_consultant.apt import (
     find_apt_get_failure,
 )
+from debian.deb822 import Release
 
 from . import DetailedFailure
 from .session import Session, run_with_tee
 
 
 class UnidentifiedError(Exception):
+
     def __init__(self, retcode, argv, lines, secondary=None):
         self.retcode = retcode
         self.argv = argv
@@ -42,11 +45,11 @@ def run_apt(session: Session, args: List[str]) -> None:
     retcode, lines = run_with_tee(session, args, cwd="/", user="root")
     if retcode == 0:
         return
-    offset, line, error = find_apt_get_failure(lines)
+    match, error = find_apt_get_failure(lines)
     if error is not None:
         raise DetailedFailure(retcode, args, error)
-    if line is not None:
+    if match is not None:
-        raise UnidentifiedError(retcode, args, lines, secondary=(offset, line))
+        raise UnidentifiedError(retcode, args, lines, secondary=(match.lineno, match.line))
     while lines and lines[-1] == "":
         lines.pop(-1)
     raise UnidentifiedError(retcode, args, lines)

ognibuild/debian/build.py

@@ -61,13 +61,11 @@ def changes_filename(package, version, arch):
 
 def get_build_architecture():
     try:
-        return (
+        return subprocess.check_output(
-            subprocess.check_output(["dpkg-architecture", "-qDEB_BUILD_ARCH"])
+            ['dpkg-architecture', '-qDEB_BUILD_ARCH']).strip().decode()
-            .strip()
-            .decode()
-        )
     except subprocess.CalledProcessError as e:
-        raise Exception("Could not find the build architecture: %s" % e)
+        raise Exception(
+            "Could not find the build architecture: %s" % e)
 
 
 def add_dummy_changelog_entry(

ognibuild/dist.py

@@ -28,12 +28,13 @@ from debian.deb822 import Deb822
 from breezy.tree import Tree
 from breezy.workingtree import WorkingTree
 
-from . import DetailedFailure
-from .buildsystem import detect_buildsystems, NoBuildToolsFound
 from buildlog_consultant.common import (
     NoSpaceOnDevice,
 )
 
+
+from . import DetailedFailure
+from .buildsystem import detect_buildsystems, NoBuildToolsFound
 from .session.schroot import SchrootSession
 from .vcs import dupe_vcs_tree, export_vcs_tree
 
@@ -47,7 +48,7 @@ SUPPORTED_DIST_EXTENSIONS = [
     ".tbz2",
     ".tar",
     ".zip",
-]
+    ]
 
 
 def is_dist_file(fn):

setup.cfg

@@ -1,5 +1,4 @@
 [flake8]
-application-package-names = ognibuild
 banned-modules = silver-platter = Should not use silver-platter
 
 [mypy]