diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 864f28f..123fa45 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -20,12 +20,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip flake8 cython \ - git+https://salsa.debian.org/python-debian-team/python-debian \ - "git+https://salsa.debian.org/debian/distro-info#subdirectory=python&egg=distro-info" \ - git+https://salsa.debian.org/jelmer/lintian-brush \ - breezy \ - testtools + python -m pip install --upgrade pip flake8 cython + python -m pip install git+https://github.com/jelmer/buildlog-consultant python setup.py develop mkdir -p ~/.config/breezy/plugins brz branch lp:brz-debian ~/.config/breezy/plugins/debian diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..b1ac686 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Jelmer Vernooij diff --git a/ROADMAP b/ROADMAP new file mode 100644 index 0000000..f48556d --- /dev/null +++ b/ROADMAP @@ -0,0 +1,44 @@ +class UpstreamRequirement(object): + + family: str + + +class PythonPackageRequirement(UpstreamRequirement): + + package: str + + +SetupPy.get_build_requirements() yields some PythonPackageRequirement objects + +apt_resolver.install([PythonPackageRequirement(...)]) then: + + * needs to translate to apt package name + + +Once we find errors during build, buildlog consultant extracts them ("MissingPythonPackage", "configure.ac needs updating"). + +fix_build then takes the problem found and converts it to an action: + + * modifying some of the source files + * resolving requirements + +Resolving requirements dependencies means creating e.g. a PythonPackageRequirement() object and feeding it to resolver.install() + +we have specific handlers for each kind of thingy + +resolver.install() needs to translate the upstream information to an apt name or a cpan name or update dependencies or raise an exception or.. + +MissingPythonPackage() -> PythonPackageRequirement() + +PythonPackageRequirement() can either: + + * directly provide apt names, if they are known + * look up apt names + +We specifically want to support multiple resolvers. In some cases a resolver can't deal with a particular kind of requirement. + +Who is responsible for taking a PythonPackageRequirement and translating it to an apt package name? + + 1) PythonPackageRequirement itself? That would mean knowledge about package naming etc, is with the requirement object, which seems wrong. + 2) PythonPackageRequirement.apt_name(apt_archive) - i.e. find the package name given an archive object of some sort + 3) The apt resolver has a list of callbacks to map requirements to apt package names diff --git a/notes/architecture.md b/notes/architecture.md new file mode 100644 index 0000000..960892c --- /dev/null +++ b/notes/architecture.md @@ -0,0 +1,30 @@ +Upstream requirements are expressed as objects derived from UpstreamRequirement. + +They can either be: + + * extracted from the build system + * extracted from errors in build logs + +The details of UpstreamRequirements are specific to the kind of requirement, +and otherwise opaque to ognibuild. + +When building a package, we first make sure that all declared upstream +requirements are met. + +Then we attempt to build. + +If any problems are found in the log, buildlog-consultant will report them. + +ognibuild can then invoke "fixers" to address Problems. + +Problems can be converted to UpstreamRequirements by UpstreamRequirementFixer + +Other Fixer can do things like e.g. upgrade configure.ac to a newer version. + +UpstreamRequirementFixer uses a UpstreamRequirementResolver object that +can translate UpstreamRequirement objects into apt package names or +e.g. cpan commands. + +ognibuild keeps finding problems, resolving them and rebuilding until it finds +a problem it can not resolve or that it thinks it has already resolved +(i.e. seen before). diff --git a/ognibuild/apt.py b/ognibuild/apt.py index 3509c73..8783a49 100644 --- a/ognibuild/apt.py +++ b/ognibuild/apt.py @@ -16,19 +16,22 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -from typing import List +import logging +import re +from typing import List, Iterator, Optional, Set import os from buildlog_consultant.apt import ( find_apt_get_failure, ) +from debian.deb822 import Release from . import DetailedFailure from .session import Session, run_with_tee class UnidentifiedError(Exception): + def __init__(self, retcode, argv, lines, secondary=None): self.retcode = retcode self.argv = argv @@ -42,11 +45,11 @@ def run_apt(session: Session, args: List[str]) -> None: retcode, lines = run_with_tee(session, args, cwd="/", user="root") if retcode == 0: return - offset, line, error = find_apt_get_failure(lines) + match, error = find_apt_get_failure(lines) if error is not None: raise DetailedFailure(retcode, args, error) - if line is not None: - raise UnidentifiedError(retcode, args, lines, secondary=(offset, line)) + if match is not None: + raise UnidentifiedError(retcode, args, lines, secondary=(match.lineno, match.line)) while lines and lines[-1] == "": lines.pop(-1) raise UnidentifiedError(retcode, args, lines) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 2345981..122f477 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -61,13 +61,11 @@ def changes_filename(package, version, arch): def get_build_architecture(): try: - return ( - subprocess.check_output(["dpkg-architecture", "-qDEB_BUILD_ARCH"]) - .strip() - .decode() - ) + return subprocess.check_output( + ['dpkg-architecture', '-qDEB_BUILD_ARCH']).strip().decode() except subprocess.CalledProcessError as e: - raise Exception("Could not find the build architecture: %s" % e) + raise Exception( + "Could not find the build architecture: %s" % e) def add_dummy_changelog_entry( diff --git a/ognibuild/dist.py b/ognibuild/dist.py index a473c22..e52dc25 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -28,12 +28,13 @@ from debian.deb822 import Deb822 from breezy.tree import Tree from breezy.workingtree import WorkingTree -from . import DetailedFailure -from .buildsystem import detect_buildsystems, NoBuildToolsFound from buildlog_consultant.common import ( NoSpaceOnDevice, ) + +from . import DetailedFailure +from .buildsystem import detect_buildsystems, NoBuildToolsFound from .session.schroot import SchrootSession from .vcs import dupe_vcs_tree, export_vcs_tree @@ -47,7 +48,7 @@ SUPPORTED_DIST_EXTENSIONS = [ ".tbz2", ".tar", ".zip", -] + ] def is_dist_file(fn): diff --git a/setup.cfg b/setup.cfg index fc17caa..99a5d9d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,4 @@ [flake8] -application-package-names = ognibuild banned-modules = silver-platter = Should not use silver-platter [mypy]