Import Debian package fixing logic.

README.md

@@ -31,6 +31,9 @@ Ognibuild has a number of subcommands:
  * ``ogni install`` - install the package
  * ``ogni test`` - run the testsuite in the source directory
 
+It also includes a subcommand that can fix up the build dependencies
+for Debian packages, called deb-fix-build.
+
 License
 -------
 

ognibuild/debian/build.py

@@ -0,0 +1,184 @@
+#!/usr/bin/python
+# Copyright (C) 2018 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+__all__ = [
+    'changes_filename',
+    'get_build_architecture',
+    'add_dummy_changelog_entry',
+    'build',
+    'SbuildFailure',
+]
+
+from datetime import datetime
+import logging
+import os
+import re
+import subprocess
+import sys
+
+from debian.changelog import Changelog
+from debmutate.changelog import get_maintainer, format_datetime
+
+from breezy import osutils
+from breezy.plugins.debian.util import (
+    changes_filename,
+    get_build_architecture,
+    )
+from breezy.mutabletree import MutableTree
+from silver_platter.debian import (
+    BuildFailedError,
+    DEFAULT_BUILDER,
+    )
+
+from buildlog_consultant.sbuild import (
+    worker_failure_from_sbuild_log,
+    SbuildFailure,
+    )
+
+
+class MissingChangesFile(Exception):
+    """Expected changes file was not written."""
+
+    def __init__(self, filename):
+        self.filename = filename
+
+
+def add_dummy_changelog_entry(
+        tree: MutableTree, subpath: str, suffix: str, suite: str,
+        message: str, timestamp=None, maintainer=None):
+    """Add a dummy changelog entry to a package.
+
+    Args:
+      directory: Directory to run in
+      suffix: Suffix for the version
+      suite: Debian suite
+      message: Changelog message
+    """
+    def add_suffix(v, suffix):
+        m = re.fullmatch('(.*)(' + re.escape(suffix) + ')([0-9]+)', v,)
+        if m:
+            return (m.group(1) + m.group(2) + '%d' % (int(m.group(3)) + 1))
+        else:
+            return v + suffix + '1'
+
+    path = os.path.join(subpath, 'debian', 'changelog')
+    if maintainer is None:
+        maintainer = get_maintainer()
+    if timestamp is None:
+        timestamp = datetime.now()
+    with tree.get_file(path) as f:
+        cl = Changelog()
+        cl.parse_changelog(
+            f, max_blocks=None, allow_empty_author=True, strict=False)
+        version = cl[0].version
+        if version.debian_revision:
+            version.debian_revision = add_suffix(
+                version.debian_revision, suffix)
+        else:
+            version.upstream_version = add_suffix(
+                version.upstream_version, suffix)
+        cl.new_block(
+            package=cl[0].package,
+            version=version,
+            urgency='low',
+            distributions=suite,
+            author='%s <%s>' % maintainer,
+            date=format_datetime(timestamp),
+            changes=['', '  * ' + message, ''])
+    cl_str = cl._format(allow_missing_author=True)
+    tree.put_file_bytes_non_atomic(path, cl_str.encode(cl._encoding))
+
+
+def get_latest_changelog_version(local_tree, subpath=''):
+    path = osutils.pathjoin(subpath, 'debian/changelog')
+    with local_tree.get_file(path) as f:
+        cl = Changelog(f, max_blocks=1)
+        return cl.package, cl.version
+
+
+def build(local_tree, outf, build_command=DEFAULT_BUILDER, result_dir=None,
+          distribution=None, subpath='', source_date_epoch=None):
+    args = [sys.executable, '-m', 'breezy', 'builddeb',
+            '--guess-upstream-branch-url', '--builder=%s' % build_command]
+    if result_dir:
+        args.append('--result-dir=%s' % result_dir)
+    outf.write('Running %r\n' % (build_command, ))
+    outf.flush()
+    env = dict(os.environ.items())
+    if distribution is not None:
+        env['DISTRIBUTION'] = distribution
+    if source_date_epoch is not None:
+        env['SOURCE_DATE_EPOCH'] = '%d' % source_date_epoch
+    logging.info('Building debian packages, running %r.', build_command)
+    try:
+        subprocess.check_call(
+            args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf,
+            env=env)
+    except subprocess.CalledProcessError:
+        raise BuildFailedError()
+
+
+def build_once(
+        local_tree, build_suite, output_directory, build_command,
+        subpath='', source_date_epoch=None):
+    build_log_path = os.path.join(output_directory, 'build.log')
+    try:
+        with open(build_log_path, 'w') as f:
+            build(local_tree, outf=f, build_command=build_command,
+                  result_dir=output_directory, distribution=build_suite,
+                  subpath=subpath, source_date_epoch=source_date_epoch)
+    except BuildFailedError:
+        with open(build_log_path, 'rb') as f:
+            raise worker_failure_from_sbuild_log(f)
+
+    (cl_package, cl_version) = get_latest_changelog_version(
+        local_tree, subpath)
+    changes_name = changes_filename(
+        cl_package, cl_version, get_build_architecture())
+    changes_path = os.path.join(output_directory, changes_name)
+    if not os.path.exists(changes_path):
+        raise MissingChangesFile(changes_name)
+    return (changes_name, cl_version)
+
+
+def gbp_dch(path):
+    subprocess.check_call(['gbp', 'dch'], cwd=path)
+
+
+def attempt_build(
+        local_tree, suffix, build_suite, output_directory, build_command,
+        build_changelog_entry='Build for debian-janitor apt repository.',
+        subpath='', source_date_epoch=None):
+    """Attempt a build, with a custom distribution set.
+
+    Args:
+      local_tree: Tree to build in
+      suffix: Suffix to add to version string
+      build_suite: Name of suite (i.e. distribution) to build for
+      output_directory: Directory to write output to
+      build_command: Build command to build package
+      build_changelog_entry: Changelog entry to use
+      subpath: Sub path in tree where package lives
+      source_date_epoch: Source date epoch to set
+    Returns: Tuple with (changes_name, cl_version)
+    """
+    add_dummy_changelog_entry(
+        local_tree, subpath, suffix, build_suite,
+        build_changelog_entry)
+    return build_once(
+        local_tree, build_suite, output_directory, build_command, subpath,
+        source_date_epoch=source_date_epoch)

ognibuild/dist.py

@@ -0,0 +1,524 @@
+#!/usr/bin/python3
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import errno
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from typing import Optional, List, Tuple, Callable, Type
+
+from debian.deb822 import Deb822
+
+from breezy.export import export
+from breezy.tree import Tree
+from breezy.workingtree import WorkingTree
+
+from breezy.plugins.debian.repack_tarball import get_filetype
+
+from .fix_build import (
+    DependencyContext,
+    resolve_error,
+    APT_FIXERS,
+    )
+from buildlog_consultant.sbuild import (
+    find_apt_get_failure,
+    find_build_failure_description,
+    Problem,
+    MissingPerlModule,
+    MissingCommand,
+    NoSpaceOnDevice,
+    )
+from ognibuild import shebang_binary
+from ognibuild.session import Session
+from ognibuild.session.schroot import SchrootSession
+
+
+def run_apt(session: Session, args: List[str]) -> None:
+    args = ['apt', '-y'] + args
+    retcode, lines = run_with_tee(session, args, cwd='/', user='root')
+    if retcode == 0:
+        return
+    offset, line, error = find_apt_get_failure(lines)
+    if error is not None:
+        raise DetailedDistCommandFailed(retcode, args, error)
+    if line is not None:
+        raise UnidentifiedError(
+            retcode, args, lines, secondary=(offset, line))
+    raise UnidentifiedError(retcode, args, lines)
+
+
+def apt_install(session: Session, packages: List[str]) -> None:
+    run_apt(session, ['install'] + packages)
+
+
+def apt_satisfy(session: Session, deps: List[str]) -> None:
+    run_apt(session, ['satisfy'] + deps)
+
+
+def satisfy_build_deps(session: Session, tree):
+    source = Deb822(tree.get_file('debian/control'))
+    deps = []
+    for name in ['Build-Depends', 'Build-Depends-Indep', 'Build-Depends-Arch']:
+        try:
+            deps.append(source[name].strip().strip(','))
+        except KeyError:
+            pass
+    for name in ['Build-Conflicts', 'Build-Conflicts-Indeo',
+                 'Build-Conflicts-Arch']:
+        try:
+            deps.append('Conflicts: ' + source[name])
+        except KeyError:
+            pass
+    deps = [
+        dep.strip().strip(',')
+        for dep in deps]
+    apt_satisfy(session, deps)
+
+
+def run_with_tee(session: Session, args: List[str], **kwargs):
+    p = session.Popen(
+        args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
+    contents = []
+    while p.poll() is None:
+        line = p.stdout.readline()
+        sys.stdout.buffer.write(line)
+        sys.stdout.buffer.flush()
+        contents.append(line.decode('utf-8', 'surrogateescape'))
+    return p.returncode, contents
+
+
+class SchrootDependencyContext(DependencyContext):
+
+    def __init__(self, session):
+        self.session = session
+
+    def add_dependency(self, package, minimum_version=None):
+        # TODO(jelmer): Handle minimum_version
+        apt_install(self.session, [package])
+        return True
+
+
+class DetailedDistCommandFailed(Exception):
+
+    def __init__(self, retcode, argv, error):
+        self.retcode = retcode
+        self.argv = argv
+        self.error = error
+
+
+class UnidentifiedError(Exception):
+
+    def __init__(self, retcode, argv, lines, secondary=None):
+        self.retcode = retcode
+        self.argv = argv
+        self.lines = lines
+        self.secondary = secondary
+
+
+def fix_perl_module_from_cpan(error, context):
+    # TODO(jelmer): Specify -T to skip tests?
+    context.session.check_call(
+        ['cpan', '-i', error.module], user='root',
+        env={'PERL_MM_USE_DEFAULT': '1'})
+    return True
+
+
+NPM_COMMAND_PACKAGES = {
+    'del-cli': 'del-cli',
+    }
+
+
+def fix_npm_missing_command(error, context):
+    try:
+        package = NPM_COMMAND_PACKAGES[error.command]
+    except KeyError:
+        return False
+
+    context.session.check_call(['npm', '-g', 'install', package])
+    return True
+
+
+GENERIC_INSTALL_FIXERS: List[
+        Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [
+    (MissingPerlModule, fix_perl_module_from_cpan),
+    (MissingCommand, fix_npm_missing_command),
+]
+
+
+def run_with_build_fixer(session: Session, args: List[str]):
+    logging.info('Running %r', args)
+    fixed_errors = []
+    while True:
+        retcode, lines = run_with_tee(session, args)
+        if retcode == 0:
+            return
+        offset, line, error = find_build_failure_description(lines)
+        if error is None:
+            logging.warning('Build failed with unidentified error. Giving up.')
+            if line is not None:
+                raise UnidentifiedError(
+                    retcode, args, lines, secondary=(offset, line))
+            raise UnidentifiedError(retcode, args, lines)
+
+        logging.info('Identified error: %r', error)
+        if error in fixed_errors:
+            logging.warning(
+                'Failed to resolve error %r, it persisted. Giving up.',
+                error)
+            raise DetailedDistCommandFailed(retcode, args, error)
+        if not resolve_error(
+                error, SchrootDependencyContext(session),
+                fixers=(APT_FIXERS + GENERIC_INSTALL_FIXERS)):
+            logging.warning(
+                'Failed to find resolution for error %r. Giving up.',
+                error)
+            raise DetailedDistCommandFailed(retcode, args, error)
+        fixed_errors.append(error)
+
+
+class NoBuildToolsFound(Exception):
+    """No supported build tools were found."""
+
+
+def run_dist_in_chroot(session):
+    apt_install(session, ['git'])
+
+    # Some things want to write to the user's home directory,
+    # e.g. pip caches in ~/.cache
+    session.create_home()
+
+    if os.path.exists('package.xml'):
+        apt_install(session, ['php-pear', 'php-horde-core'])
+        logging.info('Found package.xml, assuming pear package.')
+        session.check_call(['pear', 'package'])
+        return
+
+    if os.path.exists('pyproject.toml'):
+        import toml
+        with open('pyproject.toml', 'r') as pf:
+            pyproject = toml.load(pf)
+        if 'poetry' in pyproject.get('tool', []):
+            logging.info(
+                'Found pyproject.toml with poetry section, '
+                'assuming poetry project.')
+            apt_install(session, ['python3-venv', 'python3-pip'])
+            session.check_call(['pip3', 'install', 'poetry'], user='root')
+            session.check_call(['poetry', 'build', '-f', 'sdist'])
+            return
+
+    if os.path.exists('setup.py'):
+        logging.info('Found setup.py, assuming python project.')
+        apt_install(session, ['python3', 'python3-pip'])
+        with open('setup.py', 'r') as f:
+            setup_py_contents = f.read()
+        try:
+            with open('setup.cfg', 'r') as f:
+                setup_cfg_contents = f.read()
+        except FileNotFoundError:
+            setup_cfg_contents = ''
+        if 'setuptools' in setup_py_contents:
+            logging.info('Reference to setuptools found, installing.')
+            apt_install(session, ['python3-setuptools'])
+        if ('setuptools_scm' in setup_py_contents or
+                'setuptools_scm' in setup_cfg_contents):
+            logging.info('Reference to setuptools-scm found, installing.')
+            apt_install(
+                session, ['python3-setuptools-scm', 'git', 'mercurial'])
+
+        # TODO(jelmer): Install setup_requires
+
+        interpreter = shebang_binary('setup.py')
+        if interpreter is not None:
+            if interpreter == 'python3':
+                apt_install(session, ['python3'])
+            elif interpreter == 'python2':
+                apt_install(session, ['python2'])
+            elif interpreter == 'python':
+                apt_install(session, ['python'])
+            else:
+                raise ValueError('Unknown interpreter %s' % interpreter)
+            apt_install(session, ['python2', 'python3'])
+            run_with_build_fixer(session, ['./setup.py', 'sdist'])
+        else:
+            # Just assume it's Python 3
+            apt_install(session, ['python3'])
+            run_with_build_fixer(session, ['python3', './setup.py', 'sdist'])
+        return
+
+    if os.path.exists('setup.cfg'):
+        logging.info('Found setup.cfg, assuming python project.')
+        apt_install(session, ['python3-pep517', 'python3-pip'])
+        session.check_call(['python3', '-m', 'pep517.build', '-s', '.'])
+        return
+
+    if os.path.exists('dist.ini') and not os.path.exists('Makefile.PL'):
+        apt_install(session, ['libdist-inkt-perl'])
+        with open('dist.ini', 'rb') as f:
+            for line in f:
+                if not line.startswith(b';;'):
+                    continue
+                try:
+                    (key, value) = line[2:].split(b'=', 1)
+                except ValueError:
+                    continue
+                if (key.strip() == b'class' and
+                        value.strip().startswith(b"'Dist::Inkt")):
+                    logging.info(
+                        'Found Dist::Inkt section in dist.ini, '
+                        'assuming distinkt.')
+                    # TODO(jelmer): install via apt if possible
+                    session.check_call(
+                        ['cpan', 'install', value.decode().strip("'")],
+                        user='root')
+                    run_with_build_fixer(session, ['distinkt-dist'])
+                    return
+        # Default to invoking Dist::Zilla
+        logging.info('Found dist.ini, assuming dist-zilla.')
+        apt_install(session, ['libdist-zilla-perl'])
+        run_with_build_fixer(session, ['dzil', 'build', '--in', '..'])
+        return
+
+    if os.path.exists('package.json'):
+        apt_install(session, ['npm'])
+        run_with_build_fixer(session, ['npm', 'pack'])
+        return
+
+    gemfiles = [name for name in os.listdir('.') if name.endswith('.gem')]
+    if gemfiles:
+        apt_install(session, ['gem2deb'])
+        if len(gemfiles) > 1:
+            logging.warning('More than one gemfile. Trying the first?')
+        run_with_build_fixer(session, ['gem2tgz', gemfiles[0]])
+        return
+
+    if os.path.exists('waf'):
+        apt_install(session, ['python3'])
+        run_with_build_fixer(session, ['./waf', 'dist'])
+        return
+
+    if os.path.exists('Makefile.PL') and not os.path.exists('Makefile'):
+        apt_install(session, ['perl'])
+        run_with_build_fixer(session, ['perl', 'Makefile.PL'])
+
+    if not os.path.exists('Makefile') and not os.path.exists('configure'):
+        if os.path.exists('autogen.sh'):
+            if shebang_binary('autogen.sh') is None:
+                run_with_build_fixer(session, ['/bin/sh', './autogen.sh'])
+            try:
+                run_with_build_fixer(session, ['./autogen.sh'])
+            except UnidentifiedError as e:
+                if ("Gnulib not yet bootstrapped; "
+                        "run ./bootstrap instead.\n" in e.lines):
+                    run_with_build_fixer(session, ["./bootstrap"])
+                    run_with_build_fixer(session, ['./autogen.sh'])
+                else:
+                    raise
+
+        elif os.path.exists('configure.ac') or os.path.exists('configure.in'):
+            apt_install(session, [
+                'autoconf', 'automake', 'gettext', 'libtool', 'gnu-standards'])
+            run_with_build_fixer(session, ['autoreconf', '-i'])
+
+    if not os.path.exists('Makefile') and os.path.exists('configure'):
+        session.check_call(['./configure'])
+
+    if os.path.exists('Makefile'):
+        apt_install(session, ['make'])
+        try:
+            run_with_build_fixer(session, ['make', 'dist'])
+        except UnidentifiedError as e:
+            if "make: *** No rule to make target 'dist'.  Stop.\n" in e.lines:
+                pass
+            elif ("make[1]: *** No rule to make target 'dist'. Stop.\n"
+                    in e.lines):
+                pass
+            elif ("Reconfigure the source tree "
+                    "(via './config' or 'perl Configure'), please.\n"
+                  ) in e.lines:
+                run_with_build_fixer(session, ['./config'])
+                run_with_build_fixer(session, ['make', 'dist'])
+            elif (
+                    "Please try running 'make manifest' and then run "
+                    "'make dist' again.\n" in e.lines):
+                run_with_build_fixer(session, ['make', 'manifest'])
+                run_with_build_fixer(session, ['make', 'dist'])
+            elif "Please run ./configure first\n" in e.lines:
+                run_with_build_fixer(session, ['./configure'])
+                run_with_build_fixer(session, ['make', 'dist'])
+            elif any([re.match(
+                    r'Makefile:[0-9]+: \*\*\* Missing \'Make.inc\' '
+                    r'Run \'./configure \[options\]\' and retry.  Stop.\n',
+                    line) for line in e.lines]):
+                run_with_build_fixer(session, ['./configure'])
+                run_with_build_fixer(session, ['make', 'dist'])
+            elif any([re.match(
+                      r'Problem opening MANIFEST: No such file or directory '
+                      r'at .* line [0-9]+\.', line) for line in e.lines]):
+                run_with_build_fixer(session, ['make', 'manifest'])
+                run_with_build_fixer(session, ['make', 'dist'])
+            else:
+                raise
+        else:
+            return
+
+    raise NoBuildToolsFound()
+
+
+def export_vcs_tree(tree, directory):
+    try:
+        export(tree, directory, 'dir', None)
+    except OSError as e:
+        if e.errno == errno.ENOSPC:
+            raise DetailedDistCommandFailed(
+                1, ['export'], NoSpaceOnDevice())
+        raise
+
+
+def dupe_vcs_tree(tree, directory):
+    with tree.lock_read():
+        if isinstance(tree, WorkingTree):
+            tree = tree.basis_tree()
+    try:
+        result = tree._repository.controldir.sprout(
+            directory, create_tree_if_local=True,
+            revision_id=tree.get_revision_id())
+    except OSError as e:
+        if e.errno == errno.ENOSPC:
+            raise DetailedDistCommandFailed(
+                1, ['sprout'], NoSpaceOnDevice())
+        raise
+    # Copy parent location - some scripts need this
+    base_branch = tree._repository.controldir.open_branch()
+    parent = base_branch.get_parent()
+    if parent:
+        result.open_branch().set_parent(parent)
+
+
+def create_dist_schroot(
+        tree: Tree, target_dir: str,
+        chroot: str, packaging_tree: Optional[Tree] = None,
+        include_controldir: bool = True,
+        subdir: Optional[str] = None) -> Optional[str]:
+    if subdir is None:
+        subdir = 'package'
+    with SchrootSession(chroot) as session:
+        if packaging_tree is not None:
+            satisfy_build_deps(session, packaging_tree)
+        build_dir = os.path.join(session.location, 'build')
+
+        try:
+            directory = tempfile.mkdtemp(dir=build_dir)
+        except OSError as e:
+            if e.errno == errno.ENOSPC:
+                raise DetailedDistCommandFailed(
+                    1, ['mkdtemp'], NoSpaceOnDevice())
+        reldir = '/' + os.path.relpath(directory, session.location)
+
+        export_directory = os.path.join(directory, subdir)
+        if not include_controldir:
+            export_vcs_tree(tree, export_directory)
+        else:
+            dupe_vcs_tree(tree, export_directory)
+
+        existing_files = os.listdir(export_directory)
+
+        oldcwd = os.getcwd()
+        os.chdir(export_directory)
+        try:
+            session.chdir(os.path.join(reldir, subdir))
+            run_dist_in_chroot(session)
+        except NoBuildToolsFound:
+            logging.info(
+                'No build tools found, falling back to simple export.')
+            return None
+        finally:
+            os.chdir(oldcwd)
+
+        new_files = os.listdir(export_directory)
+        diff_files = set(new_files) - set(existing_files)
+        diff = set([n for n in diff_files if get_filetype(n) is not None])
+        if len(diff) == 1:
+            fn = diff.pop()
+            logging.info('Found tarball %s in package directory.', fn)
+            shutil.copy(
+                os.path.join(export_directory, fn),
+                target_dir)
+            return fn
+        if 'dist' in diff_files:
+            for entry in os.scandir(os.path.join(export_directory, 'dist')):
+                if get_filetype(entry.name) is not None:
+                    logging.info(
+                        'Found tarball %s in dist directory.', entry.name)
+                    shutil.copy(entry.path, target_dir)
+                    return entry.name
+            logging.info('No tarballs found in dist directory.')
+
+        diff = set(os.listdir(directory)) - set([subdir])
+        if len(diff) == 1:
+            fn = diff.pop()
+            logging.info('Found tarball %s in parent directory.', fn)
+            shutil.copy(
+                os.path.join(directory, fn),
+                target_dir)
+            return fn
+
+        logging.info('No tarball created :(')
+        return None
+
+
+if __name__ == '__main__':
+    import argparse
+    import breezy.bzr
+    import breezy.git  # noqa: F401
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--chroot', default='unstable-amd64-sbuild', type=str,
+        help='Name of chroot to use')
+    parser.add_argument(
+        'directory', default='.', type=str, nargs='?',
+        help='Directory with upstream source.')
+    parser.add_argument(
+        '--packaging-directory', type=str,
+        help='Path to packaging directory.')
+    parser.add_argument(
+        '--target-directory', type=str, default='..',
+        help='Target directory')
+    args = parser.parse_args()
+    tree = WorkingTree.open(args.directory)
+    if args.packaging_directory:
+        packaging_tree = WorkingTree.open(args.packaging_directory)
+        with packaging_tree.lock_read():
+            source = Deb822(packaging_tree.get_file('debian/control'))
+        package = source['Source']
+        subdir = package
+    else:
+        packaging_tree = None
+        subdir = None
+
+    ret = create_dist_schroot(
+        tree, subdir=subdir, target_dir=os.path.abspath(args.target_directory),
+        packaging_tree=packaging_tree,
+        chroot=args.chroot)
+    if ret:
+        sys.exit(0)
+    else:
+        sys.exit(1)

ognibuild/tests/__init__.py

@@ -22,6 +22,8 @@ import unittest
 
 def test_suite():
     names = [
+        'debian_build',
+        'debian_fix_build',
     ]
     module_names = ['ognibuild.tests.test_' + name for name in names]
     loader = unittest.TestLoader()

ognibuild/tests/test_debian_build.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import datetime
+from ..debian.build import add_dummy_changelog_entry
+
+from breezy.tests import TestCaseWithTransport
+
+
+class AddDummyChangelogEntryTests(TestCaseWithTransport):
+
+    def test_simple(self):
+        tree = self.make_branch_and_tree('.')
+        self.build_tree_contents([('debian/', ), ('debian/changelog', """\
+janitor (0.1-1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""")])
+        tree.add(['debian', 'debian/changelog'])
+        add_dummy_changelog_entry(
+            tree, '', 'jan+some', 'some-fixes', 'Dummy build.',
+            timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654),
+            maintainer=("Jelmer Vernooij", "jelmer@debian.org"))
+        self.assertFileEqual("""\
+janitor (0.1-1jan+some1) some-fixes; urgency=low
+
+  * Dummy build.
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 05 Sep 2020 12:35:04 -0000
+
+janitor (0.1-1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""", 'debian/changelog')
+
+    def test_native(self):
+        tree = self.make_branch_and_tree('.')
+        self.build_tree_contents([('debian/', ), ('debian/changelog', """\
+janitor (0.1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""")])
+        tree.add(['debian', 'debian/changelog'])
+        add_dummy_changelog_entry(
+            tree, '', 'jan+some', 'some-fixes', 'Dummy build.',
+            timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654),
+            maintainer=("Jelmer Vernooij", "jelmer@debian.org"))
+        self.assertFileEqual("""\
+janitor (0.1jan+some1) some-fixes; urgency=low
+
+  * Dummy build.
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 05 Sep 2020 12:35:04 -0000
+
+janitor (0.1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""", 'debian/changelog')
+
+    def test_exists(self):
+        tree = self.make_branch_and_tree('.')
+        self.build_tree_contents([('debian/', ), ('debian/changelog', """\
+janitor (0.1-1jan+some1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""")])
+        tree.add(['debian', 'debian/changelog'])
+        add_dummy_changelog_entry(
+            tree, '', 'jan+some', 'some-fixes', 'Dummy build.',
+            timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654),
+            maintainer=("Jelmer Vernooij", "jelmer@debian.org"))
+        self.assertFileEqual("""\
+janitor (0.1-1jan+some2) some-fixes; urgency=low
+
+  * Dummy build.
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 05 Sep 2020 12:35:04 -0000
+
+janitor (0.1-1jan+some1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""", 'debian/changelog')

ognibuild/tests/test_debian_fix_build.py

@@ -0,0 +1,201 @@
+#!/usr/bin/python
+# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+import re
+
+from debian.deb822 import Deb822
+
+from buildlog_consultant.sbuild import (
+    MissingCommand,
+    MissingGoPackage,
+    MissingPerlModule,
+    MissingPkgConfig,
+    MissingPythonModule,
+    MissingRubyFile,
+    MissingRubyGem,
+    MissingValaPackage,
+    )
+from ..debian import fix_build
+from ..debian.fix_build import (
+    resolve_error,
+    VERSIONED_PACKAGE_FIXERS,
+    APT_FIXERS,
+    BuildDependencyContext,
+    )
+from breezy.tests import TestCaseWithTransport
+
+
+class ResolveErrorTests(TestCaseWithTransport):
+
+    def setUp(self):
+        super(ResolveErrorTests, self).setUp()
+        self.tree = self.make_branch_and_tree('.')
+        self.build_tree_contents([('debian/', ), ('debian/control', """\
+Source: blah
+Build-Depends: libc6
+
+Package: python-blah
+Depends: ${python3:Depends}
+Description: A python package
+ Foo
+"""), ('debian/changelog', """\
+blah (0.1) UNRELEASED; urgency=medium
+
+  * Initial release. (Closes: #XXXXXX)
+
+ -- Jelmer Vernooij <jelmer@debian.org>  Sat, 04 Apr 2020 14:12:13 +0000
+""")])
+        self.tree.add(['debian', 'debian/control', 'debian/changelog'])
+        self.tree.commit('Initial commit')
+        self.overrideAttr(fix_build, 'search_apt_file', self._search_apt_file)
+        self._apt_files = {}
+
+    def _search_apt_file(self, path, regex=False):
+        for p, pkg in sorted(self._apt_files.items()):
+            if regex:
+                if re.match(path, p):
+                    yield pkg
+            else:
+                if path == p:
+                    yield pkg
+
+    def resolve(self, error, context=('build', )):
+        context = BuildDependencyContext(
+            self.tree, subpath='', committer='Janitor <janitor@jelmer.uk>',
+            update_changelog=True)
+        return resolve_error(
+            error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS)
+
+    def get_build_deps(self):
+        with open(self.tree.abspath('debian/control'), 'r') as f:
+            return next(Deb822.iter_paragraphs(f)).get('Build-Depends', '')
+
+    def test_missing_command_unknown(self):
+        self._apt_files = {}
+        self.assertFalse(self.resolve(
+            MissingCommand('acommandthatdoesnotexist')))
+
+    def test_missing_command_brz(self):
+        self._apt_files = {
+            '/usr/bin/b': 'bash',
+            '/usr/bin/brz': 'brz',
+            '/usr/bin/brzier': 'bash',
+            }
+        self.assertTrue(self.resolve(MissingCommand('brz')))
+        self.assertEqual('libc6, brz', self.get_build_deps())
+        rev = self.tree.branch.repository.get_revision(
+            self.tree.branch.last_revision())
+        self.assertEqual(
+            'Add missing build dependency on brz.\n',
+            rev.message)
+        self.assertFalse(self.resolve(MissingCommand('brz')))
+        self.assertEqual('libc6, brz', self.get_build_deps())
+
+    def test_missing_command_ps(self):
+        self._apt_files = {
+            '/bin/ps': 'procps',
+            '/usr/bin/pscal': 'xcal',
+        }
+        self.assertTrue(self.resolve(MissingCommand('ps')))
+        self.assertEqual('libc6, procps', self.get_build_deps())
+
+    def test_missing_ruby_file(self):
+        self._apt_files = {
+            '/usr/lib/ruby/vendor_ruby/rake/testtask.rb': 'rake',
+            }
+        self.assertTrue(self.resolve(MissingRubyFile('rake/testtask')))
+        self.assertEqual('libc6, rake', self.get_build_deps())
+
+    def test_missing_ruby_file_from_gem(self):
+        self._apt_files = {
+            '/usr/share/rubygems-integration/all/gems/activesupport-'
+            '5.2.3/lib/active_support/core_ext/string/strip.rb':
+            'ruby-activesupport'}
+        self.assertTrue(self.resolve(
+            MissingRubyFile('active_support/core_ext/string/strip')))
+        self.assertEqual('libc6, ruby-activesupport', self.get_build_deps())
+
+    def test_missing_ruby_gem(self):
+        self._apt_files = {
+            '/usr/share/rubygems-integration/all/specifications/'
+            'bio-1.5.2.gemspec': 'ruby-bio',
+            '/usr/share/rubygems-integration/all/specifications/'
+            'bio-2.0.2.gemspec': 'ruby-bio',
+            }
+        self.assertTrue(self.resolve(MissingRubyGem('bio', None)))
+        self.assertEqual('libc6, ruby-bio', self.get_build_deps())
+        self.assertTrue(self.resolve(MissingRubyGem('bio', '2.0.3')))
+        self.assertEqual('libc6, ruby-bio (>= 2.0.3)', self.get_build_deps())
+
+    def test_missing_perl_module(self):
+        self._apt_files = {
+            '/usr/share/perl5/App/cpanminus/fatscript.pm': 'cpanminus'}
+        self.assertTrue(self.resolve(MissingPerlModule(
+            'App/cpanminus/fatscript.pm', 'App::cpanminus::fatscript', [
+                '/<<PKGBUILDDIR>>/blib/lib',
+                '/<<PKGBUILDDIR>>/blib/arch',
+                '/etc/perl',
+                '/usr/local/lib/x86_64-linux-gnu/perl/5.30.0',
+                '/usr/local/share/perl/5.30.0',
+                '/usr/lib/x86_64-linux-gnu/perl5/5.30',
+                '/usr/share/perl5',
+                '/usr/lib/x86_64-linux-gnu/perl/5.30',
+                '/usr/share/perl/5.30',
+                '/usr/local/lib/site_perl',
+                '/usr/lib/x86_64-linux-gnu/perl-base',
+                '.'])))
+        self.assertEqual('libc6, cpanminus', self.get_build_deps())
+
+    def test_missing_pkg_config(self):
+        self._apt_files = {
+            '/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc':
+            'libxcb-xfixes0-dev'}
+        self.assertTrue(self.resolve(MissingPkgConfig('xcb-xfixes')))
+        self.assertEqual('libc6, libxcb-xfixes0-dev', self.get_build_deps())
+
+    def test_missing_pkg_config_versioned(self):
+        self._apt_files = {
+            '/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc':
+            'libxcb-xfixes0-dev'}
+        self.assertTrue(self.resolve(MissingPkgConfig('xcb-xfixes', '1.0')))
+        self.assertEqual(
+            'libc6, libxcb-xfixes0-dev (>= 1.0)', self.get_build_deps())
+
+    def test_missing_python_module(self):
+        self._apt_files = {
+            '/usr/lib/python3/dist-packages/m2r.py': 'python3-m2r'
+            }
+        self.assertTrue(self.resolve(MissingPythonModule('m2r')))
+        self.assertEqual('libc6, python3-m2r', self.get_build_deps())
+
+    def test_missing_go_package(self):
+        self._apt_files = {
+            '/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go':
+                'golang-github-chzyer-readline-dev',
+            }
+        self.assertTrue(self.resolve(
+            MissingGoPackage('github.com/chzyer/readline')))
+        self.assertEqual(
+            'libc6, golang-github-chzyer-readline-dev',
+            self.get_build_deps())
+
+    def test_missing_vala_package(self):
+        self._apt_files = {
+            '/usr/share/vala-0.48/vapi/posix.vapi': 'valac-0.48-vapi',
+            }
+        self.assertTrue(self.resolve(MissingValaPackage('posix')))
+        self.assertEqual('libc6, valac-0.48-vapi', self.get_build_deps())

setup.cfg

@@ -1,5 +1,6 @@
 [flake8]
 application-package-names = ognibuild
+banned-modules = silver-platter = Should not use silver-platter
 
 [mypy]
 # A number of ognibuilds' dependencies don't have type hints yet

setup.py

@@ -23,5 +23,10 @@ setup(name="ognibuild",
       ],
       entry_points={
         "console_scripts": [
-            "ogni=ognibuild.__main__:main"]
-      })
+            "ogni=ognibuild.__main__:main",
+            "deb-fix-build=ognibuild.debian.fix_build:main",
+        ]
+      },
+      install_requires=['breezy', 'buildlog-consultant'],
+      test_suite='ognibuild.tests.test_suite',
+      )