From 43dfe81f66998c1b89d9065fc0fbc72885b27a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 10 Feb 2021 02:19:06 +0000 Subject: [PATCH] Fix formatting. --- .flake8 | 5 + ognibuild/__init__.py | 15 +- ognibuild/__main__.py | 32 +- ognibuild/apt.py | 26 +- ognibuild/buildsystem.py | 303 ++++----- ognibuild/debian/__init__.py | 15 +- ognibuild/debian/build.py | 167 +++-- ognibuild/debian/fix_build.py | 783 +++++++++++++---------- ognibuild/dist.py | 79 +-- ognibuild/fix_build.py | 41 +- ognibuild/session/__init__.py | 37 +- ognibuild/session/plain.py | 5 +- ognibuild/session/schroot.py | 111 ++-- ognibuild/tests/__init__.py | 6 +- ognibuild/tests/test_debian_build.py | 94 ++- ognibuild/tests/test_debian_fix_build.py | 206 +++--- ognibuild/vcs.py | 14 +- 17 files changed, 1085 insertions(+), 854 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bde4be3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +extend-ignore = E203, E266, E501, W293, W291 +max-line-length = 88 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 3822fbe..f9e1a36 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -21,11 +21,10 @@ import stat import sys -DEFAULT_PYTHON = 'python3' +DEFAULT_PYTHON = "python3" class DetailedFailure(Exception): - def __init__(self, retcode, argv, error): self.retcode = retcode self.argv = argv @@ -35,19 +34,19 @@ class DetailedFailure(Exception): def shebang_binary(p): if not (os.stat(p).st_mode & stat.S_IEXEC): return None - with open(p, 'rb') as f: + with open(p, "rb") as f: firstline = f.readline() - if not firstline.startswith(b'#!'): + if not firstline.startswith(b"#!"): return None - args = firstline[2:].split(b' ') - if args[0] in (b'/usr/bin/env', b'env'): + args = firstline[2:].split(b" ") + if args[0] in (b"/usr/bin/env", b"env"): return os.path.basename(args[1].decode()).strip() return os.path.basename(args[0].decode()).strip() def note(m): - sys.stdout.write('%s\n' % m) + sys.stdout.write("%s\n" % m) def warning(m): - sys.stderr.write('WARNING: %s\n' % m) + sys.stderr.write("WARNING: %s\n" % m) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 32a1739..c2b3939 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -28,40 +28,44 @@ from .test import run_test def main(): import argparse + parser = argparse.ArgumentParser() parser.add_argument( - 'subcommand', type=str, - choices=['dist', 'build', 'clean', 'test', 'install']) + "subcommand", type=str, choices=["dist", "build", "clean", "test", "install"] + ) parser.add_argument( - '--directory', '-d', type=str, help='Directory for project.', - default='.') + "--directory", "-d", type=str, help="Directory for project.", default="." + ) + parser.add_argument("--schroot", type=str, help="schroot to run in.") parser.add_argument( - '--schroot', type=str, help='schroot to run in.') - parser.add_argument( - '--resolve', choices=['explain', 'apt', 'native'], - help='What to do about missing dependencies') + "--resolve", + choices=["explain", "apt", "native"], + help="What to do about missing dependencies", + ) args = parser.parse_args() if args.schroot: from .session.schroot import SchrootSession + session = SchrootSession(args.schroot) else: from .session.plain import PlainSession + session = PlainSession() with session: os.chdir(args.directory) try: - if args.subcommand == 'dist': + if args.subcommand == "dist": run_dist(session) - if args.subcommand == 'build': + if args.subcommand == "build": run_build(session) - if args.subcommand == 'clean': + if args.subcommand == "clean": run_clean(session) - if args.subcommand == 'install': + if args.subcommand == "install": run_install(session) - if args.subcommand == 'test': + if args.subcommand == "test": run_test(session) except NoBuildToolsFound: - note('No build tools found.') + note("No build tools found.") return 1 return 0 diff --git a/ognibuild/apt.py b/ognibuild/apt.py index acd52b4..3f849c2 100644 --- a/ognibuild/apt.py +++ b/ognibuild/apt.py @@ -23,14 +23,13 @@ import apt_pkg import os from buildlog_consultant.apt import ( find_apt_get_failure, - ) +) from . import DetailedFailure from .session import Session, run_with_tee class UnidentifiedError(Exception): - def __init__(self, retcode, argv, lines, secondary=None): self.retcode = retcode self.argv = argv @@ -40,17 +39,16 @@ class UnidentifiedError(Exception): def run_apt(session: Session, args: List[str]) -> None: """Run apt.""" - args = ['apt', '-y'] + args - retcode, lines = run_with_tee(session, args, cwd='/', user='root') + args = ["apt", "-y"] + args + retcode, lines = run_with_tee(session, args, cwd="/", user="root") if retcode == 0: return offset, line, error = find_apt_get_failure(lines) if error is not None: raise DetailedFailure(retcode, args, error) if line is not None: - raise UnidentifiedError( - retcode, args, lines, secondary=(offset, line)) - while lines and lines[-1] == '': + raise UnidentifiedError(retcode, args, lines, secondary=(offset, line)) + while lines and lines[-1] == "": lines.pop(-1) raise UnidentifiedError(retcode, args, lines) @@ -63,23 +61,23 @@ class AptManager(object): self.session = session def missing(self, packages): - root = getattr(self.session, 'location', '/') - status_path = os.path.join(root, 'var/lib/dpkg/status') + root = getattr(self.session, "location", "/") + status_path = os.path.join(root, "var/lib/dpkg/status") missing = set(packages) with apt_pkg.TagFile(status_path) as tagf: while missing: tagf.step() if not tagf.section: break - if tagf.section['Package'] in missing: - if tagf.section['Status'] == 'install ok installed': - missing.remove(tagf.section['Package']) + if tagf.section["Package"] in missing: + if tagf.section["Status"] == "install ok installed": + missing.remove(tagf.section["Package"]) return list(missing) def install(self, packages: List[str]) -> None: packages = self.missing(packages) if packages: - run_apt(self.session, ['install'] + packages) + run_apt(self.session, ["install"] + packages) def satisfy(self, deps: List[str]) -> None: - run_apt(self.session, ['satisfy'] + deps) + run_apt(self.session, ["satisfy"] + deps) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 784376b..998803e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -52,22 +52,21 @@ class BuildSystem(object): class Pear(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['php-pear']) + apt.install(["php-pear"]) def dist(self): self.setup() - run_with_build_fixer(self.session, ['pear', 'package']) + run_with_build_fixer(self.session, ["pear", "package"]) def test(self): self.setup() - run_with_build_fixer(self.session, ['pear', 'run-tests']) + run_with_build_fixer(self.session, ["pear", "run-tests"]) def build(self): self.setup() - run_with_build_fixer(self.session, ['pear', 'build']) + run_with_build_fixer(self.session, ["pear", "build"]) def clean(self): self.setup() @@ -75,242 +74,251 @@ class Pear(BuildSystem): def install(self): self.setup() - run_with_build_fixer(self.session, ['pear', 'install']) + run_with_build_fixer(self.session, ["pear", "install"]) class SetupPy(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['python3', 'python3-pip']) - with open('setup.py', 'r') as f: + apt.install(["python3", "python3-pip"]) + with open("setup.py", "r") as f: setup_py_contents = f.read() try: - with open('setup.cfg', 'r') as f: + with open("setup.cfg", "r") as f: setup_cfg_contents = f.read() except FileNotFoundError: - setup_cfg_contents = '' - if 'setuptools' in setup_py_contents: - logging.info('Reference to setuptools found, installing.') - apt.install(['python3-setuptools']) - if ('setuptools_scm' in setup_py_contents or - 'setuptools_scm' in setup_cfg_contents): - logging.info('Reference to setuptools-scm found, installing.') - apt.install(['python3-setuptools-scm', 'git', 'mercurial']) + setup_cfg_contents = "" + if "setuptools" in setup_py_contents: + logging.info("Reference to setuptools found, installing.") + apt.install(["python3-setuptools"]) + if ( + "setuptools_scm" in setup_py_contents + or "setuptools_scm" in setup_cfg_contents + ): + logging.info("Reference to setuptools-scm found, installing.") + apt.install(["python3-setuptools-scm", "git", "mercurial"]) # TODO(jelmer): Install setup_requires def test(self): self.setup() - self._run_setup(['test']) + self._run_setup(["test"]) def dist(self): self.setup() - self._run_setup(['sdist']) + self._run_setup(["sdist"]) def clean(self): self.setup() - self._run_setup(['clean']) + self._run_setup(["clean"]) def install(self): self.setup() - self._run_setup(['install']) + self._run_setup(["install"]) def _run_setup(self, args): apt = AptManager(self.session) - interpreter = shebang_binary('setup.py') + interpreter = shebang_binary("setup.py") if interpreter is not None: - if interpreter == 'python3': - apt.install(['python3']) - elif interpreter == 'python2': - apt.install(['python2']) - elif interpreter == 'python': - apt.install(['python']) + if interpreter == "python3": + apt.install(["python3"]) + elif interpreter == "python2": + apt.install(["python2"]) + elif interpreter == "python": + apt.install(["python"]) else: - raise ValueError('Unknown interpreter %r' % interpreter) - apt.install(['python2', 'python3']) - run_with_build_fixer( - self.session, ['./setup.py'] + args) + raise ValueError("Unknown interpreter %r" % interpreter) + apt.install(["python2", "python3"]) + run_with_build_fixer(self.session, ["./setup.py"] + args) else: # Just assume it's Python 3 - apt.install(['python3']) - run_with_build_fixer( - self.session, ['python3', './setup.py'] + args) + apt.install(["python3"]) + run_with_build_fixer(self.session, ["python3", "./setup.py"] + args) class PyProject(BuildSystem): - def load_toml(self): import toml - with open('pyproject.toml', 'r') as pf: + + with open("pyproject.toml", "r") as pf: return toml.load(pf) def dist(self): apt = AptManager(self.session) pyproject = self.load_toml() - if 'poetry' in pyproject.get('tool', []): + if "poetry" in pyproject.get("tool", []): logging.info( - 'Found pyproject.toml with poetry section, ' - 'assuming poetry project.') - apt.install(['python3-venv', 'python3-pip']) - self.session.check_call(['pip3', 'install', 'poetry'], user='root') - self.session.check_call(['poetry', 'build', '-f', 'sdist']) + "Found pyproject.toml with poetry section, " "assuming poetry project." + ) + apt.install(["python3-venv", "python3-pip"]) + self.session.check_call(["pip3", "install", "poetry"], user="root") + self.session.check_call(["poetry", "build", "-f", "sdist"]) return - raise AssertionError('no supported section in pyproject.toml') + raise AssertionError("no supported section in pyproject.toml") class SetupCfg(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['python3-pep517', 'python3-pip']) + apt.install(["python3-pep517", "python3-pip"]) def dist(self): - self.session.check_call(['python3', '-m', 'pep517.build', '-s', '.']) + self.session.check_call(["python3", "-m", "pep517.build", "-s", "."]) class NpmPackage(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['npm']) + apt.install(["npm"]) def dist(self): self.setup() - run_with_build_fixer(self.session, ['npm', 'pack']) + run_with_build_fixer(self.session, ["npm", "pack"]) class Waf(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['python3']) + apt.install(["python3"]) def dist(self): self.setup() - run_with_build_fixer(self.session, ['./waf', 'dist']) + run_with_build_fixer(self.session, ["./waf", "dist"]) class Gem(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['gem2deb']) + apt.install(["gem2deb"]) def dist(self): self.setup() - gemfiles = [entry.name for entry in self.session.scandir('.') - if entry.name.endswith('.gem')] + gemfiles = [ + entry.name + for entry in self.session.scandir(".") + if entry.name.endswith(".gem") + ] if len(gemfiles) > 1: - logging.warning('More than one gemfile. Trying the first?') - run_with_build_fixer(self.session, ['gem2tgz', gemfiles[0]]) + logging.warning("More than one gemfile. Trying the first?") + run_with_build_fixer(self.session, ["gem2tgz", gemfiles[0]]) class DistInkt(BuildSystem): - def setup(self): apt = AptManager(self.session) - apt.install(['libdist-inkt-perl']) + apt.install(["libdist-inkt-perl"]) def dist(self): self.setup() apt = AptManager(self.session) - with open('dist.ini', 'rb') as f: + with open("dist.ini", "rb") as f: for line in f: - if not line.startswith(b';;'): + if not line.startswith(b";;"): continue try: - (key, value) = line[2:].split(b'=', 1) + (key, value) = line[2:].split(b"=", 1) except ValueError: continue - if (key.strip() == b'class' and - value.strip().startswith(b"'Dist::Inkt")): + if key.strip() == b"class" and value.strip().startswith(b"'Dist::Inkt"): logging.info( - 'Found Dist::Inkt section in dist.ini, ' - 'assuming distinkt.') + "Found Dist::Inkt section in dist.ini, " "assuming distinkt." + ) # TODO(jelmer): install via apt if possible self.session.check_call( - ['cpan', 'install', value.decode().strip("'")], - user='root') - run_with_build_fixer(self.session, ['distinkt-dist']) + ["cpan", "install", value.decode().strip("'")], user="root" + ) + run_with_build_fixer(self.session, ["distinkt-dist"]) return # Default to invoking Dist::Zilla - logging.info('Found dist.ini, assuming dist-zilla.') - apt.install(['libdist-zilla-perl']) - run_with_build_fixer(self.session, ['dzil', 'build', '--in', '..']) + logging.info("Found dist.ini, assuming dist-zilla.") + apt.install(["libdist-zilla-perl"]) + run_with_build_fixer(self.session, ["dzil", "build", "--in", ".."]) class Make(BuildSystem): - def setup(self): apt = AptManager(self.session) - if self.session.exists('Makefile.PL') and not self.session.exists('Makefile'): - apt.install(['perl']) - run_with_build_fixer(self.session, ['perl', 'Makefile.PL']) + if self.session.exists("Makefile.PL") and not self.session.exists("Makefile"): + apt.install(["perl"]) + run_with_build_fixer(self.session, ["perl", "Makefile.PL"]) - if not self.session.exists('Makefile') and not self.session.exists('configure'): - if self.session.exists('autogen.sh'): - if shebang_binary('autogen.sh') is None: - run_with_build_fixer( - self.session, ['/bin/sh', './autogen.sh']) + if not self.session.exists("Makefile") and not self.session.exists("configure"): + if self.session.exists("autogen.sh"): + if shebang_binary("autogen.sh") is None: + run_with_build_fixer(self.session, ["/bin/sh", "./autogen.sh"]) try: - run_with_build_fixer( - self.session, ['./autogen.sh']) + run_with_build_fixer(self.session, ["./autogen.sh"]) except UnidentifiedError as e: - if ("Gnulib not yet bootstrapped; " - "run ./bootstrap instead.\n" in e.lines): + if ( + "Gnulib not yet bootstrapped; " + "run ./bootstrap instead.\n" in e.lines + ): run_with_build_fixer(self.session, ["./bootstrap"]) - run_with_build_fixer(self.session, ['./autogen.sh']) + run_with_build_fixer(self.session, ["./autogen.sh"]) else: raise - elif (self.session.exists('configure.ac') or - self.session.exists('configure.in')): - apt.install([ - 'autoconf', 'automake', 'gettext', 'libtool', - 'gnu-standards']) - run_with_build_fixer(self.session, ['autoreconf', '-i']) + elif self.session.exists("configure.ac") or self.session.exists( + "configure.in" + ): + apt.install( + ["autoconf", "automake", "gettext", "libtool", "gnu-standards"] + ) + run_with_build_fixer(self.session, ["autoreconf", "-i"]) - if not self.session.exists('Makefile') and self.session.exists('configure'): - self.session.check_call(['./configure']) + if not self.session.exists("Makefile") and self.session.exists("configure"): + self.session.check_call(["./configure"]) def dist(self): self.setup() apt = AptManager(self.session) - apt.install(['make']) + apt.install(["make"]) try: - run_with_build_fixer(self.session, ['make', 'dist']) + run_with_build_fixer(self.session, ["make", "dist"]) except UnidentifiedError as e: - if ("make: *** No rule to make target 'dist'. Stop.\n" - in e.lines): + if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: pass - elif ("make[1]: *** No rule to make target 'dist'. Stop.\n" - in e.lines): + elif "make[1]: *** No rule to make target 'dist'. Stop.\n" in e.lines: pass - elif ("Reconfigure the source tree " - "(via './config' or 'perl Configure'), please.\n" - ) in e.lines: - run_with_build_fixer(self.session, ['./config']) - run_with_build_fixer(self.session, ['make', 'dist']) elif ( - "Please try running 'make manifest' and then run " - "'make dist' again.\n" in e.lines): - run_with_build_fixer(self.session, ['make', 'manifest']) - run_with_build_fixer(self.session, ['make', 'dist']) + "Reconfigure the source tree " + "(via './config' or 'perl Configure'), please.\n" + ) in e.lines: + run_with_build_fixer(self.session, ["./config"]) + run_with_build_fixer(self.session, ["make", "dist"]) + elif ( + "Please try running 'make manifest' and then run " + "'make dist' again.\n" in e.lines + ): + run_with_build_fixer(self.session, ["make", "manifest"]) + run_with_build_fixer(self.session, ["make", "dist"]) elif "Please run ./configure first\n" in e.lines: - run_with_build_fixer(self.session, ['./configure']) - run_with_build_fixer(self.session, ['make', 'dist']) - elif any([re.match( - r'Makefile:[0-9]+: \*\*\* Missing \'Make.inc\' ' - r'Run \'./configure \[options\]\' and retry. Stop.\n', - line) for line in e.lines]): - run_with_build_fixer(self.session, ['./configure']) - run_with_build_fixer(self.session, ['make', 'dist']) - elif any([re.match( - r'Problem opening MANIFEST: No such file or directory ' - r'at .* line [0-9]+\.', line) for line in e.lines]): - run_with_build_fixer(self.session, ['make', 'manifest']) - run_with_build_fixer(self.session, ['make', 'dist']) + run_with_build_fixer(self.session, ["./configure"]) + run_with_build_fixer(self.session, ["make", "dist"]) + elif any( + [ + re.match( + r"Makefile:[0-9]+: \*\*\* Missing \'Make.inc\' " + r"Run \'./configure \[options\]\' and retry. Stop.\n", + line, + ) + for line in e.lines + ] + ): + run_with_build_fixer(self.session, ["./configure"]) + run_with_build_fixer(self.session, ["make", "dist"]) + elif any( + [ + re.match( + r"Problem opening MANIFEST: No such file or directory " + r"at .* line [0-9]+\.", + line, + ) + for line in e.lines + ] + ): + run_with_build_fixer(self.session, ["make", "manifest"]) + run_with_build_fixer(self.session, ["make", "dist"]) else: raise else: @@ -319,40 +327,49 @@ class Make(BuildSystem): def detect_buildsystems(session): """Detect build systems.""" - if session.exists('package.xml'): - logging.info('Found package.xml, assuming pear package.') + if session.exists("package.xml"): + logging.info("Found package.xml, assuming pear package.") yield Pear(session) - if session.exists('setup.py'): - logging.info('Found setup.py, assuming python project.') + if session.exists("setup.py"): + logging.info("Found setup.py, assuming python project.") yield SetupPy(session) - if session.exists('pyproject.toml'): - logging.info('Found pyproject.toml, assuming python project.') + if session.exists("pyproject.toml"): + logging.info("Found pyproject.toml, assuming python project.") yield PyProject(session) - if session.exists('setup.cfg'): - logging.info('Found setup.cfg, assuming python project.') + if session.exists("setup.cfg"): + logging.info("Found setup.cfg, assuming python project.") yield SetupCfg(session) - if session.exists('package.json'): - logging.info('Found package.json, assuming node package.') + if session.exists("package.json"): + logging.info("Found package.json, assuming node package.") yield NpmPackage(session) - if session.exists('waf'): - logging.info('Found waf, assuming waf package.') + if session.exists("waf"): + logging.info("Found waf, assuming waf package.") yield Waf(session) gemfiles = [ - entry.name for entry in session.scandir('.') - if entry.name.endswith('.gem')] + entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") + ] if gemfiles: yield Gem(session) - if session.exists('dist.ini') and not session.exists('Makefile.PL'): + if session.exists("dist.ini") and not session.exists("Makefile.PL"): yield DistInkt(session) - if any([session.exists(p) for p in [ - 'Makefile', 'Makefile.PL', 'autogen.sh', 'configure.ac', - 'configure.in']]): + if any( + [ + session.exists(p) + for p in [ + "Makefile", + "Makefile.PL", + "autogen.sh", + "configure.ac", + "configure.in", + ] + ] + ): yield Make(session) diff --git a/ognibuild/debian/__init__.py b/ognibuild/debian/__init__.py index 9d64ab8..449cea0 100644 --- a/ognibuild/debian/__init__.py +++ b/ognibuild/debian/__init__.py @@ -23,21 +23,18 @@ from ..session import Session # TODO(jelmer): move this to debian/ def satisfy_build_deps(session: Session, tree): - source = Deb822(tree.get_file('debian/control')) + source = Deb822(tree.get_file("debian/control")) deps = [] - for name in ['Build-Depends', 'Build-Depends-Indep', 'Build-Depends-Arch']: + for name in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"]: try: - deps.append(source[name].strip().strip(',')) + deps.append(source[name].strip().strip(",")) except KeyError: pass - for name in ['Build-Conflicts', 'Build-Conflicts-Indep', - 'Build-Conflicts-Arch']: + for name in ["Build-Conflicts", "Build-Conflicts-Indep", "Build-Conflicts-Arch"]: try: - deps.append('Conflicts: ' + source[name]) + deps.append("Conflicts: " + source[name]) except KeyError: pass - deps = [ - dep.strip().strip(',') - for dep in deps] + deps = [dep.strip().strip(",") for dep in deps] apt = AptManager(session) apt.satisfy(deps) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index f82f1d1..fab4568 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -16,11 +16,11 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA __all__ = [ - 'changes_filename', - 'get_build_architecture', - 'add_dummy_changelog_entry', - 'build', - 'SbuildFailure', + "changes_filename", + "get_build_architecture", + "add_dummy_changelog_entry", + "build", + "SbuildFailure", ] from datetime import datetime @@ -38,12 +38,12 @@ from breezy.mutabletree import MutableTree from silver_platter.debian import ( BuildFailedError, DEFAULT_BUILDER, - ) +) from buildlog_consultant.sbuild import ( worker_failure_from_sbuild_log, SbuildFailure, - ) +) class MissingChangesFile(Exception): @@ -62,16 +62,24 @@ def changes_filename(package, version, arch): def get_build_architecture(): try: - return subprocess.check_output( - ['dpkg-architecture', '-qDEB_BUILD_ARCH']).strip().decode() + return ( + subprocess.check_output(["dpkg-architecture", "-qDEB_BUILD_ARCH"]) + .strip() + .decode() + ) except subprocess.CalledProcessError as e: - raise Exception( - "Could not find the build architecture: %s" % e) + raise Exception("Could not find the build architecture: %s" % e) def add_dummy_changelog_entry( - tree: MutableTree, subpath: str, suffix: str, suite: str, - message: str, timestamp=None, maintainer=None): + tree: MutableTree, + subpath: str, + suffix: str, + suite: str, + message: str, + timestamp=None, + maintainer=None, +): """Add a dummy changelog entry to a package. Args: @@ -80,87 +88,111 @@ def add_dummy_changelog_entry( suite: Debian suite message: Changelog message """ - def add_suffix(v, suffix): - m = re.fullmatch('(.*)(' + re.escape(suffix) + ')([0-9]+)', v,) - if m: - return (m.group(1) + m.group(2) + '%d' % (int(m.group(3)) + 1)) - else: - return v + suffix + '1' - path = os.path.join(subpath, 'debian', 'changelog') + def add_suffix(v, suffix): + m = re.fullmatch( + "(.*)(" + re.escape(suffix) + ")([0-9]+)", + v, + ) + if m: + return m.group(1) + m.group(2) + "%d" % (int(m.group(3)) + 1) + else: + return v + suffix + "1" + + path = os.path.join(subpath, "debian", "changelog") if maintainer is None: maintainer = get_maintainer() if timestamp is None: timestamp = datetime.now() with tree.get_file(path) as f: cl = Changelog() - cl.parse_changelog( - f, max_blocks=None, allow_empty_author=True, strict=False) + cl.parse_changelog(f, max_blocks=None, allow_empty_author=True, strict=False) version = cl[0].version if version.debian_revision: - version.debian_revision = add_suffix( - version.debian_revision, suffix) + version.debian_revision = add_suffix(version.debian_revision, suffix) else: - version.upstream_version = add_suffix( - version.upstream_version, suffix) + version.upstream_version = add_suffix(version.upstream_version, suffix) cl.new_block( package=cl[0].package, version=version, - urgency='low', + urgency="low", distributions=suite, - author='%s <%s>' % maintainer, + author="%s <%s>" % maintainer, date=format_datetime(timestamp), - changes=['', ' * ' + message, '']) + changes=["", " * " + message, ""], + ) cl_str = cl._format(allow_missing_author=True) tree.put_file_bytes_non_atomic(path, cl_str.encode(cl._encoding)) -def get_latest_changelog_version(local_tree, subpath=''): - path = osutils.pathjoin(subpath, 'debian/changelog') +def get_latest_changelog_version(local_tree, subpath=""): + path = osutils.pathjoin(subpath, "debian/changelog") with local_tree.get_file(path) as f: cl = Changelog(f, max_blocks=1) return cl.package, cl.version -def build(local_tree, outf, build_command=DEFAULT_BUILDER, result_dir=None, - distribution=None, subpath='', source_date_epoch=None): - args = [sys.executable, '-m', 'breezy', 'builddeb', - '--guess-upstream-branch-url', '--builder=%s' % build_command] +def build( + local_tree, + outf, + build_command=DEFAULT_BUILDER, + result_dir=None, + distribution=None, + subpath="", + source_date_epoch=None, +): + args = [ + sys.executable, + "-m", + "breezy", + "builddeb", + "--guess-upstream-branch-url", + "--builder=%s" % build_command, + ] if result_dir: - args.append('--result-dir=%s' % result_dir) - outf.write('Running %r\n' % (build_command, )) + args.append("--result-dir=%s" % result_dir) + outf.write("Running %r\n" % (build_command,)) outf.flush() env = dict(os.environ.items()) if distribution is not None: - env['DISTRIBUTION'] = distribution + env["DISTRIBUTION"] = distribution if source_date_epoch is not None: - env['SOURCE_DATE_EPOCH'] = '%d' % source_date_epoch - logging.info('Building debian packages, running %r.', build_command) + env["SOURCE_DATE_EPOCH"] = "%d" % source_date_epoch + logging.info("Building debian packages, running %r.", build_command) try: subprocess.check_call( - args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf, - env=env) + args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf, env=env + ) except subprocess.CalledProcessError: raise BuildFailedError() def build_once( - local_tree, build_suite, output_directory, build_command, - subpath='', source_date_epoch=None): - build_log_path = os.path.join(output_directory, 'build.log') + local_tree, + build_suite, + output_directory, + build_command, + subpath="", + source_date_epoch=None, +): + build_log_path = os.path.join(output_directory, "build.log") try: - with open(build_log_path, 'w') as f: - build(local_tree, outf=f, build_command=build_command, - result_dir=output_directory, distribution=build_suite, - subpath=subpath, source_date_epoch=source_date_epoch) + with open(build_log_path, "w") as f: + build( + local_tree, + outf=f, + build_command=build_command, + result_dir=output_directory, + distribution=build_suite, + subpath=subpath, + source_date_epoch=source_date_epoch, + ) except BuildFailedError: - with open(build_log_path, 'rb') as f: + with open(build_log_path, "rb") as f: raise worker_failure_from_sbuild_log(f) - (cl_package, cl_version) = get_latest_changelog_version( - local_tree, subpath) - changes_name = changes_filename( - cl_package, cl_version, get_build_architecture()) + (cl_package, cl_version) = get_latest_changelog_version(local_tree, subpath) + changes_name = changes_filename(cl_package, cl_version, get_build_architecture()) changes_path = os.path.join(output_directory, changes_name) if not os.path.exists(changes_path): raise MissingChangesFile(changes_name) @@ -168,13 +200,19 @@ def build_once( def gbp_dch(path): - subprocess.check_call(['gbp', 'dch'], cwd=path) + subprocess.check_call(["gbp", "dch"], cwd=path) def attempt_build( - local_tree, suffix, build_suite, output_directory, build_command, - build_changelog_entry='Build for debian-janitor apt repository.', - subpath='', source_date_epoch=None): + local_tree, + suffix, + build_suite, + output_directory, + build_command, + build_changelog_entry="Build for debian-janitor apt repository.", + subpath="", + source_date_epoch=None, +): """Attempt a build, with a custom distribution set. Args: @@ -189,8 +227,13 @@ def attempt_build( Returns: Tuple with (changes_name, cl_version) """ add_dummy_changelog_entry( - local_tree, subpath, suffix, build_suite, - build_changelog_entry) + local_tree, subpath, suffix, build_suite, build_changelog_entry + ) return build_once( - local_tree, build_suite, output_directory, build_command, subpath, - source_date_epoch=source_date_epoch) + local_tree, + build_suite, + output_directory, + build_command, + subpath, + source_date_epoch=source_date_epoch, + ) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 3e1d7b1..cf88224 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -16,7 +16,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA __all__ = [ - 'build_incrementally', + "build_incrementally", ] import logging @@ -30,7 +30,7 @@ from debian.deb822 import ( Deb822, PkgRelation, Release, - ) +) from breezy.commit import PointlessCommit from breezy.tree import Tree @@ -39,34 +39,34 @@ from debmutate.control import ( ensure_minimum_version, pg_buildext_updatecontrol, ControlEditor, - ) +) from debmutate.debhelper import ( get_debhelper_compat_level, - ) +) from debmutate.deb822 import ( Deb822Editor, - ) +) from debmutate.reformatting import ( FormattingUnpreservable, GeneratedFile, - ) +) from lintian_brush import ( reset_tree, - ) +) from lintian_brush.changelog import ( add_changelog_entry, - ) +) from debmutate._rules import ( dh_invoke_add_with, update_rules, - ) +) from silver_platter.debian import ( debcommit, DEFAULT_BUILDER, - ) +) -from .build import attempt_build +from .build import attempt_build, get_build_architecture from buildlog_consultant import Problem from buildlog_consultant.common import ( MissingConfigStatusInput, @@ -102,13 +102,13 @@ from buildlog_consultant.common import ( MissingMavenArtifacts, GnomeCommonMissing, MissingGnomeCommonDependency, - ) +) from buildlog_consultant.apt import ( AptFetchFailure, - ) +) from buildlog_consultant.sbuild import ( SbuildFailure, - ) +) DEFAULT_MAX_ITERATIONS = 10 @@ -122,9 +122,7 @@ class CircularDependency(Exception): class DependencyContext(object): - - def __init__(self, tree, subpath='', committer=None, - update_changelog=True): + def __init__(self, tree, subpath="", committer=None, update_changelog=True): self.tree = tree self.subpath = subpath self.committer = committer @@ -135,36 +133,50 @@ class DependencyContext(object): class BuildDependencyContext(DependencyContext): - def add_dependency(self, package, minimum_version=None): return add_build_dependency( - self.tree, package, minimum_version=minimum_version, - committer=self.committer, subpath=self.subpath, - update_changelog=self.update_changelog) + self.tree, + package, + minimum_version=minimum_version, + committer=self.committer, + subpath=self.subpath, + update_changelog=self.update_changelog, + ) class AutopkgtestDependencyContext(DependencyContext): - - def __init__(self, testname, tree, subpath='', committer=None, - update_changelog=True): + def __init__( + self, testname, tree, subpath="", committer=None, update_changelog=True + ): self.testname = testname super(AutopkgtestDependencyContext, self).__init__( - tree, subpath, committer, update_changelog) + tree, subpath, committer, update_changelog + ) def add_dependency(self, package, minimum_version=None): return add_test_dependency( - self.tree, self.testname, package, + self.tree, + self.testname, + package, minimum_version=minimum_version, - committer=self.committer, subpath=self.subpath, - update_changelog=self.update_changelog) + committer=self.committer, + subpath=self.subpath, + update_changelog=self.update_changelog, + ) -def add_build_dependency(tree, package, minimum_version=None, - committer=None, subpath='', update_changelog=True): +def add_build_dependency( + tree, + package, + minimum_version=None, + committer=None, + subpath="", + update_changelog=True, +): if not isinstance(package, str): raise TypeError(package) - control_path = os.path.join(tree.abspath(subpath), 'debian/control') + control_path = os.path.join(tree.abspath(subpath), "debian/control") try: with ControlEditor(path=control_path) as updater: for binary in updater.binaries: @@ -172,15 +184,14 @@ def add_build_dependency(tree, package, minimum_version=None, raise CircularDependency(package) if minimum_version: updater.source["Build-Depends"] = ensure_minimum_version( - updater.source.get("Build-Depends", ""), - package, minimum_version) + updater.source.get("Build-Depends", ""), package, minimum_version + ) else: updater.source["Build-Depends"] = ensure_some_version( - updater.source.get("Build-Depends", ""), package) + updater.source.get("Build-Depends", ""), package + ) except FormattingUnpreservable as e: - logging.info( - 'Unable to edit %s in a way that preserves formatting.', - e.path) + logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False if minimum_version: @@ -189,22 +200,32 @@ def add_build_dependency(tree, package, minimum_version=None, desc = package if not updater.changed: - logging.info('Giving up; dependency %s was already present.', desc) + logging.info("Giving up; dependency %s was already present.", desc) return False logging.info("Adding build dependency: %s", desc) return commit_debian_changes( - tree, subpath, "Add missing build dependency on %s." % desc, - committer=committer, update_changelog=update_changelog) + tree, + subpath, + "Add missing build dependency on %s." % desc, + committer=committer, + update_changelog=update_changelog, + ) -def add_test_dependency(tree, testname, package, minimum_version=None, - committer=None, subpath='', update_changelog=True): +def add_test_dependency( + tree, + testname, + package, + minimum_version=None, + committer=None, + subpath="", + update_changelog=True, +): if not isinstance(package, str): raise TypeError(package) - tests_control_path = os.path.join( - tree.abspath(subpath), 'debian/tests/control') + tests_control_path = os.path.join(tree.abspath(subpath), "debian/tests/control") try: with Deb822Editor(path=tests_control_path) as updater: @@ -219,15 +240,14 @@ def add_test_dependency(tree, testname, package, minimum_version=None, continue if minimum_version: control["Depends"] = ensure_minimum_version( - control.get("Depends", ""), - package, minimum_version) + control.get("Depends", ""), package, minimum_version + ) else: control["Depends"] = ensure_some_version( - control.get("Depends", ""), package) + control.get("Depends", ""), package + ) except FormattingUnpreservable as e: - logging.info( - 'Unable to edit %s in a way that preserves formatting.', - e.path) + logging.info("Unable to edit %s in a way that preserves formatting.", e.path) return False if not updater.changed: return False @@ -239,22 +259,27 @@ def add_test_dependency(tree, testname, package, minimum_version=None, logging.info("Adding dependency to test %s: %s", testname, desc) return commit_debian_changes( - tree, subpath, + tree, + subpath, "Add missing dependency for test %s on %s." % (testname, desc), - update_changelog=update_changelog) + update_changelog=update_changelog, + ) -def commit_debian_changes(tree, subpath, summary, committer=None, - update_changelog=True): +def commit_debian_changes( + tree, subpath, summary, committer=None, update_changelog=True +): with tree.lock_write(): try: if update_changelog: add_changelog_entry( - tree, os.path.join(subpath, 'debian/changelog'), [summary]) + tree, os.path.join(subpath, "debian/changelog"), [summary] + ) debcommit(tree, committer=committer, subpath=subpath) else: - tree.commit(message=summary, committer=committer, - specific_files=[subpath]) + tree.commit( + message=summary, committer=committer, specific_files=[subpath] + ) except PointlessCommit: return False else: @@ -262,7 +287,6 @@ def commit_debian_changes(tree, subpath, summary, committer=None, class FileSearcher(object): - def search_files(self, path, regex=False): raise NotImplementedError(self.search_files) @@ -272,13 +296,12 @@ class ContentsFileNotFound(Exception): class AptContentsFileSearcher(FileSearcher): - def __init__(self): self._db = {} @classmethod def from_env(cls): - sources = os.environ['REPOSITORIES'].split(':') + sources = os.environ["REPOSITORIES"].split(":") return cls.from_repositories(sources) def __setitem__(self, path, package): @@ -296,9 +319,9 @@ class AptContentsFileSearcher(FileSearcher): def load_file(self, f): for line in f: (path, rest) = line.rsplit(maxsplit=1) - package = rest.split(b'/')[-1] - decoded_path = '/' + path.decode('utf-8', 'surrogateescape') - self[decoded_path] = package.decode('utf-8') + package = rest.split(b"/")[-1] + decoded_path = "/" + path.decode("utf-8", "surrogateescape") + self[decoded_path] = package.decode("utf-8") @classmethod def from_urls(cls, urls): @@ -311,54 +334,56 @@ class AptContentsFileSearcher(FileSearcher): def from_repositories(cls, sources): # TODO(jelmer): Verify signatures, etc. urls = [] - arches = [get_build_architecture(), 'all'] + arches = [get_build_architecture(), "all"] for source in sources: - parts = source.split(' ') - if parts[0] != 'deb': - logging.warning('Invalid line in sources: %r', source) + parts = source.split(" ") + if parts[0] != "deb": + logging.warning("Invalid line in sources: %r", source) continue base_url = parts[1] name = parts[2] components = parts[3:] - response = cls._get('%s/%s/Release' % (base_url, name)) + response = cls._get("%s/%s/Release" % (base_url, name)) r = Release(response) desired_files = set() for component in components: for arch in arches: - desired_files.add('%s/Contents-%s' % (component, arch)) - for entry in r['MD5Sum']: - if entry['name'] in desired_files: - urls.append('%s/%s/%s' % (base_url, name, entry['name'])) + desired_files.add("%s/Contents-%s" % (component, arch)) + for entry in r["MD5Sum"]: + if entry["name"] in desired_files: + urls.append("%s/%s/%s" % (base_url, name, entry["name"])) return cls.from_urls(urls) @staticmethod def _get(url): from urllib.request import urlopen, Request - request = Request(url, headers={'User-Agent': 'Debian Janitor'}) + + request = Request(url, headers={"User-Agent": "Debian Janitor"}) return urlopen(request) def load_url(self, url): from urllib.error import HTTPError + try: response = self._get(url) except HTTPError as e: if e.status == 404: raise ContentsFileNotFound(url) raise - if url.endswith('.gz'): + if url.endswith(".gz"): import gzip + f = gzip.GzipFile(fileobj=response) - elif response.headers.get_content_type() == 'text/plain': + elif response.headers.get_content_type() == "text/plain": f = response else: raise Exception( - 'Unknown content type %r' % - response.headers.get_content_type()) + "Unknown content type %r" % response.headers.get_content_type() + ) self.load_file(f) class GeneratedFileSearcher(FileSearcher): - def __init__(self, db): self._db = db @@ -373,10 +398,13 @@ class GeneratedFileSearcher(FileSearcher): # TODO(jelmer): read from a file -GENERATED_FILE_SEARCHER = GeneratedFileSearcher({ - '/etc/locale.gen': 'locales', - # Alternative - '/usr/bin/rst2html': '/usr/share/docutils/scripts/python3/rst2html'}) +GENERATED_FILE_SEARCHER = GeneratedFileSearcher( + { + "/etc/locale.gen": "locales", + # Alternative + "/usr/bin/rst2html": "/usr/share/docutils/scripts/python3/rst2html", + } +) _apt_file_searcher = None @@ -399,12 +427,12 @@ def get_package_for_paths(paths, regex=False): if candidates: break if len(candidates) == 0: - logging.warning('No packages found that contain %r', paths) + logging.warning("No packages found that contain %r", paths) return None if len(candidates) > 1: logging.warning( - 'More than 1 packages found that contain %r: %r', - path, candidates) + "More than 1 packages found that contain %r: %r", path, candidates + ) # Euhr. Pick the one with the shortest name? return sorted(candidates, key=len)[0] else: @@ -412,71 +440,75 @@ def get_package_for_paths(paths, regex=False): def get_package_for_python_module(module, python_version): - if python_version == 'python3': + if python_version == "python3": paths = [ os.path.join( - '/usr/lib/python3/dist-packages', - module.replace('.', '/'), - '__init__.py'), + "/usr/lib/python3/dist-packages", + module.replace(".", "/"), + "__init__.py", + ), os.path.join( - '/usr/lib/python3/dist-packages', - module.replace('.', '/') + '.py'), + "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py" + ), os.path.join( - '/usr/lib/python3\\.[0-9]+/lib-dynload', - module.replace('.', '/') + '\\.cpython-.*\\.so'), + "/usr/lib/python3\\.[0-9]+/lib-dynload", + module.replace(".", "/") + "\\.cpython-.*\\.so", + ), os.path.join( - '/usr/lib/python3\\.[0-9]+/', - module.replace('.', '/') + '.py'), + "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py" + ), os.path.join( - '/usr/lib/python3\\.[0-9]+/', - module.replace('.', '/'), '__init__.py'), - ] - elif python_version == 'python2': + "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py" + ), + ] + elif python_version == "python2": paths = [ os.path.join( - '/usr/lib/python2\\.[0-9]/dist-packages', - module.replace('.', '/'), - '__init__.py'), + "/usr/lib/python2\\.[0-9]/dist-packages", + module.replace(".", "/"), + "__init__.py", + ), os.path.join( - '/usr/lib/python2\\.[0-9]/dist-packages', - module.replace('.', '/') + '.py'), + "/usr/lib/python2\\.[0-9]/dist-packages", + module.replace(".", "/") + ".py", + ), os.path.join( - '/usr/lib/python2.\\.[0-9]/lib-dynload', - module.replace('.', '/') + '.so')] - elif python_version == 'pypy': + "/usr/lib/python2.\\.[0-9]/lib-dynload", + module.replace(".", "/") + ".so", + ), + ] + elif python_version == "pypy": paths = [ os.path.join( - '/usr/lib/pypy/dist-packages', - module.replace('.', '/'), - '__init__.py'), + "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py" + ), os.path.join( - '/usr/lib/pypy/dist-packages', - module.replace('.', '/') + '.py'), + "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py" + ), os.path.join( - '/usr/lib/pypy/dist-packages', - module.replace('.', '/') + '\\.pypy-.*\\.so'), - ] + "/usr/lib/pypy/dist-packages", + module.replace(".", "/") + "\\.pypy-.*\\.so", + ), + ] else: - raise AssertionError( - 'unknown python version %r' % python_version) + raise AssertionError("unknown python version %r" % python_version) return get_package_for_paths(paths, regex=True) def targeted_python_versions(tree: Tree) -> Set[str]: - with tree.get_file('debian/control') as f: + with tree.get_file("debian/control") as f: control = Deb822(f) - build_depends = PkgRelation.parse_relations( - control.get('Build-Depends', '')) + build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) all_build_deps: Set[str] = set() for or_deps in build_depends: - all_build_deps.update(or_dep['name'] for or_dep in or_deps) + all_build_deps.update(or_dep["name"] for or_dep in or_deps) targeted = set() - if any(x.startswith('pypy') for x in all_build_deps): - targeted.add('pypy') - if any(x.startswith('python-') for x in all_build_deps): - targeted.add('cpython2') - if any(x.startswith('python3-') for x in all_build_deps): - targeted.add('cpython3') + if any(x.startswith("pypy") for x in all_build_deps): + targeted.add("pypy") + if any(x.startswith("python-") for x in all_build_deps): + targeted.add("cpython2") + if any(x.startswith("python3-") for x in all_build_deps): + targeted.add("cpython3") return targeted @@ -487,6 +519,7 @@ def package_exists(package): global apt_cache if apt_cache is None: import apt_pkg + apt_cache = apt_pkg.Cache() for p in apt_cache.packages: if p.name == package: @@ -495,66 +528,65 @@ def package_exists(package): def fix_missing_javascript_runtime(error, context): - package = get_package_for_paths( - ['/usr/bin/node', '/usr/bin/duk'], - regex=False) + package = get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False) if package is None: return False return context.add_dependency(package) -def fix_missing_python_distribution(error, context): +def fix_missing_python_distribution(error, context): # noqa: C901 targeted = targeted_python_versions(context.tree) default = not targeted pypy_pkg = get_package_for_paths( - ['/usr/lib/pypy/dist-packages/%s-.*.egg-info' % error.distribution], - regex=True) + ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True + ) if pypy_pkg is None: - pypy_pkg = 'pypy-%s' % error.distribution + pypy_pkg = "pypy-%s" % error.distribution if not package_exists(pypy_pkg): pypy_pkg = None py2_pkg = get_package_for_paths( - ['/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info' % - error.distribution], regex=True) + ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution], + regex=True, + ) if py2_pkg is None: - py2_pkg = 'python-%s' % error.distribution + py2_pkg = "python-%s" % error.distribution if not package_exists(py2_pkg): py2_pkg = None py3_pkg = get_package_for_paths( - ['/usr/lib/python3/dist-packages/%s-.*.egg-info' % - error.distribution], regex=True) + ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution], + regex=True, + ) if py3_pkg is None: - py3_pkg = 'python3-%s' % error.distribution + py3_pkg = "python3-%s" % error.distribution if not package_exists(py3_pkg): py3_pkg = None extra_build_deps = [] if error.python_version == 2: - if 'pypy' in targeted: + if "pypy" in targeted: if not pypy_pkg: - logging.warning('no pypy package found for %s', error.module) + logging.warning("no pypy package found for %s", error.module) else: extra_build_deps.append(pypy_pkg) - if 'cpython2' in targeted or default: + if "cpython2" in targeted or default: if not py2_pkg: - logging.warning( - 'no python 2 package found for %s', error.module) + logging.warning("no python 2 package found for %s", error.module) return False extra_build_deps.append(py2_pkg) elif error.python_version == 3: if not py3_pkg: - logging.warning('no python 3 package found for %s', error.module) + logging.warning("no python 3 package found for %s", error.module) return False extra_build_deps.append(py3_pkg) else: - if py3_pkg and ('cpython3' in targeted or default): + if py3_pkg and ("cpython3" in targeted or default): extra_build_deps.append(py3_pkg) - if py2_pkg and ('cpython2' in targeted or default): + if py2_pkg and ("cpython2" in targeted or default): extra_build_deps.append(py2_pkg) - if pypy_pkg and 'pypy' in targeted: + if pypy_pkg and "pypy" in targeted: extra_build_deps.append(pypy_pkg) if not extra_build_deps: @@ -562,48 +594,45 @@ def fix_missing_python_distribution(error, context): for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency( - dep_pkg, minimum_version=error.minimum_version): + if not context.add_dependency(dep_pkg, minimum_version=error.minimum_version): return False return True def fix_missing_python_module(error, context): - if getattr(context, 'tree', None) is not None: + if getattr(context, "tree", None) is not None: targeted = targeted_python_versions(context.tree) else: targeted = set() - default = (not targeted) + default = not targeted - pypy_pkg = get_package_for_python_module(error.module, 'pypy') - py2_pkg = get_package_for_python_module(error.module, 'python2') - py3_pkg = get_package_for_python_module(error.module, 'python3') + pypy_pkg = get_package_for_python_module(error.module, "pypy") + py2_pkg = get_package_for_python_module(error.module, "python2") + py3_pkg = get_package_for_python_module(error.module, "python3") extra_build_deps = [] if error.python_version == 2: - if 'pypy' in targeted: + if "pypy" in targeted: if not pypy_pkg: - logging.warning('no pypy package found for %s', error.module) + logging.warning("no pypy package found for %s", error.module) else: extra_build_deps.append(pypy_pkg) - if 'cpython2' in targeted or default: + if "cpython2" in targeted or default: if not py2_pkg: - logging.warning( - 'no python 2 package found for %s', error.module) + logging.warning("no python 2 package found for %s", error.module) return False extra_build_deps.append(py2_pkg) elif error.python_version == 3: if not py3_pkg: - logging.warning( - 'no python 3 package found for %s', error.module) + logging.warning("no python 3 package found for %s", error.module) return False extra_build_deps.append(py3_pkg) else: - if py3_pkg and ('cpython3' in targeted or default): + if py3_pkg and ("cpython3" in targeted or default): extra_build_deps.append(py3_pkg) - if py2_pkg and ('cpython2' in targeted or default): + if py2_pkg and ("cpython2" in targeted or default): extra_build_deps.append(py2_pkg) - if pypy_pkg and 'pypy' in targeted: + if pypy_pkg and "pypy" in targeted: extra_build_deps.append(pypy_pkg) if not extra_build_deps: @@ -618,8 +647,8 @@ def fix_missing_python_module(error, context): def fix_missing_go_package(error, context): package = get_package_for_paths( - [os.path.join('/usr/share/gocode/src', error.package, '.*')], - regex=True) + [os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True + ) if package is None: return False return context.add_dependency(package) @@ -627,10 +656,12 @@ def fix_missing_go_package(error, context): def fix_missing_c_header(error, context): package = get_package_for_paths( - [os.path.join('/usr/include', error.header)], regex=False) + [os.path.join("/usr/include", error.header)], regex=False + ) if package is None: package = get_package_for_paths( - [os.path.join('/usr/include', '.*', error.header)], regex=True) + [os.path.join("/usr/include", ".*", error.header)], regex=True + ) if package is None: return False return context.add_dependency(package) @@ -638,16 +669,16 @@ def fix_missing_c_header(error, context): def fix_missing_pkg_config(error, context): package = get_package_for_paths( - [os.path.join('/usr/lib/pkgconfig', error.module + '.pc')]) + [os.path.join("/usr/lib/pkgconfig", error.module + ".pc")] + ) if package is None: package = get_package_for_paths( - [os.path.join('/usr/lib', '.*', 'pkgconfig', - error.module + '.pc')], - regex=True) + [os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")], + regex=True, + ) if package is None: return False - return context.add_dependency( - package, minimum_version=error.minimum_version) + return context.add_dependency(package, minimum_version=error.minimum_version) def fix_missing_command(error, context): @@ -655,11 +686,11 @@ def fix_missing_command(error, context): paths = [error.command] else: paths = [ - os.path.join(dirname, error.command) - for dirname in ['/usr/bin', '/bin']] + os.path.join(dirname, error.command) for dirname in ["/usr/bin", "/bin"] + ] package = get_package_for_paths(paths) if package is None: - logging.info('No packages found that contain %r', paths) + logging.info("No packages found that contain %r", paths) return False return context.add_dependency(package) @@ -672,10 +703,10 @@ def fix_missing_file(error, context): def fix_missing_sprockets_file(error, context): - if error.content_type == 'application/javascript': - path = '/usr/share/.*/app/assets/javascripts/%s.js$' % error.name + if error.content_type == "application/javascript": + path = "/usr/share/.*/app/assets/javascripts/%s.js$" % error.name else: - logging.warning('unable to handle content type %s', error.content_type) + logging.warning("unable to handle content type %s", error.content_type) return False package = get_package_for_paths([path], regex=True) if package is None: @@ -683,22 +714,23 @@ def fix_missing_sprockets_file(error, context): return context.add_dependency(package) -DEFAULT_PERL_PATHS = ['/usr/share/perl5'] +DEFAULT_PERL_PATHS = ["/usr/share/perl5"] def fix_missing_perl_file(error, context): - if (error.filename == 'Makefile.PL' and - not context.tree.has_filename('Makefile.PL') and - context.tree.has_filename('dist.ini')): + if ( + error.filename == "Makefile.PL" + and not context.tree.has_filename("Makefile.PL") + and context.tree.has_filename("dist.ini") + ): # TODO(jelmer): add dist-zilla add-on to debhelper raise NotImplementedError if error.inc is None: if error.filename is None: - filename = error.module.replace('::', '/') + '.pm' - paths = [os.path.join(inc, filename) - for inc in DEFAULT_PERL_PATHS] + filename = error.module.replace("::", "/") + ".pm" + paths = [os.path.join(inc, filename) for inc in DEFAULT_PERL_PATHS] elif not os.path.isabs(error.filename): return False else: @@ -707,41 +739,42 @@ def fix_missing_perl_file(error, context): paths = [os.path.join(inc, error.filename) for inc in error.inc] package = get_package_for_paths(paths, regex=False) if package is None: - if getattr(error, 'module', None): + if getattr(error, "module", None): logging.warning( - 'no perl package found for %s (%r).', - error.module, error.filename) + "no perl package found for %s (%r).", error.module, error.filename + ) else: logging.warning( - 'perl file %s not found (paths searched for: %r).', - error.filename, paths) + "perl file %s not found (paths searched for: %r).", + error.filename, + paths, + ) return False return context.add_dependency(package) def get_package_for_node_package(node_package): paths = [ - '/usr/share/nodejs/.*/node_modules/%s/package.json' % node_package, - '/usr/lib/nodejs/%s/package.json' % node_package, - '/usr/share/nodejs/%s/package.json' % node_package] + "/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package, + "/usr/lib/nodejs/%s/package.json" % node_package, + "/usr/share/nodejs/%s/package.json" % node_package, + ] return get_package_for_paths(paths, regex=True) def fix_missing_node_module(error, context): package = get_package_for_node_package(error.module) if package is None: - logging.warning( - 'no node package found for %s.', - error.module) + logging.warning("no node package found for %s.", error.module) return False return context.add_dependency(package) def fix_missing_dh_addon(error, context): - paths = [os.path.join('/usr/share/perl5', error.path)] + paths = [os.path.join("/usr/share/perl5", error.path)] package = get_package_for_paths(paths) if package is None: - logging.warning('no package for debhelper addon %s', error.name) + logging.warning("no package for debhelper addon %s", error.name) return False return context.add_dependency(package) @@ -751,32 +784,33 @@ def retry_apt_failure(error, context): def fix_missing_php_class(error, context): - path = '/usr/share/php/%s.php' % error.php_class.replace('\\', '/') + path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/") package = get_package_for_paths([path]) if package is None: - logging.warning('no package for PHP class %s', error.php_class) + logging.warning("no package for PHP class %s", error.php_class) return False return context.add_dependency(package) def fix_missing_jdk_file(error, context): - path = error.jdk_path + '.*/' + error.filename + path = error.jdk_path + ".*/" + error.filename package = get_package_for_paths([path], regex=True) if package is None: logging.warning( - 'no package found for %s (JDK: %s) - regex %s', - error.filename, error.jdk_path, path) + "no package found for %s (JDK: %s) - regex %s", + error.filename, + error.jdk_path, + path, + ) return False return context.add_dependency(package) def fix_missing_vala_package(error, context): - path = '/usr/share/vala-[0-9.]+/vapi/%s.vapi' % error.package + path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package package = get_package_for_paths([path], regex=True) if package is None: - logging.warning( - 'no file found for package %s - regex %s', - error.package, path) + logging.warning("no file found for package %s - regex %s", error.package, path) return False return context.add_dependency(package) @@ -785,12 +819,11 @@ def fix_missing_xml_entity(error, context): # Ideally we should be using the XML catalog for this, but hardcoding # a few URLs will do for now.. URL_MAP = { - 'http://www.oasis-open.org/docbook/xml/': - '/usr/share/xml/docbook/schema/dtd/' + "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/" } for url, path in URL_MAP.items(): if error.url.startswith(url): - search_path = os.path.join(path, error.url[len(url):]) + search_path = os.path.join(path, error.url[len(url) :]) break else: return False @@ -802,69 +835,73 @@ def fix_missing_xml_entity(error, context): def fix_missing_library(error, context): - paths = [os.path.join('/usr/lib/lib%s.so$' % error.library), - os.path.join('/usr/lib/.*/lib%s.so$' % error.library), - os.path.join('/usr/lib/lib%s.a$' % error.library), - os.path.join('/usr/lib/.*/lib%s.a$' % error.library)] + paths = [ + os.path.join("/usr/lib/lib%s.so$" % error.library), + os.path.join("/usr/lib/.*/lib%s.so$" % error.library), + os.path.join("/usr/lib/lib%s.a$" % error.library), + os.path.join("/usr/lib/.*/lib%s.a$" % error.library), + ] package = get_package_for_paths(paths, regex=True) if package is None: - logging.warning('no package for library %s', error.library) + logging.warning("no package for library %s", error.library) return False return context.add_dependency(package) def fix_missing_ruby_gem(error, context): - paths = [os.path.join( - '/usr/share/rubygems-integration/all/' - 'specifications/%s-.*\\.gemspec' % error.gem)] + paths = [ + os.path.join( + "/usr/share/rubygems-integration/all/" + "specifications/%s-.*\\.gemspec" % error.gem + ) + ] package = get_package_for_paths(paths, regex=True) if package is None: - logging.warning('no package for gem %s', error.gem) + logging.warning("no package for gem %s", error.gem) return False return context.add_dependency(package, minimum_version=error.version) def fix_missing_ruby_file(error, context): - paths = [ - os.path.join('/usr/lib/ruby/vendor_ruby/%s.rb' % error.filename)] + paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)] package = get_package_for_paths(paths) if package is not None: return context.add_dependency(package) paths = [ - os.path.join(r'/usr/share/rubygems-integration/all/gems/([^/]+)/' - 'lib/%s.rb' % error.filename)] + os.path.join( + r"/usr/share/rubygems-integration/all/gems/([^/]+)/" + "lib/%s.rb" % error.filename + ) + ] package = get_package_for_paths(paths, regex=True) if package is not None: return context.add_dependency(package) - logging.warning('no package for ruby file %s', error.filename) + logging.warning("no package for ruby file %s", error.filename) return False def fix_missing_r_package(error, context): - paths = [os.path.join('/usr/lib/R/site-library/.*/R/%s$' % error.package)] + paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)] package = get_package_for_paths(paths, regex=True) if package is None: - logging.warning('no package for R package %s', error.package) + logging.warning("no package for R package %s", error.package) return False - return context.add_dependency( - package, minimum_version=error.minimum_version) + return context.add_dependency(package, minimum_version=error.minimum_version) def fix_missing_java_class(error, context): # Unfortunately this only finds classes in jars installed on the host # system :( - output = subprocess.check_output( - ["java-propose-classpath", "-c" + error.classname]) - classpath = [ - p for p in output.decode().strip(":").strip().split(':') if p] + output = subprocess.check_output(["java-propose-classpath", "-c" + error.classname]) + classpath = [p for p in output.decode().strip(":").strip().split(":") if p] if not classpath: - logging.warning('unable to find classpath for %s', error.classname) + logging.warning("unable to find classpath for %s", error.classname) return False - logging.info('Classpath for %s: %r', error.classname, classpath) + logging.info("Classpath for %s: %r", error.classname, classpath) package = get_package_for_paths(classpath) if package is None: - logging.warning('no package for files in %r', classpath) + logging.warning("no package for files in %r", classpath) return False return context.add_dependency(package) @@ -872,25 +909,26 @@ def fix_missing_java_class(error, context): def enable_dh_autoreconf(context): # Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by # default. - debhelper_compat_version = get_debhelper_compat_level( - context.tree.abspath('.')) + debhelper_compat_version = get_debhelper_compat_level(context.tree.abspath(".")) if debhelper_compat_version is not None and debhelper_compat_version < 10: + def add_with_autoreconf(line, target): - if target != b'%': + if target != b"%": return line - if not line.startswith(b'dh '): + if not line.startswith(b"dh "): return line - return dh_invoke_add_with(line, b'autoreconf') + return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return context.add_dependency('dh-autoreconf') + return context.add_dependency("dh-autoreconf") return False def fix_missing_configure(error, context): - if (not context.tree.has_filename('configure.ac') and - not context.tree.has_filename('configure.in')): + if not context.tree.has_filename("configure.ac") and not context.tree.has_filename( + "configure.in" + ): return False return enable_dh_autoreconf(context) @@ -905,95 +943,103 @@ def fix_missing_automake_input(error, context): def fix_missing_maven_artifacts(error, context): artifact = error.artifacts[0] - parts = artifact.split(':') + parts = artifact.split(":") if len(parts) == 4: (group_id, artifact_id, kind, version) = parts regex = False elif len(parts) == 3: (group_id, artifact_id, version) = parts - kind = 'jar' + kind = "jar" regex = False elif len(parts) == 2: - version = '.*' + version = ".*" (group_id, artifact_id) = parts - kind = 'jar' + kind = "jar" regex = True else: - raise AssertionError( - 'invalid number of parts to artifact %s' % artifact) - paths = [os.path.join( - '/usr/share/maven-repo', group_id.replace('.', '/'), - artifact_id, version, '%s-%s.%s' % (artifact_id, version, kind))] + raise AssertionError("invalid number of parts to artifact %s" % artifact) + paths = [ + os.path.join( + "/usr/share/maven-repo", + group_id.replace(".", "/"), + artifact_id, + version, + "%s-%s.%s" % (artifact_id, version, kind), + ) + ] package = get_package_for_paths(paths, regex=regex) if package is None: - logging.warning('no package for artifact %s', artifact) + logging.warning("no package for artifact %s", artifact) return False return context.add_dependency(package) def install_gnome_common(error, context): - return context.add_dependency('gnome-common') + return context.add_dependency("gnome-common") def install_gnome_common_dep(error, context): - if error.package == 'glib-gettext': - package = get_package_for_paths(['/usr/bin/glib-gettextize']) + if error.package == "glib-gettext": + package = get_package_for_paths(["/usr/bin/glib-gettextize"]) else: package = None if package is None: - logging.warning('No debian package for package %s', error.package) + logging.warning("No debian package for package %s", error.package) return False return context.add_dependency( - package=package, - minimum_version=error.minimum_version) + package=package, minimum_version=error.minimum_version + ) def install_xfce_dep(error, context): - if error.package == 'gtk-doc': - package = get_package_for_paths(['/usr/bin/gtkdocize']) + if error.package == "gtk-doc": + package = get_package_for_paths(["/usr/bin/gtkdocize"]) else: package = None if package is None: - logging.warning('No debian package for package %s', error.package) + logging.warning("No debian package for package %s", error.package) return False return context.add_dependency(package=package) def fix_missing_config_status_input(error, context): - autogen_path = 'autogen.sh' - rules_path = 'debian/rules' - if context.subpath not in ('.', ''): + autogen_path = "autogen.sh" + rules_path = "debian/rules" + if context.subpath not in (".", ""): autogen_path = os.path.join(context.subpath, autogen_path) rules_path = os.path.join(context.subpath, rules_path) if not context.tree.has_filename(autogen_path): return False def add_autogen(mf): - rule = any(mf.iter_rules(b'override_dh_autoreconf')) + rule = any(mf.iter_rules(b"override_dh_autoreconf")) if rule: return - rule = mf.add_rule(b'override_dh_autoreconf') - rule.append_command(b'dh_autoreconf ./autogen.sh') + rule = mf.add_rule(b"override_dh_autoreconf") + rule.append_command(b"dh_autoreconf ./autogen.sh") if not update_rules(makefile_cb=add_autogen, path=rules_path): return False if context.update_changelog: commit_debian_changes( - context.tree, context.subpath, - 'Run autogen.sh during build.', committer=context.committer, - update_changelog=context.update_changelog) + context.tree, + context.subpath, + "Run autogen.sh during build.", + committer=context.committer, + update_changelog=context.update_changelog, + ) return True def _find_aclocal_fun(macro): # TODO(jelmer): Use the API for codesearch.debian.net instead? - defun_prefix = b'AC_DEFUN([%s],' % macro.encode('ascii') - for entry in os.scandir('/usr/share/aclocal'): + defun_prefix = b"AC_DEFUN([%s]," % macro.encode("ascii") + for entry in os.scandir("/usr/share/aclocal"): if not entry.is_file(): continue - with open(entry.path, 'rb') as f: + with open(entry.path, "rb") as f: for line in f: if line.startswith(defun_prefix): return entry.path @@ -1005,46 +1051,50 @@ def run_pgbuildext_updatecontrol(error, context): # TODO(jelmer): run in the schroot pg_buildext_updatecontrol(context.tree.abspath(context.subpath)) return commit_debian_changes( - context.tree, context.subpath, "Run 'pgbuildext updatecontrol'.", - committer=context.committer, update_changelog=False) + context.tree, + context.subpath, + "Run 'pgbuildext updatecontrol'.", + committer=context.committer, + update_changelog=False, + ) def fix_missing_autoconf_macro(error, context): try: path = _find_aclocal_fun(error.macro) except KeyError: - logging.info('No local m4 file found defining %s', error.macro) + logging.info("No local m4 file found defining %s", error.macro) return False package = get_package_for_paths([path]) if package is None: - logging.warning('no package for macro file %s', path) + logging.warning("no package for macro file %s", path) return False return context.add_dependency(package) def fix_missing_c_sharp_compiler(error, context): - return context.add_dependency('mono-mcs') + return context.add_dependency("mono-mcs") def fix_missing_haskell_dependencies(error, context): path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0] package = get_package_for_paths([path], regex=True) if package is None: - logging.warning('no package for macro file %s', path) + logging.warning("no package for macro file %s", path) return False return context.add_dependency(package) VERSIONED_PACKAGE_FIXERS: List[ - Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [ + Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]] +] = [ (NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol), (MissingConfigure, fix_missing_configure), (MissingAutomakeInput, fix_missing_automake_input), ] -APT_FIXERS: List[ - Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [ +APT_FIXERS: List[Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [ (MissingPythonModule, fix_missing_python_module), (MissingPythonDistribution, fix_missing_python_distribution), (MissingCHeader, fix_missing_c_header), @@ -1085,16 +1135,14 @@ def resolve_error(error, context, fixers): if isinstance(error, error_cls): relevant_fixers.append(fixer) if not relevant_fixers: - logging.warning('No fixer found for %r', error) + logging.warning("No fixer found for %r", error) return False for fixer in relevant_fixers: - logging.info( - 'Attempting to use fixer %r to address %r', - fixer, error) + logging.info("Attempting to use fixer %r to address %r", fixer, error) try: made_changes = fixer(error, context) except GeneratedFile: - logging.warning('Control file is generated, unable to edit.') + logging.warning("Control file is generated, unable to edit.") return False if made_changes: return True @@ -1102,102 +1150,139 @@ def resolve_error(error, context, fixers): def build_incrementally( - local_tree, suffix, build_suite, output_directory, build_command, - build_changelog_entry='Build for debian-janitor apt repository.', - committer=None, max_iterations=DEFAULT_MAX_ITERATIONS, - subpath='', source_date_epoch=None, update_changelog=True): + local_tree, + suffix, + build_suite, + output_directory, + build_command, + build_changelog_entry="Build for debian-janitor apt repository.", + committer=None, + max_iterations=DEFAULT_MAX_ITERATIONS, + subpath="", + source_date_epoch=None, + update_changelog=True, +): fixed_errors = [] while True: try: return attempt_build( - local_tree, suffix, build_suite, output_directory, - build_command, build_changelog_entry, subpath=subpath, - source_date_epoch=source_date_epoch) + local_tree, + suffix, + build_suite, + output_directory, + build_command, + build_changelog_entry, + subpath=subpath, + source_date_epoch=source_date_epoch, + ) except SbuildFailure as e: if e.error is None: - logging.warning( - 'Build failed with unidentified error. Giving up.') + logging.warning("Build failed with unidentified error. Giving up.") raise if e.context is None: - logging.info('No relevant context, not making any changes.') + logging.info("No relevant context, not making any changes.") raise if (e.error, e.context) in fixed_errors: - logging.warning( - 'Error was still not fixed on second try. Giving up.') + logging.warning("Error was still not fixed on second try. Giving up.") raise - if max_iterations is not None \ - and len(fixed_errors) > max_iterations: - logging.warning( - 'Last fix did not address the issue. Giving up.') + if max_iterations is not None and len(fixed_errors) > max_iterations: + logging.warning("Last fix did not address the issue. Giving up.") raise reset_tree(local_tree, local_tree.basis_tree(), subpath=subpath) - if e.context[0] == 'build': + if e.context[0] == "build": context = BuildDependencyContext( - local_tree, subpath=subpath, committer=committer, - update_changelog=update_changelog) - elif e.context[0] == 'autopkgtest': + local_tree, + subpath=subpath, + committer=committer, + update_changelog=update_changelog, + ) + elif e.context[0] == "autopkgtest": context = AutopkgtestDependencyContext( e.context[1], - local_tree, subpath=subpath, committer=committer, - update_changelog=update_changelog) + local_tree, + subpath=subpath, + committer=committer, + update_changelog=update_changelog, + ) else: - logging.warning('unable to install for context %r', e.context) + logging.warning("unable to install for context %r", e.context) raise try: if not resolve_error( - e.error, context, - VERSIONED_PACKAGE_FIXERS + APT_FIXERS): - logging.warning( - 'Failed to resolve error %r. Giving up.', e.error) + e.error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS + ): + logging.warning("Failed to resolve error %r. Giving up.", e.error) raise except CircularDependency: logging.warning( - 'Unable to fix %r; it would introduce a circular ' - 'dependency.', e.error) + "Unable to fix %r; it would introduce a circular " "dependency.", + e.error, + ) raise e fixed_errors.append((e.error, e.context)) - if os.path.exists(os.path.join(output_directory, 'build.log')): + if os.path.exists(os.path.join(output_directory, "build.log")): i = 1 while os.path.exists( - os.path.join(output_directory, 'build.log.%d' % i)): + os.path.join(output_directory, "build.log.%d" % i) + ): i += 1 - os.rename(os.path.join(output_directory, 'build.log'), - os.path.join(output_directory, 'build.log.%d' % i)) + os.rename( + os.path.join(output_directory, "build.log"), + os.path.join(output_directory, "build.log.%d" % i), + ) def main(argv=None): import argparse - parser = argparse.ArgumentParser('janitor.fix_build') - parser.add_argument('--suffix', type=str, - help="Suffix to use for test builds.", - default='fixbuild1') - parser.add_argument('--suite', type=str, - help="Suite to target.", - default='unstable') - parser.add_argument('--output-directory', type=str, - help="Output directory.", default=None) - parser.add_argument('--committer', type=str, - help='Committer string (name and email)', - default=None) + + parser = argparse.ArgumentParser("janitor.fix_build") parser.add_argument( - '--build-command', type=str, - help='Build command', - default=(DEFAULT_BUILDER + ' -A -s -v')) + "--suffix", type=str, help="Suffix to use for test builds.", default="fixbuild1" + ) parser.add_argument( - '--no-update-changelog', action="store_false", default=None, - dest="update_changelog", help="do not update the changelog") + "--suite", type=str, help="Suite to target.", default="unstable" + ) parser.add_argument( - '--update-changelog', action="store_true", dest="update_changelog", - help="force updating of the changelog", default=None) + "--output-directory", type=str, help="Output directory.", default=None + ) + parser.add_argument( + "--committer", type=str, help="Committer string (name and email)", default=None + ) + parser.add_argument( + "--build-command", + type=str, + help="Build command", + default=(DEFAULT_BUILDER + " -A -s -v"), + ) + parser.add_argument( + "--no-update-changelog", + action="store_false", + default=None, + dest="update_changelog", + help="do not update the changelog", + ) + parser.add_argument( + "--update-changelog", + action="store_true", + dest="update_changelog", + help="force updating of the changelog", + default=None, + ) args = parser.parse_args() from breezy.workingtree import WorkingTree - tree = WorkingTree.open('.') + + tree = WorkingTree.open(".") build_incrementally( - tree, args.suffix, args.suite, args.output_directory, - args.build_command, committer=args.committer, - update_changelog=args.update_changelog) + tree, + args.suffix, + args.suite, + args.output_directory, + args.build_command, + committer=args.committer, + update_changelog=args.update_changelog, + ) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 1448f36..a473c22 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -31,12 +31,8 @@ from breezy.workingtree import WorkingTree from . import DetailedFailure from .buildsystem import detect_buildsystems, NoBuildToolsFound from buildlog_consultant.common import ( - find_build_failure_description, - Problem, - MissingPerlModule, - MissingCommand, NoSpaceOnDevice, - ) +) from .session.schroot import SchrootSession from .vcs import dupe_vcs_tree, export_vcs_tree @@ -51,7 +47,7 @@ SUPPORTED_DIST_EXTENSIONS = [ ".tbz2", ".tar", ".zip", - ] +] def is_dist_file(fn): @@ -78,7 +74,6 @@ def run_dist(session): class DistCatcher(object): - def __init__(self, directory): self.export_directory = directory self.files = [] @@ -94,7 +89,7 @@ class DistCatcher(object): diff = set([n for n in diff_files if is_dist_file(n)]) if len(diff) == 1: fn = diff.pop() - logging.info('Found tarball %s in package directory.', fn) + logging.info("Found tarball %s in package directory.", fn) self.files.append(os.path.join(self.export_directory, fn)) return fn if "dist" in diff_files: @@ -103,13 +98,13 @@ class DistCatcher(object): logging.info("Found tarball %s in dist directory.", entry.name) self.files.append(entry.path) return entry.name - logging.info('No tarballs found in dist directory.') + logging.info("No tarballs found in dist directory.") parent_directory = os.path.dirname(self.export_directory) diff = set(os.listdir(parent_directory)) - set([subdir]) if len(diff) == 1: fn = diff.pop() - logging.info('Found tarball %s in parent directory.', fn) + logging.info("Found tarball %s in parent directory.", fn) self.files.append(os.path.join(parent_directory, fn)) return fn @@ -119,25 +114,28 @@ class DistCatcher(object): def create_dist_schroot( - tree: Tree, target_dir: str, - chroot: str, packaging_tree: Optional[Tree] = None, - include_controldir: bool = True, - subdir: Optional[str] = None) -> str: + tree: Tree, + target_dir: str, + chroot: str, + packaging_tree: Optional[Tree] = None, + include_controldir: bool = True, + subdir: Optional[str] = None, +) -> str: if subdir is None: - subdir = 'package' + subdir = "package" with SchrootSession(chroot) as session: if packaging_tree is not None: from .debian import satisfy_build_deps + satisfy_build_deps(session, packaging_tree) - build_dir = os.path.join(session.location, 'build') + build_dir = os.path.join(session.location, "build") try: directory = tempfile.mkdtemp(dir=build_dir) except OSError as e: if e.errno == errno.ENOSPC: - raise DetailedFailure( - 1, ['mkdtemp'], NoSpaceOnDevice()) - reldir = '/' + os.path.relpath(directory, session.location) + raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) + reldir = "/" + os.path.relpath(directory, session.location) export_directory = os.path.join(directory, subdir) if not include_controldir: @@ -158,11 +156,11 @@ def create_dist_schroot( shutil.copy(path, target_dir) return os.path.join(target_dir, os.path.basename(path)) - logging.info('No tarball created :(') + logging.info("No tarball created :(") raise DistNoTarball() -if __name__ == '__main__': +if __name__ == "__main__": import argparse import breezy.bzr # noqa: F401 import breezy.git # noqa: F401 @@ -170,17 +168,24 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--chroot', default='unstable-amd64-sbuild', type=str, - help='Name of chroot to use') + "--chroot", + default="unstable-amd64-sbuild", + type=str, + help="Name of chroot to use", + ) parser.add_argument( - 'directory', default='.', type=str, nargs='?', - help='Directory with upstream source.') + "directory", + default=".", + type=str, + nargs="?", + help="Directory with upstream source.", + ) parser.add_argument( - '--packaging-directory', type=str, - help='Path to packaging directory.') + "--packaging-directory", type=str, help="Path to packaging directory." + ) parser.add_argument( - '--target-directory', type=str, default='..', - help='Target directory') + "--target-directory", type=str, default="..", help="Target directory" + ) args = parser.parse_args() logging.basicConfig(level=logging.INFO) @@ -189,8 +194,8 @@ if __name__ == '__main__': if args.packaging_directory: packaging_tree = WorkingTree.open(args.packaging_directory) with packaging_tree.lock_read(): - source = Deb822(packaging_tree.get_file('debian/control')) - package = source['Source'] + source = Deb822(packaging_tree.get_file("debian/control")) + package = source["Source"] subdir = package else: packaging_tree = None @@ -198,13 +203,15 @@ if __name__ == '__main__': try: ret = create_dist_schroot( - tree, subdir=subdir, + tree, + subdir=subdir, target_dir=os.path.abspath(args.target_directory), packaging_tree=packaging_tree, - chroot=args.chroot) + chroot=args.chroot, + ) except NoBuildToolsFound: - logging.info('No build tools found, falling back to simple export.') - export(tree, 'dist.tar.gz', 'tgz', None) + logging.info("No build tools found, falling back to simple export.") + export(tree, "dist.tar.gz", "tgz", None) else: - print('Created %s' % ret) + print("Created %s" % ret) sys.exit(0) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 899c34f..9b02ed6 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -23,7 +23,7 @@ from buildlog_consultant.common import ( Problem, MissingPerlModule, MissingCommand, - ) +) from . import DetailedFailure from .apt import UnidentifiedError, AptManager @@ -31,12 +31,11 @@ from .debian.fix_build import ( DependencyContext, resolve_error, APT_FIXERS, - ) +) from .session import Session, run_with_tee class SchrootDependencyContext(DependencyContext): - def __init__(self, session): self.session = session self.apt = AptManager(session) @@ -50,14 +49,14 @@ class SchrootDependencyContext(DependencyContext): def fix_perl_module_from_cpan(error, context): # TODO(jelmer): Specify -T to skip tests? context.session.check_call( - ['cpan', '-i', error.module], user='root', - env={'PERL_MM_USE_DEFAULT': '1'}) + ["cpan", "-i", error.module], user="root", env={"PERL_MM_USE_DEFAULT": "1"} + ) return True NPM_COMMAND_PACKAGES = { - 'del-cli': 'del-cli', - } + "del-cli": "del-cli", +} def fix_npm_missing_command(error, context): @@ -66,19 +65,20 @@ def fix_npm_missing_command(error, context): except KeyError: return False - context.session.check_call(['npm', '-g', 'install', package]) + context.session.check_call(["npm", "-g", "install", package]) return True GENERIC_INSTALL_FIXERS: List[ - Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [ + Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]] +] = [ (MissingPerlModule, fix_perl_module_from_cpan), (MissingCommand, fix_npm_missing_command), ] def run_with_build_fixer(session: Session, args: List[str]): - logging.info('Running %r', args) + logging.info("Running %r", args) fixed_errors = [] while True: retcode, lines = run_with_tee(session, args) @@ -86,23 +86,22 @@ def run_with_build_fixer(session: Session, args: List[str]): return offset, line, error = find_build_failure_description(lines) if error is None: - logging.warning('Build failed with unidentified error. Giving up.') + logging.warning("Build failed with unidentified error. Giving up.") if line is not None: - raise UnidentifiedError( - retcode, args, lines, secondary=(offset, line)) + raise UnidentifiedError(retcode, args, lines, secondary=(offset, line)) raise UnidentifiedError(retcode, args, lines) - logging.info('Identified error: %r', error) + logging.info("Identified error: %r", error) if error in fixed_errors: logging.warning( - 'Failed to resolve error %r, it persisted. Giving up.', - error) + "Failed to resolve error %r, it persisted. Giving up.", error + ) raise DetailedFailure(retcode, args, error) if not resolve_error( - error, SchrootDependencyContext(session), - fixers=(APT_FIXERS + GENERIC_INSTALL_FIXERS)): - logging.warning( - 'Failed to find resolution for error %r. Giving up.', - error) + error, + SchrootDependencyContext(session), + fixers=(APT_FIXERS + GENERIC_INSTALL_FIXERS), + ): + logging.warning("Failed to find resolution for error %r. Giving up.", error) raise DetailedFailure(retcode, args, error) fixed_errors.append(error) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index f8feaf0..7cdd15f 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -22,8 +22,7 @@ import subprocess class Session(object): - - def __enter__(self) -> 'Session': + def __enter__(self) -> "Session": return self def __exit__(self, exc_type, exc_val, exc_tb): @@ -37,26 +36,31 @@ class Session(object): raise NotImplementedError def check_call( - self, - argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None): + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ): raise NotImplementedError(self.check_call) def check_output( - self, - argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None) -> bytes: + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ) -> bytes: raise NotImplementedError(self.check_output) - def Popen(self, argv, cwd: Optional[str] = None, - user: Optional[str] = None, **kwargs): + def Popen( + self, argv, cwd: Optional[str] = None, user: Optional[str] = None, **kwargs + ): raise NotImplementedError(self.Popen) def call( - self, argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None): + self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None + ): raise NotImplementedError(self.call) def create_home(self) -> None: @@ -76,12 +80,11 @@ class SessionSetupFailure(Exception): def run_with_tee(session: Session, args: List[str], **kwargs): - p = session.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) + p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) contents = [] while p.poll() is None: line = p.stdout.readline() sys.stdout.buffer.write(line) sys.stdout.buffer.flush() - contents.append(line.decode('utf-8', 'surrogateescape')) + contents.append(line.decode("utf-8", "surrogateescape")) return p.returncode, contents diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 1f71b59..7a1eb6c 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -25,7 +25,7 @@ import subprocess class PlainSession(Session): """Session ignoring user.""" - location = '/' + location = "/" def create_home(self): pass @@ -34,8 +34,7 @@ class PlainSession(Session): return subprocess.check_call(args) def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): - return subprocess.Popen( - args, stdout=stdout, stderr=stderr, cwd=cwd) + return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) def exists(self, path): return os.path.exists(path) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 4e75a91..ac979b4 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -34,24 +34,30 @@ class SchrootSession(Session): def __init__(self, chroot: str): if not isinstance(chroot, str): - raise TypeError('not a valid chroot: %r' % chroot) + raise TypeError("not a valid chroot: %r" % chroot) self.chroot = chroot self._location = None self._cwd = None def _get_location(self) -> str: - return subprocess.check_output( - ['schroot', '--location', '-c', 'session:' + self.session_id - ]).strip().decode() + return ( + subprocess.check_output( + ["schroot", "--location", "-c", "session:" + self.session_id] + ) + .strip() + .decode() + ) def _end_session(self) -> None: - subprocess.check_output( - ['schroot', '-c', 'session:' + self.session_id, '-e']) + subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"]) - def __enter__(self) -> 'Session': + def __enter__(self) -> "Session": try: - self.session_id = subprocess.check_output( - ['schroot', '-c', self.chroot, '-b']).strip().decode() + self.session_id = ( + subprocess.check_output(["schroot", "-c", self.chroot, "-b"]) + .strip() + .decode() + ) except subprocess.CalledProcessError: # TODO(jelmer): Capture stderr and forward in SessionSetupFailure raise SessionSetupFailure() @@ -70,69 +76,84 @@ class SchrootSession(Session): self._location = self._get_location() return self._location - def _run_argv(self, argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None): - base_argv = ['schroot', '-r', '-c', 'session:' + self.session_id] + def _run_argv( + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ): + base_argv = ["schroot", "-r", "-c", "session:" + self.session_id] if cwd is None: cwd = self._cwd if cwd is not None: - base_argv.extend(['-d', cwd]) + base_argv.extend(["-d", cwd]) if user is not None: - base_argv.extend(['-u', user]) + base_argv.extend(["-u", user]) if env: argv = [ - 'sh', '-c', - ' '.join( - ['%s=%s ' % (key, shlex.quote(value)) - for (key, value) in env.items()] + - [shlex.quote(arg) for arg in argv])] - return base_argv + ['--'] + argv + "sh", + "-c", + " ".join( + [ + "%s=%s " % (key, shlex.quote(value)) + for (key, value) in env.items() + ] + + [shlex.quote(arg) for arg in argv] + ), + ] + return base_argv + ["--"] + argv def check_call( - self, - argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None): + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ): try: subprocess.check_call(self._run_argv(argv, cwd, user, env=env)) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv) def check_output( - self, - argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None) -> bytes: + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ) -> bytes: try: - return subprocess.check_output( - self._run_argv(argv, cwd, user, env=env)) + return subprocess.check_output(self._run_argv(argv, cwd, user, env=env)) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv) - def Popen(self, argv, cwd: Optional[str] = None, - user: Optional[str] = None, **kwargs): + def Popen( + self, argv, cwd: Optional[str] = None, user: Optional[str] = None, **kwargs + ): return subprocess.Popen(self._run_argv(argv, cwd, user), **kwargs) def call( - self, argv: List[str], cwd: Optional[str] = None, - user: Optional[str] = None): + self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None + ): return subprocess.call(self._run_argv(argv, cwd, user)) def create_home(self) -> None: """Create the user's home directory.""" - home = self.check_output( - ['sh', '-c', 'echo $HOME'], cwd='/').decode().rstrip('\n') - user = self.check_output( - ['sh', '-c', 'echo $LOGNAME'], cwd='/').decode().rstrip('\n') - logging.info('Creating directory %s', home) - self.check_call(['mkdir', '-p', home], cwd='/', user='root') - self.check_call(['chown', user, home], cwd='/', user='root') + home = ( + self.check_output(["sh", "-c", "echo $HOME"], cwd="/").decode().rstrip("\n") + ) + user = ( + self.check_output(["sh", "-c", "echo $LOGNAME"], cwd="/") + .decode() + .rstrip("\n") + ) + logging.info("Creating directory %s", home) + self.check_call(["mkdir", "-p", home], cwd="/", user="root") + self.check_call(["chown", user, home], cwd="/", user="root") def _fullpath(self, path: str) -> str: - return os.path.join( - self.location, - os.path.join(self._cwd, path).lstrip('/')) + return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/")) def exists(self, path: str) -> bool: fullpath = self._fullpath(path) diff --git a/ognibuild/tests/__init__.py b/ognibuild/tests/__init__.py index 0072367..42d25c4 100644 --- a/ognibuild/tests/__init__.py +++ b/ognibuild/tests/__init__.py @@ -22,9 +22,9 @@ import unittest def test_suite(): names = [ - 'debian_build', - 'debian_fix_build', + "debian_build", + "debian_fix_build", ] - module_names = ['ognibuild.tests.test_' + name for name in names] + module_names = ["ognibuild.tests.test_" + name for name in names] loader = unittest.TestLoader() return loader.loadTestsFromNames(module_names) diff --git a/ognibuild/tests/test_debian_build.py b/ognibuild/tests/test_debian_build.py index 1203c3d..8280946 100644 --- a/ognibuild/tests/test_debian_build.py +++ b/ognibuild/tests/test_debian_build.py @@ -22,22 +22,35 @@ from breezy.tests import TestCaseWithTransport, TestCase class AddDummyChangelogEntryTests(TestCaseWithTransport): - def test_simple(self): - tree = self.make_branch_and_tree('.') - self.build_tree_contents([('debian/', ), ('debian/changelog', """\ + tree = self.make_branch_and_tree(".") + self.build_tree_contents( + [ + ("debian/",), + ( + "debian/changelog", + """\ janitor (0.1-1) UNRELEASED; urgency=medium * Initial release. (Closes: #XXXXXX) -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 -""")]) - tree.add(['debian', 'debian/changelog']) +""", + ), + ] + ) + tree.add(["debian", "debian/changelog"]) add_dummy_changelog_entry( - tree, '', 'jan+some', 'some-fixes', 'Dummy build.', + tree, + "", + "jan+some", + "some-fixes", + "Dummy build.", timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654), - maintainer=("Jelmer Vernooij", "jelmer@debian.org")) - self.assertFileEqual("""\ + maintainer=("Jelmer Vernooij", "jelmer@debian.org"), + ) + self.assertFileEqual( + """\ janitor (0.1-1jan+some1) some-fixes; urgency=low * Dummy build. @@ -49,23 +62,39 @@ janitor (0.1-1) UNRELEASED; urgency=medium * Initial release. (Closes: #XXXXXX) -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 -""", 'debian/changelog') +""", + "debian/changelog", + ) def test_native(self): - tree = self.make_branch_and_tree('.') - self.build_tree_contents([('debian/', ), ('debian/changelog', """\ + tree = self.make_branch_and_tree(".") + self.build_tree_contents( + [ + ("debian/",), + ( + "debian/changelog", + """\ janitor (0.1) UNRELEASED; urgency=medium * Initial release. (Closes: #XXXXXX) -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 -""")]) - tree.add(['debian', 'debian/changelog']) +""", + ), + ] + ) + tree.add(["debian", "debian/changelog"]) add_dummy_changelog_entry( - tree, '', 'jan+some', 'some-fixes', 'Dummy build.', + tree, + "", + "jan+some", + "some-fixes", + "Dummy build.", timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654), - maintainer=("Jelmer Vernooij", "jelmer@debian.org")) - self.assertFileEqual("""\ + maintainer=("Jelmer Vernooij", "jelmer@debian.org"), + ) + self.assertFileEqual( + """\ janitor (0.1jan+some1) some-fixes; urgency=low * Dummy build. @@ -77,23 +106,39 @@ janitor (0.1) UNRELEASED; urgency=medium * Initial release. (Closes: #XXXXXX) -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 -""", 'debian/changelog') +""", + "debian/changelog", + ) def test_exists(self): - tree = self.make_branch_and_tree('.') - self.build_tree_contents([('debian/', ), ('debian/changelog', """\ + tree = self.make_branch_and_tree(".") + self.build_tree_contents( + [ + ("debian/",), + ( + "debian/changelog", + """\ janitor (0.1-1jan+some1) UNRELEASED; urgency=medium * Initial release. (Closes: #XXXXXX) -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 -""")]) - tree.add(['debian', 'debian/changelog']) +""", + ), + ] + ) + tree.add(["debian", "debian/changelog"]) add_dummy_changelog_entry( - tree, '', 'jan+some', 'some-fixes', 'Dummy build.', + tree, + "", + "jan+some", + "some-fixes", + "Dummy build.", timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654), - maintainer=("Jelmer Vernooij", "jelmer@debian.org")) - self.assertFileEqual("""\ + maintainer=("Jelmer Vernooij", "jelmer@debian.org"), + ) + self.assertFileEqual( + """\ janitor (0.1-1jan+some2) some-fixes; urgency=low * Dummy build. @@ -111,6 +156,5 @@ janitor (0.1-1jan+some1) UNRELEASED; urgency=medium class BuildArchitectureTests(TestCase): - def test_is_str(self): self.assertIsInstance(get_build_architecture(), str) diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index 68e1c63..0e8d716 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -28,23 +28,27 @@ from buildlog_consultant.common import ( MissingRubyFile, MissingRubyGem, MissingValaPackage, - ) +) from ..debian import fix_build from ..debian.fix_build import ( resolve_error, VERSIONED_PACKAGE_FIXERS, APT_FIXERS, BuildDependencyContext, - ) +) from breezy.tests import TestCaseWithTransport class ResolveErrorTests(TestCaseWithTransport): - def setUp(self): super(ResolveErrorTests, self).setUp() - self.tree = self.make_branch_and_tree('.') - self.build_tree_contents([('debian/', ), ('debian/control', """\ + self.tree = self.make_branch_and_tree(".") + self.build_tree_contents( + [ + ("debian/",), + ( + "debian/control", + """\ Source: blah Build-Depends: libc6 @@ -52,16 +56,23 @@ Package: python-blah Depends: ${python3:Depends} Description: A python package Foo -"""), ('debian/changelog', """\ +""", + ), + ( + "debian/changelog", + """\ blah (0.1) UNRELEASED; urgency=medium * Initial release. (Closes: #XXXXXX) -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 -""")]) - self.tree.add(['debian', 'debian/control', 'debian/changelog']) - self.tree.commit('Initial commit') - self.overrideAttr(fix_build, 'search_apt_file', self._search_apt_file) +""", + ), + ] + ) + self.tree.add(["debian", "debian/control", "debian/changelog"]) + self.tree.commit("Initial commit") + self.overrideAttr(fix_build, "search_apt_file", self._search_apt_file) self._apt_files = {} def _search_apt_file(self, path, regex=False): @@ -73,129 +84,130 @@ blah (0.1) UNRELEASED; urgency=medium if path == p: yield pkg - def resolve(self, error, context=('build', )): + def resolve(self, error, context=("build",)): context = BuildDependencyContext( - self.tree, subpath='', committer='Janitor ', - update_changelog=True) - return resolve_error( - error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS) + self.tree, + subpath="", + committer="Janitor ", + update_changelog=True, + ) + return resolve_error(error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS) def get_build_deps(self): - with open(self.tree.abspath('debian/control'), 'r') as f: - return next(Deb822.iter_paragraphs(f)).get('Build-Depends', '') + with open(self.tree.abspath("debian/control"), "r") as f: + return next(Deb822.iter_paragraphs(f)).get("Build-Depends", "") def test_missing_command_unknown(self): self._apt_files = {} - self.assertFalse(self.resolve( - MissingCommand('acommandthatdoesnotexist'))) + self.assertFalse(self.resolve(MissingCommand("acommandthatdoesnotexist"))) def test_missing_command_brz(self): self._apt_files = { - '/usr/bin/b': 'bash', - '/usr/bin/brz': 'brz', - '/usr/bin/brzier': 'bash', - } - self.assertTrue(self.resolve(MissingCommand('brz'))) - self.assertEqual('libc6, brz', self.get_build_deps()) - rev = self.tree.branch.repository.get_revision( - self.tree.branch.last_revision()) - self.assertEqual( - 'Add missing build dependency on brz.\n', - rev.message) - self.assertFalse(self.resolve(MissingCommand('brz'))) - self.assertEqual('libc6, brz', self.get_build_deps()) + "/usr/bin/b": "bash", + "/usr/bin/brz": "brz", + "/usr/bin/brzier": "bash", + } + self.assertTrue(self.resolve(MissingCommand("brz"))) + self.assertEqual("libc6, brz", self.get_build_deps()) + rev = self.tree.branch.repository.get_revision(self.tree.branch.last_revision()) + self.assertEqual("Add missing build dependency on brz.\n", rev.message) + self.assertFalse(self.resolve(MissingCommand("brz"))) + self.assertEqual("libc6, brz", self.get_build_deps()) def test_missing_command_ps(self): self._apt_files = { - '/bin/ps': 'procps', - '/usr/bin/pscal': 'xcal', + "/bin/ps": "procps", + "/usr/bin/pscal": "xcal", } - self.assertTrue(self.resolve(MissingCommand('ps'))) - self.assertEqual('libc6, procps', self.get_build_deps()) + self.assertTrue(self.resolve(MissingCommand("ps"))) + self.assertEqual("libc6, procps", self.get_build_deps()) def test_missing_ruby_file(self): self._apt_files = { - '/usr/lib/ruby/vendor_ruby/rake/testtask.rb': 'rake', - } - self.assertTrue(self.resolve(MissingRubyFile('rake/testtask'))) - self.assertEqual('libc6, rake', self.get_build_deps()) + "/usr/lib/ruby/vendor_ruby/rake/testtask.rb": "rake", + } + self.assertTrue(self.resolve(MissingRubyFile("rake/testtask"))) + self.assertEqual("libc6, rake", self.get_build_deps()) def test_missing_ruby_file_from_gem(self): self._apt_files = { - '/usr/share/rubygems-integration/all/gems/activesupport-' - '5.2.3/lib/active_support/core_ext/string/strip.rb': - 'ruby-activesupport'} - self.assertTrue(self.resolve( - MissingRubyFile('active_support/core_ext/string/strip'))) - self.assertEqual('libc6, ruby-activesupport', self.get_build_deps()) + "/usr/share/rubygems-integration/all/gems/activesupport-" + "5.2.3/lib/active_support/core_ext/string/strip.rb": "ruby-activesupport" + } + self.assertTrue( + self.resolve(MissingRubyFile("active_support/core_ext/string/strip")) + ) + self.assertEqual("libc6, ruby-activesupport", self.get_build_deps()) def test_missing_ruby_gem(self): self._apt_files = { - '/usr/share/rubygems-integration/all/specifications/' - 'bio-1.5.2.gemspec': 'ruby-bio', - '/usr/share/rubygems-integration/all/specifications/' - 'bio-2.0.2.gemspec': 'ruby-bio', - } - self.assertTrue(self.resolve(MissingRubyGem('bio', None))) - self.assertEqual('libc6, ruby-bio', self.get_build_deps()) - self.assertTrue(self.resolve(MissingRubyGem('bio', '2.0.3'))) - self.assertEqual('libc6, ruby-bio (>= 2.0.3)', self.get_build_deps()) + "/usr/share/rubygems-integration/all/specifications/" + "bio-1.5.2.gemspec": "ruby-bio", + "/usr/share/rubygems-integration/all/specifications/" + "bio-2.0.2.gemspec": "ruby-bio", + } + self.assertTrue(self.resolve(MissingRubyGem("bio", None))) + self.assertEqual("libc6, ruby-bio", self.get_build_deps()) + self.assertTrue(self.resolve(MissingRubyGem("bio", "2.0.3"))) + self.assertEqual("libc6, ruby-bio (>= 2.0.3)", self.get_build_deps()) def test_missing_perl_module(self): - self._apt_files = { - '/usr/share/perl5/App/cpanminus/fatscript.pm': 'cpanminus'} - self.assertTrue(self.resolve(MissingPerlModule( - 'App/cpanminus/fatscript.pm', 'App::cpanminus::fatscript', [ - '/<>/blib/lib', - '/<>/blib/arch', - '/etc/perl', - '/usr/local/lib/x86_64-linux-gnu/perl/5.30.0', - '/usr/local/share/perl/5.30.0', - '/usr/lib/x86_64-linux-gnu/perl5/5.30', - '/usr/share/perl5', - '/usr/lib/x86_64-linux-gnu/perl/5.30', - '/usr/share/perl/5.30', - '/usr/local/lib/site_perl', - '/usr/lib/x86_64-linux-gnu/perl-base', - '.']))) - self.assertEqual('libc6, cpanminus', self.get_build_deps()) + self._apt_files = {"/usr/share/perl5/App/cpanminus/fatscript.pm": "cpanminus"} + self.assertTrue( + self.resolve( + MissingPerlModule( + "App/cpanminus/fatscript.pm", + "App::cpanminus::fatscript", + [ + "/<>/blib/lib", + "/<>/blib/arch", + "/etc/perl", + "/usr/local/lib/x86_64-linux-gnu/perl/5.30.0", + "/usr/local/share/perl/5.30.0", + "/usr/lib/x86_64-linux-gnu/perl5/5.30", + "/usr/share/perl5", + "/usr/lib/x86_64-linux-gnu/perl/5.30", + "/usr/share/perl/5.30", + "/usr/local/lib/site_perl", + "/usr/lib/x86_64-linux-gnu/perl-base", + ".", + ], + ) + ) + ) + self.assertEqual("libc6, cpanminus", self.get_build_deps()) def test_missing_pkg_config(self): self._apt_files = { - '/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc': - 'libxcb-xfixes0-dev'} - self.assertTrue(self.resolve(MissingPkgConfig('xcb-xfixes'))) - self.assertEqual('libc6, libxcb-xfixes0-dev', self.get_build_deps()) + "/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": "libxcb-xfixes0-dev" + } + self.assertTrue(self.resolve(MissingPkgConfig("xcb-xfixes"))) + self.assertEqual("libc6, libxcb-xfixes0-dev", self.get_build_deps()) def test_missing_pkg_config_versioned(self): self._apt_files = { - '/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc': - 'libxcb-xfixes0-dev'} - self.assertTrue(self.resolve(MissingPkgConfig('xcb-xfixes', '1.0'))) - self.assertEqual( - 'libc6, libxcb-xfixes0-dev (>= 1.0)', self.get_build_deps()) + "/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": "libxcb-xfixes0-dev" + } + self.assertTrue(self.resolve(MissingPkgConfig("xcb-xfixes", "1.0"))) + self.assertEqual("libc6, libxcb-xfixes0-dev (>= 1.0)", self.get_build_deps()) def test_missing_python_module(self): - self._apt_files = { - '/usr/lib/python3/dist-packages/m2r.py': 'python3-m2r' - } - self.assertTrue(self.resolve(MissingPythonModule('m2r'))) - self.assertEqual('libc6, python3-m2r', self.get_build_deps()) + self._apt_files = {"/usr/lib/python3/dist-packages/m2r.py": "python3-m2r"} + self.assertTrue(self.resolve(MissingPythonModule("m2r"))) + self.assertEqual("libc6, python3-m2r", self.get_build_deps()) def test_missing_go_package(self): self._apt_files = { - '/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go': - 'golang-github-chzyer-readline-dev', - } - self.assertTrue(self.resolve( - MissingGoPackage('github.com/chzyer/readline'))) + "/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go": "golang-github-chzyer-readline-dev", + } + self.assertTrue(self.resolve(MissingGoPackage("github.com/chzyer/readline"))) self.assertEqual( - 'libc6, golang-github-chzyer-readline-dev', - self.get_build_deps()) + "libc6, golang-github-chzyer-readline-dev", self.get_build_deps() + ) def test_missing_vala_package(self): self._apt_files = { - '/usr/share/vala-0.48/vapi/posix.vapi': 'valac-0.48-vapi', - } - self.assertTrue(self.resolve(MissingValaPackage('posix'))) - self.assertEqual('libc6, valac-0.48-vapi', self.get_build_deps()) + "/usr/share/vala-0.48/vapi/posix.vapi": "valac-0.48-vapi", + } + self.assertTrue(self.resolve(MissingValaPackage("posix"))) + self.assertEqual("libc6, valac-0.48-vapi", self.get_build_deps()) diff --git a/ognibuild/vcs.py b/ognibuild/vcs.py index 23994e5..b9bbf77 100644 --- a/ognibuild/vcs.py +++ b/ognibuild/vcs.py @@ -23,18 +23,17 @@ from breezy.workingtree import WorkingTree from buildlog_consultant.sbuild import ( NoSpaceOnDevice, - ) +) from . import DetailedFailure def export_vcs_tree(tree, directory): try: - export(tree, directory, 'dir', None) + export(tree, directory, "dir", None) except OSError as e: if e.errno == errno.ENOSPC: - raise DetailedFailure( - 1, ['export'], NoSpaceOnDevice()) + raise DetailedFailure(1, ["export"], NoSpaceOnDevice()) raise @@ -44,12 +43,11 @@ def dupe_vcs_tree(tree, directory): tree = tree.basis_tree() try: result = tree._repository.controldir.sprout( - directory, create_tree_if_local=True, - revision_id=tree.get_revision_id()) + directory, create_tree_if_local=True, revision_id=tree.get_revision_id() + ) except OSError as e: if e.errno == errno.ENOSPC: - raise DetailedFailure( - 1, ['sprout'], NoSpaceOnDevice()) + raise DetailedFailure(1, ["sprout"], NoSpaceOnDevice()) raise if not result.has_workingtree(): raise AssertionError