billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix formatting.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-02-10 02:19:06 +00:00
parent 3fe7cb2a7e
commit 43dfe81f66
17 changed files with 1085 additions and 854 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.flake8 Normal file
View file

@ -0,0 +1,5 @@
[flake8]
extend-ignore = E203, E266, E501, W293, W291
max-line-length = 88
max-complexity = 18
select = B,C,E,F,W,T4,B9

15
ognibuild/__init__.py
View file

@ -21,11 +21,10 @@ import stat
import sys
DEFAULT_PYTHON = 'python3'
DEFAULT_PYTHON = "python3"
class DetailedFailure(Exception):
def __init__(self, retcode, argv, error):
self.retcode = retcode
self.argv = argv
@ -35,19 +34,19 @@ class DetailedFailure(Exception):
def shebang_binary(p):
if not (os.stat(p).st_mode & stat.S_IEXEC):
return None
with open(p, 'rb') as f:
with open(p, "rb") as f:
firstline = f.readline()
if not firstline.startswith(b'#!'):
if not firstline.startswith(b"#!"):
return None
args = firstline[2:].split(b' ')
if args[0] in (b'/usr/bin/env', b'env'):
args = firstline[2:].split(b" ")
if args[0] in (b"/usr/bin/env", b"env"):
return os.path.basename(args[1].decode()).strip()
return os.path.basename(args[0].decode()).strip()
def note(m):
sys.stdout.write('%s\n' % m)
sys.stdout.write("%s\n" % m)
def warning(m):
sys.stderr.write('WARNING: %s\n' % m)
sys.stderr.write("WARNING: %s\n" % m)

32
ognibuild/__main__.py
View file

@ -28,40 +28,44 @@ from .test import run_test
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
'subcommand', type=str,
choices=['dist', 'build', 'clean', 'test', 'install'])
"subcommand", type=str, choices=["dist", "build", "clean", "test", "install"]
)
parser.add_argument(
'--directory', '-d', type=str, help='Directory for project.',
default='.')
"--directory", "-d", type=str, help="Directory for project.", default="."
)
parser.add_argument("--schroot", type=str, help="schroot to run in.")
parser.add_argument(
'--schroot', type=str, help='schroot to run in.')
parser.add_argument(
'--resolve', choices=['explain', 'apt', 'native'],
help='What to do about missing dependencies')
"--resolve",
choices=["explain", "apt", "native"],
help="What to do about missing dependencies",
)
args = parser.parse_args()
if args.schroot:
from .session.schroot import SchrootSession
session = SchrootSession(args.schroot)
else:
from .session.plain import PlainSession
session = PlainSession()
with session:
os.chdir(args.directory)
try:
if args.subcommand == 'dist':
if args.subcommand == "dist":
run_dist(session)
if args.subcommand == 'build':
if args.subcommand == "build":
run_build(session)
if args.subcommand == 'clean':
if args.subcommand == "clean":
run_clean(session)
if args.subcommand == 'install':
if args.subcommand == "install":
run_install(session)
if args.subcommand == 'test':
if args.subcommand == "test":
run_test(session)
except NoBuildToolsFound:
note('No build tools found.')
note("No build tools found.")
return 1
return 0

26
ognibuild/apt.py
View file

@ -23,14 +23,13 @@ import apt_pkg
import os
from buildlog_consultant.apt import (
find_apt_get_failure,
)
)
from . import DetailedFailure
from .session import Session, run_with_tee
class UnidentifiedError(Exception):
def __init__(self, retcode, argv, lines, secondary=None):
self.retcode = retcode
self.argv = argv
@ -40,17 +39,16 @@ class UnidentifiedError(Exception):
def run_apt(session: Session, args: List[str]) -> None:
"""Run apt."""
args = ['apt', '-y'] + args
retcode, lines = run_with_tee(session, args, cwd='/', user='root')
args = ["apt", "-y"] + args
retcode, lines = run_with_tee(session, args, cwd="/", user="root")
if retcode == 0:
return
offset, line, error = find_apt_get_failure(lines)
if error is not None:
raise DetailedFailure(retcode, args, error)
if line is not None:
raise UnidentifiedError(
retcode, args, lines, secondary=(offset, line))
while lines and lines[-1] == '':
raise UnidentifiedError(retcode, args, lines, secondary=(offset, line))
while lines and lines[-1] == "":
lines.pop(-1)
raise UnidentifiedError(retcode, args, lines)
@ -63,23 +61,23 @@ class AptManager(object):
self.session = session
def missing(self, packages):
root = getattr(self.session, 'location', '/')
status_path = os.path.join(root, 'var/lib/dpkg/status')
root = getattr(self.session, "location", "/")
status_path = os.path.join(root, "var/lib/dpkg/status")
missing = set(packages)
with apt_pkg.TagFile(status_path) as tagf:
while missing:
tagf.step()
if not tagf.section:
break
if tagf.section['Package'] in missing:
if tagf.section['Status'] == 'install ok installed':
missing.remove(tagf.section['Package'])
if tagf.section["Package"] in missing:
if tagf.section["Status"] == "install ok installed":
missing.remove(tagf.section["Package"])
return list(missing)
def install(self, packages: List[str]) -> None:
packages = self.missing(packages)
if packages:
run_apt(self.session, ['install'] + packages)
run_apt(self.session, ["install"] + packages)
def satisfy(self, deps: List[str]) -> None:
run_apt(self.session, ['satisfy'] + deps)
run_apt(self.session, ["satisfy"] + deps)

303
ognibuild/buildsystem.py
View file

@ -52,22 +52,21 @@ class BuildSystem(object):
class Pear(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['php-pear'])
apt.install(["php-pear"])
def dist(self):
self.setup()
run_with_build_fixer(self.session, ['pear', 'package'])
run_with_build_fixer(self.session, ["pear", "package"])
def test(self):
self.setup()
run_with_build_fixer(self.session, ['pear', 'run-tests'])
run_with_build_fixer(self.session, ["pear", "run-tests"])
def build(self):
self.setup()
run_with_build_fixer(self.session, ['pear', 'build'])
run_with_build_fixer(self.session, ["pear", "build"])
def clean(self):
self.setup()
@ -75,242 +74,251 @@ class Pear(BuildSystem):
def install(self):
self.setup()
run_with_build_fixer(self.session, ['pear', 'install'])
run_with_build_fixer(self.session, ["pear", "install"])
class SetupPy(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['python3', 'python3-pip'])
with open('setup.py', 'r') as f:
apt.install(["python3", "python3-pip"])
with open("setup.py", "r") as f:
setup_py_contents = f.read()
try:
with open('setup.cfg', 'r') as f:
with open("setup.cfg", "r") as f:
setup_cfg_contents = f.read()
except FileNotFoundError:
setup_cfg_contents = ''
if 'setuptools' in setup_py_contents:
logging.info('Reference to setuptools found, installing.')
apt.install(['python3-setuptools'])
if ('setuptools_scm' in setup_py_contents or
'setuptools_scm' in setup_cfg_contents):
logging.info('Reference to setuptools-scm found, installing.')
apt.install(['python3-setuptools-scm', 'git', 'mercurial'])
setup_cfg_contents = ""
if "setuptools" in setup_py_contents:
logging.info("Reference to setuptools found, installing.")
apt.install(["python3-setuptools"])
if (
"setuptools_scm" in setup_py_contents
or "setuptools_scm" in setup_cfg_contents
):
logging.info("Reference to setuptools-scm found, installing.")
apt.install(["python3-setuptools-scm", "git", "mercurial"])
# TODO(jelmer): Install setup_requires
def test(self):
self.setup()
self._run_setup(['test'])
self._run_setup(["test"])
def dist(self):
self.setup()
self._run_setup(['sdist'])
self._run_setup(["sdist"])
def clean(self):
self.setup()
self._run_setup(['clean'])
self._run_setup(["clean"])
def install(self):
self.setup()
self._run_setup(['install'])
self._run_setup(["install"])
def _run_setup(self, args):
apt = AptManager(self.session)
interpreter = shebang_binary('setup.py')
interpreter = shebang_binary("setup.py")
if interpreter is not None:
if interpreter == 'python3':
apt.install(['python3'])
elif interpreter == 'python2':
apt.install(['python2'])
elif interpreter == 'python':
apt.install(['python'])
if interpreter == "python3":
apt.install(["python3"])
elif interpreter == "python2":
apt.install(["python2"])
elif interpreter == "python":
apt.install(["python"])
else:
raise ValueError('Unknown interpreter %r' % interpreter)
apt.install(['python2', 'python3'])
run_with_build_fixer(
self.session, ['./setup.py'] + args)
raise ValueError("Unknown interpreter %r" % interpreter)
apt.install(["python2", "python3"])
run_with_build_fixer(self.session, ["./setup.py"] + args)
else:
# Just assume it's Python 3
apt.install(['python3'])
run_with_build_fixer(
self.session, ['python3', './setup.py'] + args)
apt.install(["python3"])
run_with_build_fixer(self.session, ["python3", "./setup.py"] + args)
class PyProject(BuildSystem):
def load_toml(self):
import toml
with open('pyproject.toml', 'r') as pf:
with open("pyproject.toml", "r") as pf:
return toml.load(pf)
def dist(self):
apt = AptManager(self.session)
pyproject = self.load_toml()
if 'poetry' in pyproject.get('tool', []):
if "poetry" in pyproject.get("tool", []):
logging.info(
'Found pyproject.toml with poetry section, '
'assuming poetry project.')
apt.install(['python3-venv', 'python3-pip'])
self.session.check_call(['pip3', 'install', 'poetry'], user='root')
self.session.check_call(['poetry', 'build', '-f', 'sdist'])
"Found pyproject.toml with poetry section, " "assuming poetry project."
)
apt.install(["python3-venv", "python3-pip"])
self.session.check_call(["pip3", "install", "poetry"], user="root")
self.session.check_call(["poetry", "build", "-f", "sdist"])
return
raise AssertionError('no supported section in pyproject.toml')
raise AssertionError("no supported section in pyproject.toml")
class SetupCfg(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['python3-pep517', 'python3-pip'])
apt.install(["python3-pep517", "python3-pip"])
def dist(self):
self.session.check_call(['python3', '-m', 'pep517.build', '-s', '.'])
self.session.check_call(["python3", "-m", "pep517.build", "-s", "."])
class NpmPackage(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['npm'])
apt.install(["npm"])
def dist(self):
self.setup()
run_with_build_fixer(self.session, ['npm', 'pack'])
run_with_build_fixer(self.session, ["npm", "pack"])
class Waf(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['python3'])
apt.install(["python3"])
def dist(self):
self.setup()
run_with_build_fixer(self.session, ['./waf', 'dist'])
run_with_build_fixer(self.session, ["./waf", "dist"])
class Gem(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['gem2deb'])
apt.install(["gem2deb"])
def dist(self):
self.setup()
gemfiles = [entry.name for entry in self.session.scandir('.')
if entry.name.endswith('.gem')]
gemfiles = [
entry.name
for entry in self.session.scandir(".")
if entry.name.endswith(".gem")
]
if len(gemfiles) > 1:
logging.warning('More than one gemfile. Trying the first?')
run_with_build_fixer(self.session, ['gem2tgz', gemfiles[0]])
logging.warning("More than one gemfile. Trying the first?")
run_with_build_fixer(self.session, ["gem2tgz", gemfiles[0]])
class DistInkt(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(['libdist-inkt-perl'])
apt.install(["libdist-inkt-perl"])
def dist(self):
self.setup()
apt = AptManager(self.session)
with open('dist.ini', 'rb') as f:
with open("dist.ini", "rb") as f:
for line in f:
if not line.startswith(b';;'):
if not line.startswith(b";;"):
continue
try:
(key, value) = line[2:].split(b'=', 1)
(key, value) = line[2:].split(b"=", 1)
except ValueError:
continue
if (key.strip() == b'class' and
value.strip().startswith(b"'Dist::Inkt")):
if key.strip() == b"class" and value.strip().startswith(b"'Dist::Inkt"):
logging.info(
'Found Dist::Inkt section in dist.ini, '
'assuming distinkt.')
"Found Dist::Inkt section in dist.ini, " "assuming distinkt."
)
# TODO(jelmer): install via apt if possible
self.session.check_call(
['cpan', 'install', value.decode().strip("'")],
user='root')
run_with_build_fixer(self.session, ['distinkt-dist'])
["cpan", "install", value.decode().strip("'")], user="root"
)
run_with_build_fixer(self.session, ["distinkt-dist"])
return
# Default to invoking Dist::Zilla
logging.info('Found dist.ini, assuming dist-zilla.')
apt.install(['libdist-zilla-perl'])
run_with_build_fixer(self.session, ['dzil', 'build', '--in', '..'])
logging.info("Found dist.ini, assuming dist-zilla.")
apt.install(["libdist-zilla-perl"])
run_with_build_fixer(self.session, ["dzil", "build", "--in", ".."])
class Make(BuildSystem):
def setup(self):
apt = AptManager(self.session)
if self.session.exists('Makefile.PL') and not self.session.exists('Makefile'):
apt.install(['perl'])
run_with_build_fixer(self.session, ['perl', 'Makefile.PL'])
if self.session.exists("Makefile.PL") and not self.session.exists("Makefile"):
apt.install(["perl"])
run_with_build_fixer(self.session, ["perl", "Makefile.PL"])
if not self.session.exists('Makefile') and not self.session.exists('configure'):
if self.session.exists('autogen.sh'):
if shebang_binary('autogen.sh') is None:
run_with_build_fixer(
self.session, ['/bin/sh', './autogen.sh'])
if not self.session.exists("Makefile") and not self.session.exists("configure"):
if self.session.exists("autogen.sh"):
if shebang_binary("autogen.sh") is None:
run_with_build_fixer(self.session, ["/bin/sh", "./autogen.sh"])
try:
run_with_build_fixer(
self.session, ['./autogen.sh'])
run_with_build_fixer(self.session, ["./autogen.sh"])
except UnidentifiedError as e:
if ("Gnulib not yet bootstrapped; "
"run ./bootstrap instead.\n" in e.lines):
if (
"Gnulib not yet bootstrapped; "
"run ./bootstrap instead.\n" in e.lines
):
run_with_build_fixer(self.session, ["./bootstrap"])
run_with_build_fixer(self.session, ['./autogen.sh'])
run_with_build_fixer(self.session, ["./autogen.sh"])
else:
raise
elif (self.session.exists('configure.ac') or
self.session.exists('configure.in')):
apt.install([
'autoconf', 'automake', 'gettext', 'libtool',
'gnu-standards'])
run_with_build_fixer(self.session, ['autoreconf', '-i'])
elif self.session.exists("configure.ac") or self.session.exists(
"configure.in"
):
apt.install(
["autoconf", "automake", "gettext", "libtool", "gnu-standards"]
)
run_with_build_fixer(self.session, ["autoreconf", "-i"])
if not self.session.exists('Makefile') and self.session.exists('configure'):
self.session.check_call(['./configure'])
if not self.session.exists("Makefile") and self.session.exists("configure"):
self.session.check_call(["./configure"])
def dist(self):
self.setup()
apt = AptManager(self.session)
apt.install(['make'])
apt.install(["make"])
try:
run_with_build_fixer(self.session, ['make', 'dist'])
run_with_build_fixer(self.session, ["make", "dist"])
except UnidentifiedError as e:
if ("make: *** No rule to make target 'dist'. Stop.\n"
in e.lines):
if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines:
pass
elif ("make[1]: *** No rule to make target 'dist'. Stop.\n"
in e.lines):
elif "make[1]: *** No rule to make target 'dist'. Stop.\n" in e.lines:
pass
elif ("Reconfigure the source tree "
"(via './config' or 'perl Configure'), please.\n"
) in e.lines:
run_with_build_fixer(self.session, ['./config'])
run_with_build_fixer(self.session, ['make', 'dist'])
elif (
"Please try running 'make manifest' and then run "
"'make dist' again.\n" in e.lines):
run_with_build_fixer(self.session, ['make', 'manifest'])
run_with_build_fixer(self.session, ['make', 'dist'])
"Reconfigure the source tree "
"(via './config' or 'perl Configure'), please.\n"
) in e.lines:
run_with_build_fixer(self.session, ["./config"])
run_with_build_fixer(self.session, ["make", "dist"])
elif (
"Please try running 'make manifest' and then run "
"'make dist' again.\n" in e.lines
):
run_with_build_fixer(self.session, ["make", "manifest"])
run_with_build_fixer(self.session, ["make", "dist"])
elif "Please run ./configure first\n" in e.lines:
run_with_build_fixer(self.session, ['./configure'])
run_with_build_fixer(self.session, ['make', 'dist'])
elif any([re.match(
r'Makefile:[0-9]+: \*\*\* Missing \'Make.inc\' '
r'Run \'./configure \[options\]\' and retry. Stop.\n',
line) for line in e.lines]):
run_with_build_fixer(self.session, ['./configure'])
run_with_build_fixer(self.session, ['make', 'dist'])
elif any([re.match(
r'Problem opening MANIFEST: No such file or directory '
r'at .* line [0-9]+\.', line) for line in e.lines]):
run_with_build_fixer(self.session, ['make', 'manifest'])
run_with_build_fixer(self.session, ['make', 'dist'])
run_with_build_fixer(self.session, ["./configure"])
run_with_build_fixer(self.session, ["make", "dist"])
elif any(
[
re.match(
r"Makefile:[0-9]+: \*\*\* Missing \'Make.inc\' "
r"Run \'./configure \[options\]\' and retry. Stop.\n",
line,
)
for line in e.lines
]
):
run_with_build_fixer(self.session, ["./configure"])
run_with_build_fixer(self.session, ["make", "dist"])
elif any(
[
re.match(
r"Problem opening MANIFEST: No such file or directory "
r"at .* line [0-9]+\.",
line,
)
for line in e.lines
]
):
run_with_build_fixer(self.session, ["make", "manifest"])
run_with_build_fixer(self.session, ["make", "dist"])
else:
raise
else:
@ -319,40 +327,49 @@ class Make(BuildSystem):
def detect_buildsystems(session):
"""Detect build systems."""
if session.exists('package.xml'):
logging.info('Found package.xml, assuming pear package.')
if session.exists("package.xml"):
logging.info("Found package.xml, assuming pear package.")
yield Pear(session)
if session.exists('setup.py'):
logging.info('Found setup.py, assuming python project.')
if session.exists("setup.py"):
logging.info("Found setup.py, assuming python project.")
yield SetupPy(session)
if session.exists('pyproject.toml'):
logging.info('Found pyproject.toml, assuming python project.')
if session.exists("pyproject.toml"):
logging.info("Found pyproject.toml, assuming python project.")
yield PyProject(session)
if session.exists('setup.cfg'):
logging.info('Found setup.cfg, assuming python project.')
if session.exists("setup.cfg"):
logging.info("Found setup.cfg, assuming python project.")
yield SetupCfg(session)
if session.exists('package.json'):
logging.info('Found package.json, assuming node package.')
if session.exists("package.json"):
logging.info("Found package.json, assuming node package.")
yield NpmPackage(session)
if session.exists('waf'):
logging.info('Found waf, assuming waf package.')
if session.exists("waf"):
logging.info("Found waf, assuming waf package.")
yield Waf(session)
gemfiles = [
entry.name for entry in session.scandir('.')
if entry.name.endswith('.gem')]
entry.name for entry in session.scandir(".") if entry.name.endswith(".gem")
]
if gemfiles:
yield Gem(session)
if session.exists('dist.ini') and not session.exists('Makefile.PL'):
if session.exists("dist.ini") and not session.exists("Makefile.PL"):
yield DistInkt(session)
if any([session.exists(p) for p in [
'Makefile', 'Makefile.PL', 'autogen.sh', 'configure.ac',
'configure.in']]):
if any(
[
session.exists(p)
for p in [
"Makefile",
"Makefile.PL",
"autogen.sh",
"configure.ac",
"configure.in",
]
]
):
yield Make(session)

15
ognibuild/debian/__init__.py
View file

@ -23,21 +23,18 @@ from ..session import Session
# TODO(jelmer): move this to debian/
def satisfy_build_deps(session: Session, tree):
source = Deb822(tree.get_file('debian/control'))
source = Deb822(tree.get_file("debian/control"))
deps = []
for name in ['Build-Depends', 'Build-Depends-Indep', 'Build-Depends-Arch']:
for name in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"]:
try:
deps.append(source[name].strip().strip(','))
deps.append(source[name].strip().strip(","))
except KeyError:
pass
for name in ['Build-Conflicts', 'Build-Conflicts-Indep',
'Build-Conflicts-Arch']:
for name in ["Build-Conflicts", "Build-Conflicts-Indep", "Build-Conflicts-Arch"]:
try:
deps.append('Conflicts: ' + source[name])
deps.append("Conflicts: " + source[name])
except KeyError:
pass
deps = [
dep.strip().strip(',')
for dep in deps]
deps = [dep.strip().strip(",") for dep in deps]
apt = AptManager(session)
apt.satisfy(deps)

167
ognibuild/debian/build.py
View file

@ -16,11 +16,11 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
__all__ = [
'changes_filename',
'get_build_architecture',
'add_dummy_changelog_entry',
'build',
'SbuildFailure',
"changes_filename",
"get_build_architecture",
"add_dummy_changelog_entry",
"build",
"SbuildFailure",
]
from datetime import datetime
@ -38,12 +38,12 @@ from breezy.mutabletree import MutableTree
from silver_platter.debian import (
BuildFailedError,
DEFAULT_BUILDER,
)
)
from buildlog_consultant.sbuild import (
worker_failure_from_sbuild_log,
SbuildFailure,
)
)
class MissingChangesFile(Exception):
@ -62,16 +62,24 @@ def changes_filename(package, version, arch):
def get_build_architecture():
try:
return subprocess.check_output(
['dpkg-architecture', '-qDEB_BUILD_ARCH']).strip().decode()
return (
subprocess.check_output(["dpkg-architecture", "-qDEB_BUILD_ARCH"])
.strip()
.decode()
)
except subprocess.CalledProcessError as e:
raise Exception(
"Could not find the build architecture: %s" % e)
raise Exception("Could not find the build architecture: %s" % e)
def add_dummy_changelog_entry(
tree: MutableTree, subpath: str, suffix: str, suite: str,
message: str, timestamp=None, maintainer=None):
tree: MutableTree,
subpath: str,
suffix: str,
suite: str,
message: str,
timestamp=None,
maintainer=None,
):
"""Add a dummy changelog entry to a package.
Args:
@ -80,87 +88,111 @@ def add_dummy_changelog_entry(
suite: Debian suite
message: Changelog message
"""
def add_suffix(v, suffix):
m = re.fullmatch('(.*)(' + re.escape(suffix) + ')([0-9]+)', v,)
if m:
return (m.group(1) + m.group(2) + '%d' % (int(m.group(3)) + 1))
else:
return v + suffix + '1'
path = os.path.join(subpath, 'debian', 'changelog')
def add_suffix(v, suffix):
m = re.fullmatch(
"(.*)(" + re.escape(suffix) + ")([0-9]+)",
v,
)
if m:
return m.group(1) + m.group(2) + "%d" % (int(m.group(3)) + 1)
else:
return v + suffix + "1"
path = os.path.join(subpath, "debian", "changelog")
if maintainer is None:
maintainer = get_maintainer()
if timestamp is None:
timestamp = datetime.now()
with tree.get_file(path) as f:
cl = Changelog()
cl.parse_changelog(
f, max_blocks=None, allow_empty_author=True, strict=False)
cl.parse_changelog(f, max_blocks=None, allow_empty_author=True, strict=False)
version = cl[0].version
if version.debian_revision:
version.debian_revision = add_suffix(
version.debian_revision, suffix)
version.debian_revision = add_suffix(version.debian_revision, suffix)
else:
version.upstream_version = add_suffix(
version.upstream_version, suffix)
version.upstream_version = add_suffix(version.upstream_version, suffix)
cl.new_block(
package=cl[0].package,
version=version,
urgency='low',
urgency="low",
distributions=suite,
author='%s <%s>' % maintainer,
author="%s <%s>" % maintainer,
date=format_datetime(timestamp),
changes=['', ' * ' + message, ''])
changes=["", " * " + message, ""],
)
cl_str = cl._format(allow_missing_author=True)
tree.put_file_bytes_non_atomic(path, cl_str.encode(cl._encoding))
def get_latest_changelog_version(local_tree, subpath=''):
path = osutils.pathjoin(subpath, 'debian/changelog')
def get_latest_changelog_version(local_tree, subpath=""):
path = osutils.pathjoin(subpath, "debian/changelog")
with local_tree.get_file(path) as f:
cl = Changelog(f, max_blocks=1)
return cl.package, cl.version
def build(local_tree, outf, build_command=DEFAULT_BUILDER, result_dir=None,
distribution=None, subpath='', source_date_epoch=None):
args = [sys.executable, '-m', 'breezy', 'builddeb',
'--guess-upstream-branch-url', '--builder=%s' % build_command]
def build(
local_tree,
outf,
build_command=DEFAULT_BUILDER,
result_dir=None,
distribution=None,
subpath="",
source_date_epoch=None,
):
args = [
sys.executable,
"-m",
"breezy",
"builddeb",
"--guess-upstream-branch-url",
"--builder=%s" % build_command,
]
if result_dir:
args.append('--result-dir=%s' % result_dir)
outf.write('Running %r\n' % (build_command, ))
args.append("--result-dir=%s" % result_dir)
outf.write("Running %r\n" % (build_command,))
outf.flush()
env = dict(os.environ.items())
if distribution is not None:
env['DISTRIBUTION'] = distribution
env["DISTRIBUTION"] = distribution
if source_date_epoch is not None:
env['SOURCE_DATE_EPOCH'] = '%d' % source_date_epoch
logging.info('Building debian packages, running %r.', build_command)
env["SOURCE_DATE_EPOCH"] = "%d" % source_date_epoch
logging.info("Building debian packages, running %r.", build_command)
try:
subprocess.check_call(
args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf,
env=env)
args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf, env=env
)
except subprocess.CalledProcessError:
raise BuildFailedError()
def build_once(
local_tree, build_suite, output_directory, build_command,
subpath='', source_date_epoch=None):
build_log_path = os.path.join(output_directory, 'build.log')
local_tree,
build_suite,
output_directory,
build_command,
subpath="",
source_date_epoch=None,
):
build_log_path = os.path.join(output_directory, "build.log")
try:
with open(build_log_path, 'w') as f:
build(local_tree, outf=f, build_command=build_command,
result_dir=output_directory, distribution=build_suite,
subpath=subpath, source_date_epoch=source_date_epoch)
with open(build_log_path, "w") as f:
build(
local_tree,
outf=f,
build_command=build_command,
result_dir=output_directory,
distribution=build_suite,
subpath=subpath,
source_date_epoch=source_date_epoch,
)
except BuildFailedError:
with open(build_log_path, 'rb') as f:
with open(build_log_path, "rb") as f:
raise worker_failure_from_sbuild_log(f)
(cl_package, cl_version) = get_latest_changelog_version(
local_tree, subpath)
changes_name = changes_filename(
cl_package, cl_version, get_build_architecture())
(cl_package, cl_version) = get_latest_changelog_version(local_tree, subpath)
changes_name = changes_filename(cl_package, cl_version, get_build_architecture())
changes_path = os.path.join(output_directory, changes_name)
if not os.path.exists(changes_path):
raise MissingChangesFile(changes_name)
@ -168,13 +200,19 @@ def build_once(
def gbp_dch(path):
subprocess.check_call(['gbp', 'dch'], cwd=path)
subprocess.check_call(["gbp", "dch"], cwd=path)
def attempt_build(
local_tree, suffix, build_suite, output_directory, build_command,
build_changelog_entry='Build for debian-janitor apt repository.',
subpath='', source_date_epoch=None):
local_tree,
suffix,
build_suite,
output_directory,
build_command,
build_changelog_entry="Build for debian-janitor apt repository.",
subpath="",
source_date_epoch=None,
):
"""Attempt a build, with a custom distribution set.
Args:
@ -189,8 +227,13 @@ def attempt_build(
Returns: Tuple with (changes_name, cl_version)
"""
add_dummy_changelog_entry(
local_tree, subpath, suffix, build_suite,
build_changelog_entry)
local_tree, subpath, suffix, build_suite, build_changelog_entry
)
return build_once(
local_tree, build_suite, output_directory, build_command, subpath,
source_date_epoch=source_date_epoch)
local_tree,
build_suite,
output_directory,
build_command,
subpath,
source_date_epoch=source_date_epoch,
)

783
ognibuild/debian/fix_build.py
View file

File diff suppressed because it is too large Load diff

79
ognibuild/dist.py
View file

@ -31,12 +31,8 @@ from breezy.workingtree import WorkingTree
from . import DetailedFailure
from .buildsystem import detect_buildsystems, NoBuildToolsFound
from buildlog_consultant.common import (
find_build_failure_description,
Problem,
MissingPerlModule,
MissingCommand,
NoSpaceOnDevice,
)
)
from .session.schroot import SchrootSession
from .vcs import dupe_vcs_tree, export_vcs_tree
@ -51,7 +47,7 @@ SUPPORTED_DIST_EXTENSIONS = [
".tbz2",
".tar",
".zip",
]
]
def is_dist_file(fn):
@ -78,7 +74,6 @@ def run_dist(session):
class DistCatcher(object):
def __init__(self, directory):
self.export_directory = directory
self.files = []
@ -94,7 +89,7 @@ class DistCatcher(object):
diff = set([n for n in diff_files if is_dist_file(n)])
if len(diff) == 1:
fn = diff.pop()
logging.info('Found tarball %s in package directory.', fn)
logging.info("Found tarball %s in package directory.", fn)
self.files.append(os.path.join(self.export_directory, fn))
return fn
if "dist" in diff_files:
@ -103,13 +98,13 @@ class DistCatcher(object):
logging.info("Found tarball %s in dist directory.", entry.name)
self.files.append(entry.path)
return entry.name
logging.info('No tarballs found in dist directory.')
logging.info("No tarballs found in dist directory.")
parent_directory = os.path.dirname(self.export_directory)
diff = set(os.listdir(parent_directory)) - set([subdir])
if len(diff) == 1:
fn = diff.pop()
logging.info('Found tarball %s in parent directory.', fn)
logging.info("Found tarball %s in parent directory.", fn)
self.files.append(os.path.join(parent_directory, fn))
return fn
@ -119,25 +114,28 @@ class DistCatcher(object):
def create_dist_schroot(
tree: Tree, target_dir: str,
chroot: str, packaging_tree: Optional[Tree] = None,
include_controldir: bool = True,
subdir: Optional[str] = None) -> str:
tree: Tree,
target_dir: str,
chroot: str,
packaging_tree: Optional[Tree] = None,
include_controldir: bool = True,
subdir: Optional[str] = None,
) -> str:
if subdir is None:
subdir = 'package'
subdir = "package"
with SchrootSession(chroot) as session:
if packaging_tree is not None:
from .debian import satisfy_build_deps
satisfy_build_deps(session, packaging_tree)
build_dir = os.path.join(session.location, 'build')
build_dir = os.path.join(session.location, "build")
try:
directory = tempfile.mkdtemp(dir=build_dir)
except OSError as e:
if e.errno == errno.ENOSPC:
raise DetailedFailure(
1, ['mkdtemp'], NoSpaceOnDevice())
reldir = '/' + os.path.relpath(directory, session.location)
raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice())
reldir = "/" + os.path.relpath(directory, session.location)
export_directory = os.path.join(directory, subdir)
if not include_controldir:
@ -158,11 +156,11 @@ def create_dist_schroot(
shutil.copy(path, target_dir)
return os.path.join(target_dir, os.path.basename(path))
logging.info('No tarball created :(')
logging.info("No tarball created :(")
raise DistNoTarball()
if __name__ == '__main__':
if __name__ == "__main__":
import argparse
import breezy.bzr # noqa: F401
import breezy.git # noqa: F401
@ -170,17 +168,24 @@ if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'--chroot', default='unstable-amd64-sbuild', type=str,
help='Name of chroot to use')
"--chroot",
default="unstable-amd64-sbuild",
type=str,
help="Name of chroot to use",
)
parser.add_argument(
'directory', default='.', type=str, nargs='?',
help='Directory with upstream source.')
"directory",
default=".",
type=str,
nargs="?",
help="Directory with upstream source.",
)
parser.add_argument(
'--packaging-directory', type=str,
help='Path to packaging directory.')
"--packaging-directory", type=str, help="Path to packaging directory."
)
parser.add_argument(
'--target-directory', type=str, default='..',
help='Target directory')
"--target-directory", type=str, default="..", help="Target directory"
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
@ -189,8 +194,8 @@ if __name__ == '__main__':
if args.packaging_directory:
packaging_tree = WorkingTree.open(args.packaging_directory)
with packaging_tree.lock_read():
source = Deb822(packaging_tree.get_file('debian/control'))
package = source['Source']
source = Deb822(packaging_tree.get_file("debian/control"))
package = source["Source"]
subdir = package
else:
packaging_tree = None
@ -198,13 +203,15 @@ if __name__ == '__main__':
try:
ret = create_dist_schroot(
tree, subdir=subdir,
tree,
subdir=subdir,
target_dir=os.path.abspath(args.target_directory),
packaging_tree=packaging_tree,
chroot=args.chroot)
chroot=args.chroot,
)
except NoBuildToolsFound:
logging.info('No build tools found, falling back to simple export.')
export(tree, 'dist.tar.gz', 'tgz', None)
logging.info("No build tools found, falling back to simple export.")
export(tree, "dist.tar.gz", "tgz", None)
else:
print('Created %s' % ret)
print("Created %s" % ret)
sys.exit(0)

41
ognibuild/fix_build.py
View file

@ -23,7 +23,7 @@ from buildlog_consultant.common import (
Problem,
MissingPerlModule,
MissingCommand,
)
)
from . import DetailedFailure
from .apt import UnidentifiedError, AptManager
@ -31,12 +31,11 @@ from .debian.fix_build import (
DependencyContext,
resolve_error,
APT_FIXERS,
)
)
from .session import Session, run_with_tee
class SchrootDependencyContext(DependencyContext):
def __init__(self, session):
self.session = session
self.apt = AptManager(session)
@ -50,14 +49,14 @@ class SchrootDependencyContext(DependencyContext):
def fix_perl_module_from_cpan(error, context):
# TODO(jelmer): Specify -T to skip tests?
context.session.check_call(
['cpan', '-i', error.module], user='root',
env={'PERL_MM_USE_DEFAULT': '1'})
["cpan", "-i", error.module], user="root", env={"PERL_MM_USE_DEFAULT": "1"}
)
return True
NPM_COMMAND_PACKAGES = {
'del-cli': 'del-cli',
}
"del-cli": "del-cli",
}
def fix_npm_missing_command(error, context):
@ -66,19 +65,20 @@ def fix_npm_missing_command(error, context):
except KeyError:
return False
context.session.check_call(['npm', '-g', 'install', package])
context.session.check_call(["npm", "-g", "install", package])
return True
GENERIC_INSTALL_FIXERS: List[
Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [
Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]
] = [
(MissingPerlModule, fix_perl_module_from_cpan),
(MissingCommand, fix_npm_missing_command),
]
def run_with_build_fixer(session: Session, args: List[str]):
logging.info('Running %r', args)
logging.info("Running %r", args)
fixed_errors = []
while True:
retcode, lines = run_with_tee(session, args)
@ -86,23 +86,22 @@ def run_with_build_fixer(session: Session, args: List[str]):
return
offset, line, error = find_build_failure_description(lines)
if error is None:
logging.warning('Build failed with unidentified error. Giving up.')
logging.warning("Build failed with unidentified error. Giving up.")
if line is not None:
raise UnidentifiedError(
retcode, args, lines, secondary=(offset, line))
raise UnidentifiedError(retcode, args, lines, secondary=(offset, line))
raise UnidentifiedError(retcode, args, lines)
logging.info('Identified error: %r', error)
logging.info("Identified error: %r", error)
if error in fixed_errors:
logging.warning(
'Failed to resolve error %r, it persisted. Giving up.',
error)
"Failed to resolve error %r, it persisted. Giving up.", error
)
raise DetailedFailure(retcode, args, error)
if not resolve_error(
error, SchrootDependencyContext(session),
fixers=(APT_FIXERS + GENERIC_INSTALL_FIXERS)):
logging.warning(
'Failed to find resolution for error %r. Giving up.',
error)
error,
SchrootDependencyContext(session),
fixers=(APT_FIXERS + GENERIC_INSTALL_FIXERS),
):
logging.warning("Failed to find resolution for error %r. Giving up.", error)
raise DetailedFailure(retcode, args, error)
fixed_errors.append(error)

37
ognibuild/session/__init__.py
View file

@ -22,8 +22,7 @@ import subprocess
class Session(object):
def __enter__(self) -> 'Session':
def __enter__(self) -> "Session":
return self
def __exit__(self, exc_type, exc_val, exc_tb):
@ -37,26 +36,31 @@ class Session(object):
raise NotImplementedError
def check_call(
self,
argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None):
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
):
raise NotImplementedError(self.check_call)
def check_output(
self,
argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None) -> bytes:
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
) -> bytes:
raise NotImplementedError(self.check_output)
def Popen(self, argv, cwd: Optional[str] = None,
user: Optional[str] = None, **kwargs):
def Popen(
self, argv, cwd: Optional[str] = None, user: Optional[str] = None, **kwargs
):
raise NotImplementedError(self.Popen)
def call(
self, argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None):
self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None
):
raise NotImplementedError(self.call)
def create_home(self) -> None:
@ -76,12 +80,11 @@ class SessionSetupFailure(Exception):
def run_with_tee(session: Session, args: List[str], **kwargs):
p = session.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
contents = []
while p.poll() is None:
line = p.stdout.readline()
sys.stdout.buffer.write(line)
sys.stdout.buffer.flush()
contents.append(line.decode('utf-8', 'surrogateescape'))
contents.append(line.decode("utf-8", "surrogateescape"))
return p.returncode, contents

5
ognibuild/session/plain.py
View file

@ -25,7 +25,7 @@ import subprocess
class PlainSession(Session):
"""Session ignoring user."""
location = '/'
location = "/"
def create_home(self):
pass
@ -34,8 +34,7 @@ class PlainSession(Session):
return subprocess.check_call(args)
def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None):
return subprocess.Popen(
args, stdout=stdout, stderr=stderr, cwd=cwd)
return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd)
def exists(self, path):
return os.path.exists(path)

111
ognibuild/session/schroot.py
View file

@ -34,24 +34,30 @@ class SchrootSession(Session):
def __init__(self, chroot: str):
if not isinstance(chroot, str):
raise TypeError('not a valid chroot: %r' % chroot)
raise TypeError("not a valid chroot: %r" % chroot)
self.chroot = chroot
self._location = None
self._cwd = None
def _get_location(self) -> str:
return subprocess.check_output(
['schroot', '--location', '-c', 'session:' + self.session_id
]).strip().decode()
return (
subprocess.check_output(
["schroot", "--location", "-c", "session:" + self.session_id]
)
.strip()
.decode()
)
def _end_session(self) -> None:
subprocess.check_output(
['schroot', '-c', 'session:' + self.session_id, '-e'])
subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"])
def __enter__(self) -> 'Session':
def __enter__(self) -> "Session":
try:
self.session_id = subprocess.check_output(
['schroot', '-c', self.chroot, '-b']).strip().decode()
self.session_id = (
subprocess.check_output(["schroot", "-c", self.chroot, "-b"])
.strip()
.decode()
)
except subprocess.CalledProcessError:
# TODO(jelmer): Capture stderr and forward in SessionSetupFailure
raise SessionSetupFailure()
@ -70,69 +76,84 @@ class SchrootSession(Session):
self._location = self._get_location()
return self._location
def _run_argv(self, argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None):
base_argv = ['schroot', '-r', '-c', 'session:' + self.session_id]
def _run_argv(
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
):
base_argv = ["schroot", "-r", "-c", "session:" + self.session_id]
if cwd is None:
cwd = self._cwd
if cwd is not None:
base_argv.extend(['-d', cwd])
base_argv.extend(["-d", cwd])
if user is not None:
base_argv.extend(['-u', user])
base_argv.extend(["-u", user])
if env:
argv = [
'sh', '-c',
' '.join(
['%s=%s ' % (key, shlex.quote(value))
for (key, value) in env.items()] +
[shlex.quote(arg) for arg in argv])]
return base_argv + ['--'] + argv
"sh",
"-c",
" ".join(
[
"%s=%s " % (key, shlex.quote(value))
for (key, value) in env.items()
]
+ [shlex.quote(arg) for arg in argv]
),
]
return base_argv + ["--"] + argv
def check_call(
self,
argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None):
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
):
try:
subprocess.check_call(self._run_argv(argv, cwd, user, env=env))
except subprocess.CalledProcessError as e:
raise subprocess.CalledProcessError(e.returncode, argv)
def check_output(
self,
argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None) -> bytes:
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
) -> bytes:
try:
return subprocess.check_output(
self._run_argv(argv, cwd, user, env=env))
return subprocess.check_output(self._run_argv(argv, cwd, user, env=env))
except subprocess.CalledProcessError as e:
raise subprocess.CalledProcessError(e.returncode, argv)
def Popen(self, argv, cwd: Optional[str] = None,
user: Optional[str] = None, **kwargs):
def Popen(
self, argv, cwd: Optional[str] = None, user: Optional[str] = None, **kwargs
):
return subprocess.Popen(self._run_argv(argv, cwd, user), **kwargs)
def call(
self, argv: List[str], cwd: Optional[str] = None,
user: Optional[str] = None):
self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None
):
return subprocess.call(self._run_argv(argv, cwd, user))
def create_home(self) -> None:
"""Create the user's home directory."""
home = self.check_output(
['sh', '-c', 'echo $HOME'], cwd='/').decode().rstrip('\n')
user = self.check_output(
['sh', '-c', 'echo $LOGNAME'], cwd='/').decode().rstrip('\n')
logging.info('Creating directory %s', home)
self.check_call(['mkdir', '-p', home], cwd='/', user='root')
self.check_call(['chown', user, home], cwd='/', user='root')
home = (
self.check_output(["sh", "-c", "echo $HOME"], cwd="/").decode().rstrip("\n")
)
user = (
self.check_output(["sh", "-c", "echo $LOGNAME"], cwd="/")
.decode()
.rstrip("\n")
)
logging.info("Creating directory %s", home)
self.check_call(["mkdir", "-p", home], cwd="/", user="root")
self.check_call(["chown", user, home], cwd="/", user="root")
def _fullpath(self, path: str) -> str:
return os.path.join(
self.location,
os.path.join(self._cwd, path).lstrip('/'))
return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/"))
def exists(self, path: str) -> bool:
fullpath = self._fullpath(path)

6
ognibuild/tests/__init__.py
View file

@ -22,9 +22,9 @@ import unittest
def test_suite():
names = [
'debian_build',
'debian_fix_build',
"debian_build",
"debian_fix_build",
]
module_names = ['ognibuild.tests.test_' + name for name in names]
module_names = ["ognibuild.tests.test_" + name for name in names]
loader = unittest.TestLoader()
return loader.loadTestsFromNames(module_names)

94
ognibuild/tests/test_debian_build.py
View file

@ -22,22 +22,35 @@ from breezy.tests import TestCaseWithTransport, TestCase
class AddDummyChangelogEntryTests(TestCaseWithTransport):
def test_simple(self):
tree = self.make_branch_and_tree('.')
self.build_tree_contents([('debian/', ), ('debian/changelog', """\
tree = self.make_branch_and_tree(".")
self.build_tree_contents(
[
("debian/",),
(
"debian/changelog",
"""\
janitor (0.1-1) UNRELEASED; urgency=medium
* Initial release. (Closes: #XXXXXX)
-- Jelmer Vernooij <jelmer@debian.org> Sat, 04 Apr 2020 14:12:13 +0000
""")])
tree.add(['debian', 'debian/changelog'])
""",
),
]
)
tree.add(["debian", "debian/changelog"])
add_dummy_changelog_entry(
tree, '', 'jan+some', 'some-fixes', 'Dummy build.',
tree,
"",
"jan+some",
"some-fixes",
"Dummy build.",
timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654),
maintainer=("Jelmer Vernooij", "jelmer@debian.org"))
self.assertFileEqual("""\
maintainer=("Jelmer Vernooij", "jelmer@debian.org"),
)
self.assertFileEqual(
"""\
janitor (0.1-1jan+some1) some-fixes; urgency=low
* Dummy build.
@ -49,23 +62,39 @@ janitor (0.1-1) UNRELEASED; urgency=medium
* Initial release. (Closes: #XXXXXX)
-- Jelmer Vernooij <jelmer@debian.org> Sat, 04 Apr 2020 14:12:13 +0000
""", 'debian/changelog')
""",
"debian/changelog",
)
def test_native(self):
tree = self.make_branch_and_tree('.')
self.build_tree_contents([('debian/', ), ('debian/changelog', """\
tree = self.make_branch_and_tree(".")
self.build_tree_contents(
[
("debian/",),
(
"debian/changelog",
"""\
janitor (0.1) UNRELEASED; urgency=medium
* Initial release. (Closes: #XXXXXX)
-- Jelmer Vernooij <jelmer@debian.org> Sat, 04 Apr 2020 14:12:13 +0000
""")])
tree.add(['debian', 'debian/changelog'])
""",
),
]
)
tree.add(["debian", "debian/changelog"])
add_dummy_changelog_entry(
tree, '', 'jan+some', 'some-fixes', 'Dummy build.',
tree,
"",
"jan+some",
"some-fixes",
"Dummy build.",
timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654),
maintainer=("Jelmer Vernooij", "jelmer@debian.org"))
self.assertFileEqual("""\
maintainer=("Jelmer Vernooij", "jelmer@debian.org"),
)
self.assertFileEqual(
"""\
janitor (0.1jan+some1) some-fixes; urgency=low
* Dummy build.
@ -77,23 +106,39 @@ janitor (0.1) UNRELEASED; urgency=medium
* Initial release. (Closes: #XXXXXX)
-- Jelmer Vernooij <jelmer@debian.org> Sat, 04 Apr 2020 14:12:13 +0000
""", 'debian/changelog')
""",
"debian/changelog",
)
def test_exists(self):
tree = self.make_branch_and_tree('.')
self.build_tree_contents([('debian/', ), ('debian/changelog', """\
tree = self.make_branch_and_tree(".")
self.build_tree_contents(
[
("debian/",),
(
"debian/changelog",
"""\
janitor (0.1-1jan+some1) UNRELEASED; urgency=medium
* Initial release. (Closes: #XXXXXX)
-- Jelmer Vernooij <jelmer@debian.org> Sat, 04 Apr 2020 14:12:13 +0000
""")])
tree.add(['debian', 'debian/changelog'])
""",
),
]
)
tree.add(["debian", "debian/changelog"])
add_dummy_changelog_entry(
tree, '', 'jan+some', 'some-fixes', 'Dummy build.',
tree,
"",
"jan+some",
"some-fixes",
"Dummy build.",
timestamp=datetime.datetime(2020, 9, 5, 12, 35, 4, 899654),
maintainer=("Jelmer Vernooij", "jelmer@debian.org"))
self.assertFileEqual("""\
maintainer=("Jelmer Vernooij", "jelmer@debian.org"),
)
self.assertFileEqual(
"""\
janitor (0.1-1jan+some2) some-fixes; urgency=low
* Dummy build.
@ -111,6 +156,5 @@ janitor (0.1-1jan+some1) UNRELEASED; urgency=medium
class BuildArchitectureTests(TestCase):
def test_is_str(self):
self.assertIsInstance(get_build_architecture(), str)

206
ognibuild/tests/test_debian_fix_build.py
View file

@ -28,23 +28,27 @@ from buildlog_consultant.common import (
MissingRubyFile,
MissingRubyGem,
MissingValaPackage,
)
)
from ..debian import fix_build
from ..debian.fix_build import (
resolve_error,
VERSIONED_PACKAGE_FIXERS,
APT_FIXERS,
BuildDependencyContext,
)
)
from breezy.tests import TestCaseWithTransport
class ResolveErrorTests(TestCaseWithTransport):
def setUp(self):
super(ResolveErrorTests, self).setUp()
self.tree = self.make_branch_and_tree('.')
self.build_tree_contents([('debian/', ), ('debian/control', """\
self.tree = self.make_branch_and_tree(".")
self.build_tree_contents(
[
("debian/",),
(
"debian/control",
"""\
Source: blah
Build-Depends: libc6
@ -52,16 +56,23 @@ Package: python-blah
Depends: ${python3:Depends}
Description: A python package
Foo
"""), ('debian/changelog', """\
""",
),
(
"debian/changelog",
"""\
blah (0.1) UNRELEASED; urgency=medium
* Initial release. (Closes: #XXXXXX)
-- Jelmer Vernooij <jelmer@debian.org> Sat, 04 Apr 2020 14:12:13 +0000
""")])
self.tree.add(['debian', 'debian/control', 'debian/changelog'])
self.tree.commit('Initial commit')
self.overrideAttr(fix_build, 'search_apt_file', self._search_apt_file)
""",
),
]
)
self.tree.add(["debian", "debian/control", "debian/changelog"])
self.tree.commit("Initial commit")
self.overrideAttr(fix_build, "search_apt_file", self._search_apt_file)
self._apt_files = {}
def _search_apt_file(self, path, regex=False):
@ -73,129 +84,130 @@ blah (0.1) UNRELEASED; urgency=medium
if path == p:
yield pkg
def resolve(self, error, context=('build', )):
def resolve(self, error, context=("build",)):
context = BuildDependencyContext(
self.tree, subpath='', committer='Janitor <janitor@jelmer.uk>',
update_changelog=True)
return resolve_error(
error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS)
self.tree,
subpath="",
committer="Janitor <janitor@jelmer.uk>",
update_changelog=True,
)
return resolve_error(error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS)
def get_build_deps(self):
with open(self.tree.abspath('debian/control'), 'r') as f:
return next(Deb822.iter_paragraphs(f)).get('Build-Depends', '')
with open(self.tree.abspath("debian/control"), "r") as f:
return next(Deb822.iter_paragraphs(f)).get("Build-Depends", "")
def test_missing_command_unknown(self):
self._apt_files = {}
self.assertFalse(self.resolve(
MissingCommand('acommandthatdoesnotexist')))
self.assertFalse(self.resolve(MissingCommand("acommandthatdoesnotexist")))
def test_missing_command_brz(self):
self._apt_files = {
'/usr/bin/b': 'bash',
'/usr/bin/brz': 'brz',
'/usr/bin/brzier': 'bash',
}
self.assertTrue(self.resolve(MissingCommand('brz')))
self.assertEqual('libc6, brz', self.get_build_deps())
rev = self.tree.branch.repository.get_revision(
self.tree.branch.last_revision())
self.assertEqual(
'Add missing build dependency on brz.\n',
rev.message)
self.assertFalse(self.resolve(MissingCommand('brz')))
self.assertEqual('libc6, brz', self.get_build_deps())
"/usr/bin/b": "bash",
"/usr/bin/brz": "brz",
"/usr/bin/brzier": "bash",
}
self.assertTrue(self.resolve(MissingCommand("brz")))
self.assertEqual("libc6, brz", self.get_build_deps())
rev = self.tree.branch.repository.get_revision(self.tree.branch.last_revision())
self.assertEqual("Add missing build dependency on brz.\n", rev.message)
self.assertFalse(self.resolve(MissingCommand("brz")))
self.assertEqual("libc6, brz", self.get_build_deps())
def test_missing_command_ps(self):
self._apt_files = {
'/bin/ps': 'procps',
'/usr/bin/pscal': 'xcal',
"/bin/ps": "procps",
"/usr/bin/pscal": "xcal",
}
self.assertTrue(self.resolve(MissingCommand('ps')))
self.assertEqual('libc6, procps', self.get_build_deps())
self.assertTrue(self.resolve(MissingCommand("ps")))
self.assertEqual("libc6, procps", self.get_build_deps())
def test_missing_ruby_file(self):
self._apt_files = {
'/usr/lib/ruby/vendor_ruby/rake/testtask.rb': 'rake',
}
self.assertTrue(self.resolve(MissingRubyFile('rake/testtask')))
self.assertEqual('libc6, rake', self.get_build_deps())
"/usr/lib/ruby/vendor_ruby/rake/testtask.rb": "rake",
}
self.assertTrue(self.resolve(MissingRubyFile("rake/testtask")))
self.assertEqual("libc6, rake", self.get_build_deps())
def test_missing_ruby_file_from_gem(self):
self._apt_files = {
'/usr/share/rubygems-integration/all/gems/activesupport-'
'5.2.3/lib/active_support/core_ext/string/strip.rb':
'ruby-activesupport'}
self.assertTrue(self.resolve(
MissingRubyFile('active_support/core_ext/string/strip')))
self.assertEqual('libc6, ruby-activesupport', self.get_build_deps())
"/usr/share/rubygems-integration/all/gems/activesupport-"
"5.2.3/lib/active_support/core_ext/string/strip.rb": "ruby-activesupport"
}
self.assertTrue(
self.resolve(MissingRubyFile("active_support/core_ext/string/strip"))
)
self.assertEqual("libc6, ruby-activesupport", self.get_build_deps())
def test_missing_ruby_gem(self):
self._apt_files = {
'/usr/share/rubygems-integration/all/specifications/'
'bio-1.5.2.gemspec': 'ruby-bio',
'/usr/share/rubygems-integration/all/specifications/'
'bio-2.0.2.gemspec': 'ruby-bio',
}
self.assertTrue(self.resolve(MissingRubyGem('bio', None)))
self.assertEqual('libc6, ruby-bio', self.get_build_deps())
self.assertTrue(self.resolve(MissingRubyGem('bio', '2.0.3')))
self.assertEqual('libc6, ruby-bio (>= 2.0.3)', self.get_build_deps())
"/usr/share/rubygems-integration/all/specifications/"
"bio-1.5.2.gemspec": "ruby-bio",
"/usr/share/rubygems-integration/all/specifications/"
"bio-2.0.2.gemspec": "ruby-bio",
}
self.assertTrue(self.resolve(MissingRubyGem("bio", None)))
self.assertEqual("libc6, ruby-bio", self.get_build_deps())
self.assertTrue(self.resolve(MissingRubyGem("bio", "2.0.3")))
self.assertEqual("libc6, ruby-bio (>= 2.0.3)", self.get_build_deps())
def test_missing_perl_module(self):
self._apt_files = {
'/usr/share/perl5/App/cpanminus/fatscript.pm': 'cpanminus'}
self.assertTrue(self.resolve(MissingPerlModule(
'App/cpanminus/fatscript.pm', 'App::cpanminus::fatscript', [
'/<<PKGBUILDDIR>>/blib/lib',
'/<<PKGBUILDDIR>>/blib/arch',
'/etc/perl',
'/usr/local/lib/x86_64-linux-gnu/perl/5.30.0',
'/usr/local/share/perl/5.30.0',
'/usr/lib/x86_64-linux-gnu/perl5/5.30',
'/usr/share/perl5',
'/usr/lib/x86_64-linux-gnu/perl/5.30',
'/usr/share/perl/5.30',
'/usr/local/lib/site_perl',
'/usr/lib/x86_64-linux-gnu/perl-base',
'.'])))
self.assertEqual('libc6, cpanminus', self.get_build_deps())
self._apt_files = {"/usr/share/perl5/App/cpanminus/fatscript.pm": "cpanminus"}
self.assertTrue(
self.resolve(
MissingPerlModule(
"App/cpanminus/fatscript.pm",
"App::cpanminus::fatscript",
[
"/<<PKGBUILDDIR>>/blib/lib",
"/<<PKGBUILDDIR>>/blib/arch",
"/etc/perl",
"/usr/local/lib/x86_64-linux-gnu/perl/5.30.0",
"/usr/local/share/perl/5.30.0",
"/usr/lib/x86_64-linux-gnu/perl5/5.30",
"/usr/share/perl5",
"/usr/lib/x86_64-linux-gnu/perl/5.30",
"/usr/share/perl/5.30",
"/usr/local/lib/site_perl",
"/usr/lib/x86_64-linux-gnu/perl-base",
".",
],
)
)
)
self.assertEqual("libc6, cpanminus", self.get_build_deps())
def test_missing_pkg_config(self):
self._apt_files = {
'/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc':
'libxcb-xfixes0-dev'}
self.assertTrue(self.resolve(MissingPkgConfig('xcb-xfixes')))
self.assertEqual('libc6, libxcb-xfixes0-dev', self.get_build_deps())
"/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": "libxcb-xfixes0-dev"
}
self.assertTrue(self.resolve(MissingPkgConfig("xcb-xfixes")))
self.assertEqual("libc6, libxcb-xfixes0-dev", self.get_build_deps())
def test_missing_pkg_config_versioned(self):
self._apt_files = {
'/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc':
'libxcb-xfixes0-dev'}
self.assertTrue(self.resolve(MissingPkgConfig('xcb-xfixes', '1.0')))
self.assertEqual(
'libc6, libxcb-xfixes0-dev (>= 1.0)', self.get_build_deps())
"/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": "libxcb-xfixes0-dev"
}
self.assertTrue(self.resolve(MissingPkgConfig("xcb-xfixes", "1.0")))
self.assertEqual("libc6, libxcb-xfixes0-dev (>= 1.0)", self.get_build_deps())
def test_missing_python_module(self):
self._apt_files = {
'/usr/lib/python3/dist-packages/m2r.py': 'python3-m2r'
}
self.assertTrue(self.resolve(MissingPythonModule('m2r')))
self.assertEqual('libc6, python3-m2r', self.get_build_deps())
self._apt_files = {"/usr/lib/python3/dist-packages/m2r.py": "python3-m2r"}
self.assertTrue(self.resolve(MissingPythonModule("m2r")))
self.assertEqual("libc6, python3-m2r", self.get_build_deps())
def test_missing_go_package(self):
self._apt_files = {
'/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go':
'golang-github-chzyer-readline-dev',
}
self.assertTrue(self.resolve(
MissingGoPackage('github.com/chzyer/readline')))
"/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go": "golang-github-chzyer-readline-dev",
}
self.assertTrue(self.resolve(MissingGoPackage("github.com/chzyer/readline")))
self.assertEqual(
'libc6, golang-github-chzyer-readline-dev',
self.get_build_deps())
"libc6, golang-github-chzyer-readline-dev", self.get_build_deps()
)
def test_missing_vala_package(self):
self._apt_files = {
'/usr/share/vala-0.48/vapi/posix.vapi': 'valac-0.48-vapi',
}
self.assertTrue(self.resolve(MissingValaPackage('posix')))
self.assertEqual('libc6, valac-0.48-vapi', self.get_build_deps())
"/usr/share/vala-0.48/vapi/posix.vapi": "valac-0.48-vapi",
}
self.assertTrue(self.resolve(MissingValaPackage("posix")))
self.assertEqual("libc6, valac-0.48-vapi", self.get_build_deps())

14
ognibuild/vcs.py
View file

@ -23,18 +23,17 @@ from breezy.workingtree import WorkingTree
from buildlog_consultant.sbuild import (
NoSpaceOnDevice,
)
)
from . import DetailedFailure
def export_vcs_tree(tree, directory):
try:
export(tree, directory, 'dir', None)
export(tree, directory, "dir", None)
except OSError as e:
if e.errno == errno.ENOSPC:
raise DetailedFailure(
1, ['export'], NoSpaceOnDevice())
raise DetailedFailure(1, ["export"], NoSpaceOnDevice())
raise
@ -44,12 +43,11 @@ def dupe_vcs_tree(tree, directory):
tree = tree.basis_tree()
try:
result = tree._repository.controldir.sprout(
directory, create_tree_if_local=True,
revision_id=tree.get_revision_id())
directory, create_tree_if_local=True, revision_id=tree.get_revision_id()
)
except OSError as e:
if e.errno == errno.ENOSPC:
raise DetailedFailure(
1, ['sprout'], NoSpaceOnDevice())
raise DetailedFailure(1, ["sprout"], NoSpaceOnDevice())
raise
if not result.has_workingtree():
raise AssertionError