diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 1261574..d769c5e 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -18,27 +18,24 @@ import os import stat -import subprocess import sys -from typing import List -from .session import run_with_tee DEFAULT_PYTHON = 'python3' -class UnidentifiedError(Exception): - - def __init__(self, retcode, argv, lines): - self.retcode = retcode - self.argv = argv - self.lines = lines - - class NoBuildToolsFound(Exception): """No supported build tools were found.""" +class DetailedFailure(Exception): + + def __init__(self, retcode, argv, error): + self.retcode = retcode + self.argv = argv + self.error = error + + def shebang_binary(p): if not (os.stat(p).st_mode & stat.S_IEXEC): return None @@ -48,8 +45,8 @@ def shebang_binary(p): return None args = firstline[2:].split(b' ') if args[0] in (b'/usr/bin/env', b'env'): - return os.path.basename(args[1].decode()) - return os.path.basename(args[0].decode()) + return os.path.basename(args[1].decode()).strip() + return os.path.basename(args[0].decode()).strip() def note(m): @@ -60,18 +57,6 @@ def warning(m): sys.stderr.write('WARNING: %s\n' % m) -def run_apt(session, args: List[str]) -> None: - args = ['apt', '-y'] + args - retcode, lines = run_with_tee(session, args, cwd='/', user='root') - if retcode == 0: - return - raise UnidentifiedError(retcode, args, lines) - - -def apt_install(session, packages: List[str]) -> None: - run_apt(session, ['install'] + packages) - - def run_with_build_fixer(session, args): session.check_call(args) @@ -90,143 +75,3 @@ def run_test(session): def run_install(session): raise NotImplementedError - - -def run_dist(session): - # TODO(jelmer): Check $PATH rather than hardcoding? - if not os.path.exists('/usr/bin/git'): - apt_install(session, ['git']) - - # Some things want to write to the user's home directory, - # e.g. pip caches in ~/.cache - session.create_home() - - if os.path.exists('package.xml'): - apt_install(session, ['php-pear', 'php-horde-core']) - note('Found package.xml, assuming pear package.') - session.check_call(['pear', 'package']) - return - - if os.path.exists('pyproject.toml'): - import toml - with open('pyproject.toml', 'r') as pf: - pyproject = toml.load(pf) - if 'poetry' in pyproject.get('tool', []): - note('Found pyproject.toml with poetry section, ' - 'assuming poetry project.') - apt_install(session, ['python3-venv', 'python3-pip']) - session.check_call(['pip3', 'install', 'poetry'], user='root') - session.check_call(['poetry', 'build', '-f', 'sdist']) - return - - if os.path.exists('setup.py'): - note('Found setup.py, assuming python project.') - apt_install(session, ['python3', 'python3-pip']) - with open('setup.py', 'r') as f: - setup_py_contents = f.read() - try: - with open('setup.cfg', 'r') as f: - setup_cfg_contents = f.read() - except FileNotFoundError: - setup_cfg_contents = '' - if 'setuptools' in setup_py_contents: - note('Reference to setuptools found, installing.') - apt_install(session, ['python3-setuptools']) - if ('setuptools_scm' in setup_py_contents or - 'setuptools_scm' in setup_cfg_contents): - note('Reference to setuptools-scm found, installing.') - apt_install( - session, ['python3-setuptools-scm', 'git', 'mercurial']) - - # TODO(jelmer): Install setup_requires - - interpreter = shebang_binary('setup.py') - if interpreter is not None: - if interpreter == 'python2' or interpreter.startswith('python2.'): - apt_install(session, [interpreter]) - elif (interpreter == 'python3' or - interpreter.startswith('python3.')): - apt_install(session, [interpreter]) - else: - apt_install(session, [DEFAULT_PYTHON]) - run_with_build_fixer(session, ['./setup.py', 'sdist']) - else: - # Just assume it's Python 3 - apt_install(session, ['python3']) - run_with_build_fixer(session, ['python3', './setup.py', 'sdist']) - return - - if os.path.exists('setup.cfg'): - note('Found setup.cfg, assuming python project.') - apt_install(session, ['python3-pep517', 'python3-pip']) - session.check_call(['python3', '-m', 'pep517.build', '-s', '.']) - return - - if os.path.exists('dist.ini') and not os.path.exists('Makefile.PL'): - apt_install(session, ['libdist-inkt-perl']) - with open('dist.ini', 'rb') as f: - for line in f: - if not line.startswith(b';;'): - continue - try: - (key, value) = line[2:].split(b'=', 1) - except ValueError: - continue - if (key.strip() == b'class' and - value.strip().startswith(b"'Dist::Inkt")): - note('Found Dist::Inkt section in dist.ini, ' - 'assuming distinkt.') - # TODO(jelmer): install via apt if possible - session.check_call( - ['cpan', 'install', value.decode().strip("'")], - user='root') - run_with_build_fixer(session, ['distinkt-dist']) - return - # Default to invoking Dist::Zilla - note('Found dist.ini, assuming dist-zilla.') - apt_install(session, ['libdist-zilla-perl']) - run_with_build_fixer(session, ['dzil', 'build', '--in', '..']) - return - - if os.path.exists('package.json'): - apt_install(session, ['npm']) - run_with_build_fixer(session, ['npm', 'pack']) - return - - gemfiles = [name for name in os.listdir('.') if name.endswith('.gem')] - if gemfiles: - apt_install(session, ['gem2deb']) - if len(gemfiles) > 1: - warning('More than one gemfile. Trying the first?') - run_with_build_fixer(session, ['gem2tgz', gemfiles[0]]) - return - - if os.path.exists('waf'): - apt_install(session, ['python3']) - run_with_build_fixer(session, ['./waf', 'dist']) - return - - if os.path.exists('Makefile.PL') and not os.path.exists('Makefile'): - apt_install(session, ['perl']) - run_with_build_fixer(session, ['perl', 'Makefile.PL']) - - if not os.path.exists('Makefile') and not os.path.exists('configure'): - if os.path.exists('autogen.sh'): - if shebang_binary('autogen.sh') is None: - run_with_build_fixer(session, ['/bin/sh', './autogen.sh']) - else: - run_with_build_fixer(session, ['./autogen.sh']) - - elif os.path.exists('configure.ac') or os.path.exists('configure.in'): - apt_install(session, [ - 'autoconf', 'automake', 'gettext', 'libtool', 'gnu-standards']) - run_with_build_fixer(session, ['autoreconf', '-i']) - - if not os.path.exists('Makefile') and os.path.exists('configure'): - session.check_call(['./configure']) - - if os.path.exists('Makefile'): - apt_install(session, ['make']) - run_with_build_fixer(session, ['make', 'dist']) - - raise NoBuildToolsFound() diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 2571abc..f4b6ba1 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -18,9 +18,10 @@ import os import sys from . import ( - run_dist, run_build, run_clean, run_install, run_test, NoBuildToolsFound, + run_build, run_clean, run_install, run_test, NoBuildToolsFound, note ) +from .dist import run_dist def main(): diff --git a/ognibuild/apt.py b/ognibuild/apt.py index 7454307..90510fc 100644 --- a/ognibuild/apt.py +++ b/ognibuild/apt.py @@ -17,6 +17,16 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from typing import List + +from buildlog_consultant.sbuild import ( + find_apt_get_failure, + ) + +from . import DetailedFailure +from .session import Session, run_with_tee + + class UnidentifiedError(Exception): def __init__(self, retcode, argv, lines, secondary=None): @@ -24,3 +34,28 @@ class UnidentifiedError(Exception): self.argv = argv self.lines = lines self.secondary = secondary + + +def run_apt(session: Session, args: List[str]) -> None: + """Run apt.""" + args = ['apt', '-y'] + args + retcode, lines = run_with_tee(session, args, cwd='/', user='root') + if retcode == 0: + return + offset, line, error = find_apt_get_failure(lines) + if error is not None: + raise DetailedFailure(retcode, args, error) + if line is not None: + raise UnidentifiedError( + retcode, args, lines, secondary=(offset, line)) + while lines and lines[-1] == '': + lines.pop(-1) + raise UnidentifiedError(retcode, args, lines) + + +def install(session: Session, packages: List[str]) -> None: + run_apt(session, ['install'] + packages) + + +def satisfy(session: Session, deps: List[str]) -> None: + run_apt(session, ['satisfy'] + deps) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 2213076..ee61f77 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -20,7 +20,6 @@ import logging import os import re import shutil -import subprocess import sys import tempfile from typing import Optional, List, Tuple, Callable, Type @@ -33,45 +32,21 @@ from breezy.workingtree import WorkingTree from breezy.plugins.debian.repack_tarball import get_filetype -from .session import run_with_tee +from . import apt, DetailedFailure, shebang_binary +from .session import run_with_tee, Session +from .session.schroot import SchrootSession from .debian.fix_build import ( DependencyContext, resolve_error, APT_FIXERS, ) from buildlog_consultant.sbuild import ( - find_apt_get_failure, find_build_failure_description, Problem, MissingPerlModule, MissingCommand, NoSpaceOnDevice, ) -from ognibuild import shebang_binary -from ognibuild.session import Session -from ognibuild.session.schroot import SchrootSession - - -def run_apt(session: Session, args: List[str]) -> None: - args = ['apt', '-y'] + args - retcode, lines = run_with_tee(session, args, cwd='/', user='root') - if retcode == 0: - return - offset, line, error = find_apt_get_failure(lines) - if error is not None: - raise DetailedDistCommandFailed(retcode, args, error) - if line is not None: - raise UnidentifiedError( - retcode, args, lines, secondary=(offset, line)) - raise UnidentifiedError(retcode, args, lines) - - -def apt_install(session: Session, packages: List[str]) -> None: - run_apt(session, ['install'] + packages) - - -def apt_satisfy(session: Session, deps: List[str]) -> None: - run_apt(session, ['satisfy'] + deps) def satisfy_build_deps(session: Session, tree): @@ -91,7 +66,7 @@ def satisfy_build_deps(session: Session, tree): deps = [ dep.strip().strip(',') for dep in deps] - apt_satisfy(session, deps) + apt.satisfy(session, deps) class SchrootDependencyContext(DependencyContext): @@ -101,27 +76,10 @@ class SchrootDependencyContext(DependencyContext): def add_dependency(self, package, minimum_version=None): # TODO(jelmer): Handle minimum_version - apt_install(self.session, [package]) + apt.install(self.session, [package]) return True -class DetailedDistCommandFailed(Exception): - - def __init__(self, retcode, argv, error): - self.retcode = retcode - self.argv = argv - self.error = error - - -class UnidentifiedError(Exception): - - def __init__(self, retcode, argv, lines, secondary=None): - self.retcode = retcode - self.argv = argv - self.lines = lines - self.secondary = secondary - - def fix_perl_module_from_cpan(error, context): # TODO(jelmer): Specify -T to skip tests? context.session.check_call( @@ -163,23 +121,23 @@ def run_with_build_fixer(session: Session, args: List[str]): if error is None: logging.warning('Build failed with unidentified error. Giving up.') if line is not None: - raise UnidentifiedError( + raise apt.UnidentifiedError( retcode, args, lines, secondary=(offset, line)) - raise UnidentifiedError(retcode, args, lines) + raise apt.UnidentifiedError(retcode, args, lines) logging.info('Identified error: %r', error) if error in fixed_errors: logging.warning( 'Failed to resolve error %r, it persisted. Giving up.', error) - raise DetailedDistCommandFailed(retcode, args, error) + raise DetailedFailure(retcode, args, error) if not resolve_error( error, SchrootDependencyContext(session), fixers=(APT_FIXERS + GENERIC_INSTALL_FIXERS)): logging.warning( 'Failed to find resolution for error %r. Giving up.', error) - raise DetailedDistCommandFailed(retcode, args, error) + raise DetailedFailure(retcode, args, error) fixed_errors.append(error) @@ -187,15 +145,15 @@ class NoBuildToolsFound(Exception): """No supported build tools were found.""" -def run_dist_in_chroot(session): - apt_install(session, ['git']) +def run_dist(session): + apt.install(session, ['git']) # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() if os.path.exists('package.xml'): - apt_install(session, ['php-pear', 'php-horde-core']) + apt.install(session, ['php-pear', 'php-horde-core']) logging.info('Found package.xml, assuming pear package.') session.check_call(['pear', 'package']) return @@ -208,14 +166,14 @@ def run_dist_in_chroot(session): logging.info( 'Found pyproject.toml with poetry section, ' 'assuming poetry project.') - apt_install(session, ['python3-venv', 'python3-pip']) + apt.install(session, ['python3-venv', 'python3-pip']) session.check_call(['pip3', 'install', 'poetry'], user='root') session.check_call(['poetry', 'build', '-f', 'sdist']) return if os.path.exists('setup.py'): logging.info('Found setup.py, assuming python project.') - apt_install(session, ['python3', 'python3-pip']) + apt.install(session, ['python3', 'python3-pip']) with open('setup.py', 'r') as f: setup_py_contents = f.read() try: @@ -225,11 +183,11 @@ def run_dist_in_chroot(session): setup_cfg_contents = '' if 'setuptools' in setup_py_contents: logging.info('Reference to setuptools found, installing.') - apt_install(session, ['python3-setuptools']) + apt.install(session, ['python3-setuptools']) if ('setuptools_scm' in setup_py_contents or 'setuptools_scm' in setup_cfg_contents): logging.info('Reference to setuptools-scm found, installing.') - apt_install( + apt.install( session, ['python3-setuptools-scm', 'git', 'mercurial']) # TODO(jelmer): Install setup_requires @@ -237,29 +195,29 @@ def run_dist_in_chroot(session): interpreter = shebang_binary('setup.py') if interpreter is not None: if interpreter == 'python3': - apt_install(session, ['python3']) + apt.install(session, ['python3']) elif interpreter == 'python2': - apt_install(session, ['python2']) + apt.install(session, ['python2']) elif interpreter == 'python': - apt_install(session, ['python']) + apt.install(session, ['python']) else: - raise ValueError('Unknown interpreter %s' % interpreter) - apt_install(session, ['python2', 'python3']) + raise ValueError('Unknown interpreter %r' % interpreter) + apt.install(session, ['python2', 'python3']) run_with_build_fixer(session, ['./setup.py', 'sdist']) else: # Just assume it's Python 3 - apt_install(session, ['python3']) + apt.install(session, ['python3']) run_with_build_fixer(session, ['python3', './setup.py', 'sdist']) return if os.path.exists('setup.cfg'): logging.info('Found setup.cfg, assuming python project.') - apt_install(session, ['python3-pep517', 'python3-pip']) + apt.install(session, ['python3-pep517', 'python3-pip']) session.check_call(['python3', '-m', 'pep517.build', '-s', '.']) return if os.path.exists('dist.ini') and not os.path.exists('Makefile.PL'): - apt_install(session, ['libdist-inkt-perl']) + apt.install(session, ['libdist-inkt-perl']) with open('dist.ini', 'rb') as f: for line in f: if not line.startswith(b';;'): @@ -281,30 +239,30 @@ def run_dist_in_chroot(session): return # Default to invoking Dist::Zilla logging.info('Found dist.ini, assuming dist-zilla.') - apt_install(session, ['libdist-zilla-perl']) + apt.install(session, ['libdist-zilla-perl']) run_with_build_fixer(session, ['dzil', 'build', '--in', '..']) return if os.path.exists('package.json'): - apt_install(session, ['npm']) + apt.install(session, ['npm']) run_with_build_fixer(session, ['npm', 'pack']) return gemfiles = [name for name in os.listdir('.') if name.endswith('.gem')] if gemfiles: - apt_install(session, ['gem2deb']) + apt.install(session, ['gem2deb']) if len(gemfiles) > 1: logging.warning('More than one gemfile. Trying the first?') run_with_build_fixer(session, ['gem2tgz', gemfiles[0]]) return if os.path.exists('waf'): - apt_install(session, ['python3']) + apt.install(session, ['python3']) run_with_build_fixer(session, ['./waf', 'dist']) return if os.path.exists('Makefile.PL') and not os.path.exists('Makefile'): - apt_install(session, ['perl']) + apt.install(session, ['perl']) run_with_build_fixer(session, ['perl', 'Makefile.PL']) if not os.path.exists('Makefile') and not os.path.exists('configure'): @@ -313,7 +271,7 @@ def run_dist_in_chroot(session): run_with_build_fixer(session, ['/bin/sh', './autogen.sh']) try: run_with_build_fixer(session, ['./autogen.sh']) - except UnidentifiedError as e: + except apt.UnidentifiedError as e: if ("Gnulib not yet bootstrapped; " "run ./bootstrap instead.\n" in e.lines): run_with_build_fixer(session, ["./bootstrap"]) @@ -322,7 +280,7 @@ def run_dist_in_chroot(session): raise elif os.path.exists('configure.ac') or os.path.exists('configure.in'): - apt_install(session, [ + apt.install(session, [ 'autoconf', 'automake', 'gettext', 'libtool', 'gnu-standards']) run_with_build_fixer(session, ['autoreconf', '-i']) @@ -330,10 +288,10 @@ def run_dist_in_chroot(session): session.check_call(['./configure']) if os.path.exists('Makefile'): - apt_install(session, ['make']) + apt.install(session, ['make']) try: run_with_build_fixer(session, ['make', 'dist']) - except UnidentifiedError as e: + except apt.UnidentifiedError as e: if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: pass elif ("make[1]: *** No rule to make target 'dist'. Stop.\n" @@ -376,7 +334,7 @@ def export_vcs_tree(tree, directory): export(tree, directory, 'dir', None) except OSError as e: if e.errno == errno.ENOSPC: - raise DetailedDistCommandFailed( + raise DetailedFailure( 1, ['export'], NoSpaceOnDevice()) raise @@ -391,7 +349,7 @@ def dupe_vcs_tree(tree, directory): revision_id=tree.get_revision_id()) except OSError as e: if e.errno == errno.ENOSPC: - raise DetailedDistCommandFailed( + raise DetailedFailure( 1, ['sprout'], NoSpaceOnDevice()) raise # Copy parent location - some scripts need this @@ -417,7 +375,7 @@ def create_dist_schroot( directory = tempfile.mkdtemp(dir=build_dir) except OSError as e: if e.errno == errno.ENOSPC: - raise DetailedDistCommandFailed( + raise DetailedFailure( 1, ['mkdtemp'], NoSpaceOnDevice()) reldir = '/' + os.path.relpath(directory, session.location) @@ -433,7 +391,7 @@ def create_dist_schroot( os.chdir(export_directory) try: session.chdir(os.path.join(reldir, subdir)) - run_dist_in_chroot(session) + run_dist(session) except NoBuildToolsFound: logging.info( 'No build tools found, falling back to simple export.') diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index cb03f6f..2a7388c 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -15,6 +15,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import logging import shlex import subprocess @@ -120,8 +121,9 @@ class SchrootSession(Session): def create_home(self) -> None: """Create the user's home directory.""" home = self.check_output( - ['sh', '-c', 'echo $HOME']).decode().rstrip('\n') + ['sh', '-c', 'echo $HOME'], cwd='/').decode().rstrip('\n') user = self.check_output( - ['sh', '-c', 'echo $LOGNAME']).decode().rstrip('\n') - self.check_call(['mkdir', '-p', home], user='root') - self.check_call(['chown', user, home], user='root') + ['sh', '-c', 'echo $LOGNAME'], cwd='/').decode().rstrip('\n') + logging.info('Creating directory %s', home) + self.check_call(['mkdir', '-p', home], cwd='/', user='root') + self.check_call(['chown', user, home], cwd='/', user='root')