From c286789e3772970f7f2b5f70564a8545e4f0f0b8 Mon Sep 17 00:00:00 2001 From: Tianyu Chen Date: Tue, 22 Nov 2022 11:19:55 +0800 Subject: [PATCH] New upstream version 0.0.15 --- .github/workflows/disperse.yml | 24 + .github/workflows/pythonpackage.yml | 27 +- .gitignore | 1 + Makefile | 20 + PKG-INFO | 17 - README.md | 6 + disperse.conf | 8 + ognibuild.egg-info/PKG-INFO | 17 - ognibuild.egg-info/SOURCES.txt | 52 - ognibuild.egg-info/dependency_links.txt | 1 - ognibuild.egg-info/entry_points.txt | 4 - ognibuild.egg-info/requires.txt | 8 - ognibuild.egg-info/top_level.txt | 1 - ognibuild/__init__.py | 70 +- ognibuild/__main__.py | 175 +++- ognibuild/build.py | 16 +- ognibuild/buildlog.py | 244 +++-- ognibuild/buildsystem.py | 903 ++++++++++++------ ognibuild/clean.py | 13 +- ognibuild/debian/__init__.py | 3 +- ognibuild/debian/apt.py | 27 +- ognibuild/debian/build.py | 195 ++-- ognibuild/debian/build_deps.py | 45 +- ognibuild/debian/file_search.py | 88 +- ognibuild/debian/fix_build.py | 284 +++--- ognibuild/debian/udd.py | 6 +- ognibuild/dep_server.py | 126 +++ ognibuild/dist.py | 218 +++-- ognibuild/dist_catcher.py | 13 +- ognibuild/fix_build.py | 49 +- ognibuild/fixers.py | 34 + ognibuild/info.py | 12 +- ognibuild/install.py | 19 +- ognibuild/logs.py | 105 ++ ognibuild/requirements.py | 325 +++++-- ognibuild/resolver/__init__.py | 153 ++- ognibuild/resolver/apt.py | 641 ++++++++----- ognibuild/resolver/dep_server.py | 88 ++ ognibuild/session/__init__.py | 22 +- ognibuild/session/plain.py | 14 +- ognibuild/session/schroot.py | 44 +- ognibuild/test.py | 13 +- ognibuild/upstream.py | 253 +++++ ognibuild/vcs.py | 3 +- pyproject.toml | 3 + releaser.conf | 14 - scripts/report-apt-deps-status | 89 ++ setup.cfg | 62 +- setup.py | 41 +- {ognibuild/tests => tests}/__init__.py | 7 +- tests/test_buildlog.py | 47 + .../tests => tests}/test_debian_build.py | 51 +- .../tests => tests}/test_debian_fix_build.py | 106 +- tests/test_logs.py | 95 ++ tests/test_resolver_apt.py | 47 + 55 files changed, 3578 insertions(+), 1371 deletions(-) create mode 100644 .github/workflows/disperse.yml create mode 100644 Makefile delete mode 100644 PKG-INFO create mode 100644 disperse.conf delete mode 100644 ognibuild.egg-info/PKG-INFO delete mode 100644 ognibuild.egg-info/SOURCES.txt delete mode 100644 ognibuild.egg-info/dependency_links.txt delete mode 100644 ognibuild.egg-info/entry_points.txt delete mode 100644 ognibuild.egg-info/requires.txt delete mode 100644 ognibuild.egg-info/top_level.txt create mode 100644 ognibuild/dep_server.py create mode 100644 ognibuild/logs.py create mode 100644 ognibuild/resolver/dep_server.py create mode 100644 ognibuild/upstream.py create mode 100644 pyproject.toml delete mode 100644 releaser.conf create mode 100755 scripts/report-apt-deps-status rename {ognibuild/tests => tests}/__init__.py (85%) create mode 100644 tests/test_buildlog.py rename {ognibuild/tests => tests}/test_debian_build.py (72%) rename {ognibuild/tests => tests}/test_debian_fix_build.py (72%) create mode 100644 tests/test_logs.py create mode 100644 tests/test_resolver_apt.py diff --git a/.github/workflows/disperse.yml b/.github/workflows/disperse.yml new file mode 100644 index 0000000..b65bb01 --- /dev/null +++ b/.github/workflows/disperse.yml @@ -0,0 +1,24 @@ +--- +name: Disperse configuration + +"on": + - push + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install dependencies + run: | + sudo apt install protobuf-compiler + - name: Install disperse + run: | + pip install git+https://github.com/jelmer/disperse + - name: Validate disperse.conf + run: | + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python disperse validate . diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 0725c4c..1d6b7d4 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -1,6 +1,11 @@ +--- name: Python package -on: [push, pull_request] +"on": + push: + pull_request: + schedule: + - cron: '0 6 * * *' # Daily 6AM UTC build jobs: build: @@ -9,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - python-version: [3.7, 3.8] + python-version: [3.7, 3.8, 3.9, '3.10'] fail-fast: false steps: @@ -20,28 +25,28 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip flake8 cython + python -m pip install --upgrade pip + python -m pip install -e ".[remote,dep_server,dev]" python setup.py develop - name: Install Debian-specific dependencies run: | - sudo apt install libapt-pkg-dev - python -m pip install wheel - python -m pip install git+https://salsa.debian.org/apt-team/python-apt + sudo apt update + sudo apt install python3-wheel libapt-pkg-dev + python -m pip install \ + python_apt@git+https://salsa.debian.org/apt-team/python-apt.git python -m pip install -e ".[debian]" - python -m pip install testtools - mkdir -p ~/.config/breezy/plugins - brz branch lp:brz-debian ~/.config/breezy/plugins/debian if: "matrix.python-version != 'pypy3' && matrix.os == 'ubuntu-latest'" - name: Style checks run: | + pip install flake8 python -m flake8 - name: Typing checks run: | - pip install -U mypy + pip install -U mypy types-toml python -m mypy ognibuild if: "matrix.python-version != 'pypy3'" - name: Test suite run run: | - python -m unittest ognibuild.tests.test_suite + python -m unittest tests.test_suite env: PYTHONHASHSEED: random diff --git a/.gitignore b/.gitignore index 13818ea..9bbe794 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.coverage build *~ ognibuild.egg-info diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0297c36 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +check:: style + +style: + flake8 + +check:: testsuite + +testsuite: + python3 -m unittest tests.test_suite + +check:: typing + +typing: + mypy ognibuild tests + +coverage: + python3 -m coverage run -m unittest tests.test_suite + +coverage-html: + python3 -m coverage html diff --git a/PKG-INFO b/PKG-INFO deleted file mode 100644 index 8ab0d39..0000000 --- a/PKG-INFO +++ /dev/null @@ -1,17 +0,0 @@ -Metadata-Version: 2.1 -Name: ognibuild -Version: 0.0.7 -Summary: Detect and run any build system -Home-page: https://jelmer.uk/code/ognibuild -Maintainer: Jelmer Vernooij -Maintainer-email: jelmer@jelmer.uk -License: GNU GPLv2 or later -Description: UNKNOWN -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Operating System :: POSIX -Provides-Extra: debian diff --git a/README.md b/README.md index 5374ac2..b9e493f 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ Ognibuild has a number of subcommands: It also includes a subcommand that can fix up the build dependencies for Debian packages, called deb-fix-build. +### Examples + +``` +ogni -d https://gitlab.gnome.org/GNOME/fractal install +``` + ## Status Ognibuild is functional, but sometimes rough around the edges. If you run into diff --git a/disperse.conf b/disperse.conf new file mode 100644 index 0000000..4e8de5b --- /dev/null +++ b/disperse.conf @@ -0,0 +1,8 @@ +# See https://github.com/jelmer/disperse +timeout_days: 5 +tag_name: "v$VERSION" +verify_command: "python3 -m unittest tests.test_suite" +update_version { + path: "ognibuild/__init__.py" + new_line: "__version__ = $TUPLED_VERSION" +} diff --git a/ognibuild.egg-info/PKG-INFO b/ognibuild.egg-info/PKG-INFO deleted file mode 100644 index 8ab0d39..0000000 --- a/ognibuild.egg-info/PKG-INFO +++ /dev/null @@ -1,17 +0,0 @@ -Metadata-Version: 2.1 -Name: ognibuild -Version: 0.0.7 -Summary: Detect and run any build system -Home-page: https://jelmer.uk/code/ognibuild -Maintainer: Jelmer Vernooij -Maintainer-email: jelmer@jelmer.uk -License: GNU GPLv2 or later -Description: UNKNOWN -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Operating System :: POSIX -Provides-Extra: debian diff --git a/ognibuild.egg-info/SOURCES.txt b/ognibuild.egg-info/SOURCES.txt deleted file mode 100644 index 2067e75..0000000 --- a/ognibuild.egg-info/SOURCES.txt +++ /dev/null @@ -1,52 +0,0 @@ -.flake8 -.gitignore -AUTHORS -CODE_OF_CONDUCT.md -LICENSE -README.md -SECURITY.md -TODO -releaser.conf -setup.cfg -setup.py -.github/workflows/pythonpackage.yml -notes/architecture.md -notes/concepts.md -notes/roadmap.md -ognibuild/__init__.py -ognibuild/__main__.py -ognibuild/build.py -ognibuild/buildlog.py -ognibuild/buildsystem.py -ognibuild/clean.py -ognibuild/dist.py -ognibuild/dist_catcher.py -ognibuild/fix_build.py -ognibuild/fixers.py -ognibuild/info.py -ognibuild/install.py -ognibuild/outputs.py -ognibuild/requirements.py -ognibuild/test.py -ognibuild/vcs.py -ognibuild.egg-info/PKG-INFO -ognibuild.egg-info/SOURCES.txt -ognibuild.egg-info/dependency_links.txt -ognibuild.egg-info/entry_points.txt -ognibuild.egg-info/requires.txt -ognibuild.egg-info/top_level.txt -ognibuild/debian/__init__.py -ognibuild/debian/apt.py -ognibuild/debian/build.py -ognibuild/debian/build_deps.py -ognibuild/debian/file_search.py -ognibuild/debian/fix_build.py -ognibuild/debian/udd.py -ognibuild/resolver/__init__.py -ognibuild/resolver/apt.py -ognibuild/session/__init__.py -ognibuild/session/plain.py -ognibuild/session/schroot.py -ognibuild/tests/__init__.py -ognibuild/tests/test_debian_build.py -ognibuild/tests/test_debian_fix_build.py \ No newline at end of file diff --git a/ognibuild.egg-info/dependency_links.txt b/ognibuild.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/ognibuild.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ognibuild.egg-info/entry_points.txt b/ognibuild.egg-info/entry_points.txt deleted file mode 100644 index dc61857..0000000 --- a/ognibuild.egg-info/entry_points.txt +++ /dev/null @@ -1,4 +0,0 @@ -[console_scripts] -deb-fix-build = ognibuild.debian.fix_build:main -ogni = ognibuild.__main__:main - diff --git a/ognibuild.egg-info/requires.txt b/ognibuild.egg-info/requires.txt deleted file mode 100644 index 9c36b12..0000000 --- a/ognibuild.egg-info/requires.txt +++ /dev/null @@ -1,8 +0,0 @@ -breezy -buildlog-consultant>=0.0.10 -requirements-parser - -[debian] -debmutate -python_apt -python_debian diff --git a/ognibuild.egg-info/top_level.txt b/ognibuild.egg-info/top_level.txt deleted file mode 100644 index 7e9084f..0000000 --- a/ognibuild.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -ognibuild diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 22dfaec..bc255e1 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -18,12 +18,14 @@ import os import stat +from typing import List, Dict, Type -__version__ = (0, 0, 7) +__version__ = (0, 0, 15) +version_string = '.'.join(map(str, __version__)) -USER_AGENT = "Ognibuild" +USER_AGENT = f"Ognibuild/{version_string}" class DetailedFailure(Exception): @@ -32,6 +34,12 @@ class DetailedFailure(Exception): self.argv = argv self.error = error + def __eq__(self, other): + return (isinstance(other, type(self)) and + self.retcode == other.retcode and + self.argv == other.argv and + self.error == other.error) + class UnidentifiedError(Exception): """An unidentified error.""" @@ -42,6 +50,13 @@ class UnidentifiedError(Exception): self.lines = lines self.secondary = secondary + def __eq__(self, other): + return (isinstance(other, type(self)) and + self.retcode == other.retcode and + self.argv == other.argv and + self.lines == other.lines and + self.secondary == other.secondary) + def __repr__(self): return "<%s(%r, %r, ..., secondary=%r)>" % ( type(self).__name__, @@ -64,17 +79,64 @@ def shebang_binary(p): return os.path.basename(args[0].decode()).strip() +class UnknownRequirementFamily(Exception): + """Requirement family is unknown""" + + def __init__(self, family): + self.family = family + + class Requirement(object): # Name of the family of requirements - e.g. "python-package" family: str - def __init__(self, family): - self.family = family + _JSON_DESERIALIZERS: Dict[str, Type["Requirement"]] = {} + + @classmethod + def _from_json(self, js): + raise NotImplementedError(self._from_json) + + @classmethod + def from_json(self, js): + try: + family = Requirement._JSON_DESERIALIZERS[js[0]] + except KeyError: + raise UnknownRequirementFamily(js[0]) + return family._from_json(js[1]) def met(self, session): raise NotImplementedError(self) + def _json(self): + raise NotImplementedError(self._json) + + def json(self): + return (type(self).family, self._json()) + + @classmethod + def register_json(cls, subcls): + Requirement._JSON_DESERIALIZERS[subcls.family] = subcls + + +class OneOfRequirement(Requirement): + + elements: List[Requirement] + + family = 'or' + + def __init__(self, elements): + self.elements = elements + + def met(self, session): + for req in self.elements: + if req.met(session): + return True + return False + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.elements) + class UpstreamOutput(object): def __init__(self, family): diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 27ddccd..0826eb4 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -15,11 +15,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from contextlib import ExitStack import logging import os import shlex import sys -from . import UnidentifiedError, DetailedFailure +from urllib.parse import urlparse +from . import UnidentifiedError, DetailedFailure, version_string from .buildlog import ( InstallFixer, ExplainInstallFixer, @@ -29,9 +31,10 @@ from .buildlog import ( from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, - native_resolvers, + select_resolvers, + UnsatisfiedRequirements, ) -from .resolver.apt import AptResolver +from .session import SessionSetupFailure def display_explain_commands(commands): @@ -39,34 +42,33 @@ def display_explain_commands(commands): for command, reqs in commands: if isinstance(command, list): command = shlex.join(command) - logging.info(" %s (to install %s)", command, ", ".join(map(str, reqs))) - - -def get_necessary_declared_requirements(resolver, requirements, stages): - missing = [] - for stage, req in requirements: - if stage in stages: - missing.append(req) - return missing + logging.info( + " %s (to install %s)", command, ", ".join(map(str, reqs))) def install_necessary_declared_requirements( session, resolver, fixers, buildsystems, stages, explain=False ): - relevant = [] - declared_reqs = [] - for buildsystem in buildsystems: - try: - declared_reqs.extend(buildsystem.get_declared_dependencies(session, fixers)) - except NotImplementedError: - logging.warning( - "Unable to determine declared dependencies from %r", buildsystem - ) - relevant.extend( - get_necessary_declared_requirements(resolver, declared_reqs, stages) - ) - install_missing_reqs(session, resolver, relevant, explain=explain) + if explain: + relevant = [] + for buildsystem in buildsystems: + declared_reqs = buildsystem.get_declared_dependencies( + session, fixers) + for stage, req in declared_reqs: + if stage in stages: + relevant.append(req) + install_missing_reqs(session, resolver, relevant, explain=True) + else: + for buildsystem in buildsystems: + try: + buildsystem.install_declared_requirements( + stages, session, resolver, fixers) + except NotImplementedError: + logging.warning( + "Unable to determine declared dependencies from %r", + buildsystem + ) # Types of dependencies: @@ -82,6 +84,7 @@ STAGE_MAP = { "test": ["test", "build", "core"], "build": ["build", "core"], "clean": [], + "verify": ["build", "core", "test"], } @@ -95,9 +98,13 @@ def determine_fixers(session, resolver, explain=False): def main(): # noqa: C901 import argparse - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(prog='ogni') parser.add_argument( - "--directory", "-d", type=str, help="Directory for project.", default="." + "--version", action="version", version="%(prog)s " + version_string + ) + parser.add_argument( + "--directory", "-d", type=str, help="Directory for project.", + default="." ) parser.add_argument("--schroot", type=str, help="schroot to run in.") parser.add_argument( @@ -123,6 +130,15 @@ def main(): # noqa: C901 action="store_true", help="Ignore declared dependencies, follow build errors only", ) + parser.add_argument( + "--user", action="store_true", + help="Install in local-user directories." + ) + parser.add_argument( + "--dep-server-url", type=str, + help="ognibuild dep server to use", + default=os.environ.get('OGNIBUILD_DEPS')) + parser.add_argument("--verbose", action="store_true", help="Be verbose") subparsers = parser.add_subparsers(dest="subcommand") subparsers.add_parser("dist") @@ -130,12 +146,11 @@ def main(): # noqa: C901 subparsers.add_parser("clean") subparsers.add_parser("test") subparsers.add_parser("info") + subparsers.add_parser("verify") exec_parser = subparsers.add_parser("exec") - exec_parser.add_argument('subargv', nargs=argparse.REMAINDER, help='Command to run.') + exec_parser.add_argument( + 'subargv', nargs=argparse.REMAINDER, help='Command to run.') install_parser = subparsers.add_parser("install") - install_parser.add_argument( - "--user", action="store_true", help="Install in local-user directories." - ) install_parser.add_argument( "--prefix", type=str, help='Prefix to install in') @@ -155,38 +170,72 @@ def main(): # noqa: C901 from .session.plain import PlainSession session = PlainSession() - with session: - logging.info("Preparing directory %s", args.directory) - external_dir, internal_dir = session.setup_from_directory(args.directory) + + with ExitStack() as es: + try: + es.enter_context(session) + except SessionSetupFailure as e: + logging.debug('Error lines: %r', e.errlines) + logging.fatal('Failed to set up session: %s', e.reason) + return 1 + + parsed_url = urlparse(args.directory) + # TODO(jelmer): Get a list of supported schemes from breezy? + if parsed_url.scheme in ('git', 'http', 'https', 'ssh'): + import breezy.git # noqa: F401 + import breezy.bzr # noqa: F401 + from breezy.branch import Branch + from silver_platter.utils import TemporarySprout + b = Branch.open(args.directory) + logging.info("Cloning %s", args.directory) + wt = es.enter_context(TemporarySprout(b)) + external_dir, internal_dir = session.setup_from_vcs(wt) + else: + if parsed_url.scheme == 'file': + directory = parsed_url.path + else: + directory = args.directory + logging.info("Preparing directory %s", directory) + external_dir, internal_dir = session.setup_from_directory( + directory) session.chdir(internal_dir) os.chdir(external_dir) if not session.is_temporary and args.subcommand == 'info': args.explain = True - if args.resolve == "apt": - resolver = AptResolver.from_session(session) - elif args.resolve == "native": - resolver = native_resolvers(session, user_local=args.user) - elif args.resolve == "auto": + if args.resolve == "auto": resolver = auto_resolver(session, explain=args.explain) + else: + resolver = select_resolvers( + session, user_local=args.user, + resolvers=args.resolve.split(','), + dep_server_url=args.dep_server_url) logging.info("Using requirement resolver: %s", resolver) fixers = determine_fixers(session, resolver, explain=args.explain) try: if args.subcommand == "exec": from .fix_build import run_with_build_fixers - run_with_build_fixers(session, args.subargv, fixers) + run_with_build_fixers(fixers, session, args.subargv) return 0 - bss = list(detect_buildsystems(args.directory)) + bss = list(detect_buildsystems(external_dir)) logging.info("Detected buildsystems: %s", ", ".join(map(str, bss))) if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: - logging.info("Checking that declared requirements are present") + logging.info( + "Checking that declared requirements are present") try: install_necessary_declared_requirements( - session, resolver, fixers, bss, stages, explain=args.explain + session, resolver, fixers, bss, stages, + explain=args.explain ) + except UnsatisfiedRequirements as e: + logging.info( + 'Unable to install declared dependencies:') + for req in e.requirements: + logging.info(' * %s', req) + return 1 except ExplainInstall as e: display_explain_commands(e.commands) return 1 @@ -207,11 +256,15 @@ def main(): # noqa: C901 if args.subcommand == "build": from .build import run_build - run_build(session, buildsystems=bss, resolver=resolver, fixers=fixers) + run_build( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "clean": from .clean import run_clean - run_clean(session, buildsystems=bss, resolver=resolver, fixers=fixers) + run_clean( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "install": from .install import run_install @@ -226,14 +279,42 @@ def main(): # noqa: C901 if args.subcommand == "test": from .test import run_test - run_test(session, buildsystems=bss, resolver=resolver, fixers=fixers) + run_test( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) if args.subcommand == "info": from .info import run_info run_info(session, buildsystems=bss, fixers=fixers) + + if args.subcommand == "verify": + from .build import run_build + from .test import run_test + + run_build( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) + run_test( + session, buildsystems=bss, resolver=resolver, + fixers=fixers) + except ExplainInstall as e: display_explain_commands(e.commands) - except (UnidentifiedError, DetailedFailure): + except UnidentifiedError: + logging.info( + 'If there is a clear indication of a problem in the build ' + 'log, please consider filing a request to update the patterns ' + 'in buildlog-consultant at ' + 'https://github.com/jelmer/buildlog-consultant/issues/new') + return 1 + except DetailedFailure: + if not args.verbose: + logging.info( + 'Run with --verbose to get more information ' + 'about steps taken to try to resolve error') + logging.info( + 'Please consider filing a bug report at ' + 'https://github.com/jelmer/ognibuild/issues/new') return 1 except NoBuildToolsFound: logging.info("No build tools found.") diff --git a/ognibuild/build.py b/ognibuild/build.py index 1b03bf5..17df903 100644 --- a/ognibuild/build.py +++ b/ognibuild/build.py @@ -15,16 +15,28 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from functools import partial + from .buildsystem import NoBuildToolsFound +from .fix_build import iterate_with_build_fixers +from .logs import NoLogManager -def run_build(session, buildsystems, resolver, fixers): +BUILD_LOG_FILENAME = 'build.log' + + +def run_build(session, buildsystems, resolver, fixers, log_manager=None): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() + if log_manager is None: + log_manager = NoLogManager() + for buildsystem in buildsystems: - buildsystem.build(session, resolver, fixers) + iterate_with_build_fixers( + fixers, log_manager.wrap( + partial(buildsystem.build, session, resolver))) return raise NoBuildToolsFound() diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 037f2c9..29bcddd 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -19,65 +19,32 @@ """ import logging +from typing import Optional, List, Callable, Union, Tuple from buildlog_consultant.common import ( - MissingPythonModule, - MissingPythonDistribution, - MissingCHeader, - MissingPkgConfig, - MissingCommand, - MissingFile, - MissingJavaScriptRuntime, - MissingSprocketsFile, - MissingGoPackage, + Problem, MissingPerlFile, - MissingPerlModule, - MissingXmlEntity, - MissingJDKFile, - MissingJDK, - MissingJRE, - MissingNodeModule, - MissingNodePackage, - MissingPhpClass, - MissingRubyGem, - MissingLibrary, MissingSetupPyCommand, - MissingJavaClass, - MissingCSharpCompiler, - MissingRPackage, - MissingRubyFile, - MissingAutoconfMacro, - MissingValaPackage, - MissingBoostComponents, + MissingCMakeComponents, MissingXfceDependency, MissingHaskellDependencies, - MissingVagueDependency, - DhAddonLoadFailure, MissingMavenArtifacts, - MissingIntrospectionTypelib, - GnomeCommonMissing, MissingGnomeCommonDependency, - UnknownCertificateAuthority, - CMakeFilesMissing, - MissingLibtool, - MissingQt, - MissingX11, MissingPerlPredeclared, MissingLatexFile, MissingCargoCrate, - MissingStaticLibrary, ) -from buildlog_consultant.apt import UnsatisfiedAptDependencies +from . import OneOfRequirement from .fix_build import BuildFixer from .requirements import ( + Requirement, BinaryRequirement, PathRequirement, PkgConfigRequirement, CHeaderRequirement, JavaScriptRuntimeRequirement, ValaPackageRequirement, - RubyGemRequirement, GoPackageRequirement, DhAddonRequirement, PhpClassRequirement, @@ -92,6 +59,7 @@ from .requirements import ( HaskellPackageRequirement, MavenArtifactRequirement, BoostComponentRequirement, + KF5ComponentRequirement, GnomeCommonRequirement, JDKFileRequirement, JDKRequirement, @@ -112,86 +80,124 @@ from .requirements import ( LatexPackageRequirement, CargoCrateRequirement, StaticLibraryRequirement, + GnulibDirectoryRequirement, + LuaModuleRequirement, + PHPExtensionRequirement, + VcsControlDirectoryAccessRequirement, + RubyGemRequirement, + QtModuleRequirement, ) from .resolver import UnsatisfiedRequirements -def problem_to_upstream_requirement(problem): # noqa: C901 - if isinstance(problem, MissingFile): - return PathRequirement(problem.path) - elif isinstance(problem, MissingCommand): - return BinaryRequirement(problem.command) - elif isinstance(problem, MissingPkgConfig): - return PkgConfigRequirement(problem.module, problem.minimum_version) - elif isinstance(problem, MissingCHeader): - return CHeaderRequirement(problem.header) - elif isinstance(problem, MissingIntrospectionTypelib): - return IntrospectionTypelibRequirement(problem.library) - elif isinstance(problem, MissingJavaScriptRuntime): - return JavaScriptRuntimeRequirement() - elif isinstance(problem, MissingRubyGem): - return RubyGemRequirement(problem.gem, problem.version) - elif isinstance(problem, MissingValaPackage): - return ValaPackageRequirement(problem.package) - elif isinstance(problem, MissingGoPackage): - return GoPackageRequirement(problem.package) - elif isinstance(problem, MissingBoostComponents): - return [BoostComponentRequirement(name) for name in problem.components] - elif isinstance(problem, DhAddonLoadFailure): - return DhAddonRequirement(problem.path) - elif isinstance(problem, MissingPhpClass): - return PhpClassRequirement(problem.php_class) - elif isinstance(problem, MissingRPackage): - return RPackageRequirement(problem.package, problem.minimum_version) - elif isinstance(problem, MissingNodeModule): - return NodeModuleRequirement(problem.module) - elif isinstance(problem, MissingStaticLibrary): - return StaticLibraryRequirement(problem.library, problem.filename) - elif isinstance(problem, MissingNodePackage): - return NodePackageRequirement(problem.package) +def map_pytest_arguments_to_plugin(args): + # TODO(jelmer): Map argument to PytestPluginRequirement + return None + + +ProblemToRequirementConverter = Callable[[Problem], Optional[Requirement]] + + +PROBLEM_CONVERTERS: List[Union[ + Tuple[str, ProblemToRequirementConverter], + Tuple[str, ProblemToRequirementConverter, str]]] = [ + ('missing-file', lambda p: PathRequirement(p.path)), + ('command-missing', lambda p: BinaryRequirement(p.command)), + ('valac-cannot-compile', lambda p: VagueDependencyRequirement('valac'), + '0.0.27'), + ('missing-cmake-files', lambda p: OneOfRequirement( + [CMakefileRequirement(filename, p.version) + for filename in p.filenames])), + ('missing-command-or-build-file', lambda p: BinaryRequirement(p.command)), + ('missing-pkg-config-package', + lambda p: PkgConfigRequirement(p.module, p.minimum_version)), + ('missing-c-header', lambda p: CHeaderRequirement(p.header)), + ('missing-introspection-typelib', + lambda p: IntrospectionTypelibRequirement(p.library)), + ('missing-python-module', lambda p: PythonModuleRequirement( + p.module, python_version=p.python_version, + minimum_version=p.minimum_version)), + ('missing-python-distribution', lambda p: PythonPackageRequirement( + p.distribution, python_version=p.python_version, + minimum_version=p.minimum_version)), + ('javascript-runtime-missing', lambda p: JavaScriptRuntimeRequirement()), + ('missing-node-module', lambda p: NodeModuleRequirement(p.module)), + ('missing-node-package', lambda p: NodePackageRequirement(p.package)), + ('missing-ruby-gem', lambda p: RubyGemRequirement(p.gem, p.version)), + ('missing-qt-modules', lambda p: QtModuleRequirement(p.modules[0]), + '0.0.27'), + ('missing-php-class', lambda p: PhpClassRequirement(p.php_class)), + ('missing-r-package', lambda p: RPackageRequirement( + p.package, p.minimum_version)), + ('missing-vague-dependency', + lambda p: VagueDependencyRequirement( + p.name, minimum_version=p.minimum_version)), + ('missing-c#-compiler', lambda p: BinaryRequirement("msc")), + ('missing-gnome-common', lambda p: GnomeCommonRequirement()), + ('missing-jdk', lambda p: JDKRequirement()), + ('missing-jre', lambda p: JRERequirement()), + ('missing-qt', lambda p: QTRequirement()), + ('missing-x11', lambda p: X11Requirement()), + ('missing-libtool', lambda p: LibtoolRequirement()), + ('missing-php-extension', + lambda p: PHPExtensionRequirement(p.extension)), + ('missing-rust-compiler', lambda p: BinaryRequirement("rustc")), + ('missing-java-class', lambda p: JavaClassRequirement(p.classname)), + ('missing-go-package', lambda p: GoPackageRequirement(p.package)), + ('missing-autoconf-macro', lambda p: AutoconfMacroRequirement(p.macro)), + ('missing-vala-package', lambda p: ValaPackageRequirement(p.package)), + ('missing-lua-module', lambda p: LuaModuleRequirement(p.module)), + ('missing-jdk-file', lambda p: JDKFileRequirement(p.jdk_path, p.filename)), + ('missing-ruby-file', lambda p: RubyFileRequirement(p.filename)), + ('missing-library', lambda p: LibraryRequirement(p.library)), + ('missing-sprockets-file', + lambda p: SprocketsFileRequirement(p.content_type, p.name)), + ('dh-addon-load-failure', lambda p: DhAddonRequirement(p.path)), + ('missing-xml-entity', lambda p: XmlEntityRequirement(p.url)), + ('missing-gnulib-directory', + lambda p: GnulibDirectoryRequirement(p.directory)), + ('vcs-control-directory-needed', + lambda p: VcsControlDirectoryAccessRequirement(p.vcs)), + ('missing-static-library', + lambda p: StaticLibraryRequirement(p.library, p.filename)), + ('missing-perl-module', + lambda p: PerlModuleRequirement( + module=p.module, filename=p.filename, inc=p.inc)), + ('unknown-certificate-authority', + lambda p: CertificateAuthorityRequirement(p.url)), + ('unsupported-pytest-arguments', + lambda p: map_pytest_arguments_to_plugin(p.args), '0.0.27'), +] + + +def problem_to_upstream_requirement( + problem: Problem) -> Optional[Requirement]: # noqa: C901 + for entry in PROBLEM_CONVERTERS: + kind, fn = entry[:2] + if kind == problem.kind: + return fn(problem) + if isinstance(problem, MissingCMakeComponents): + if problem.name.lower() == 'boost': + return OneOfRequirement( + [BoostComponentRequirement(name) + for name in problem.components]) + elif problem.name.lower() == 'kf5': + return OneOfRequirement( + [KF5ComponentRequirement(name) for name in problem.components]) + return None elif isinstance(problem, MissingLatexFile): if problem.filename.endswith('.sty'): return LatexPackageRequirement(problem.filename[:-4]) return None - elif isinstance(problem, MissingVagueDependency): - return VagueDependencyRequirement(problem.name, minimum_version=problem.minimum_version) - elif isinstance(problem, MissingLibrary): - return LibraryRequirement(problem.library) - elif isinstance(problem, MissingRubyFile): - return RubyFileRequirement(problem.filename) - elif isinstance(problem, MissingXmlEntity): - return XmlEntityRequirement(problem.url) - elif isinstance(problem, MissingSprocketsFile): - return SprocketsFileRequirement(problem.content_type, problem.name) - elif isinstance(problem, MissingJavaClass): - return JavaClassRequirement(problem.classname) - elif isinstance(problem, CMakeFilesMissing): - return [CMakefileRequirement(filename) for filename in problem.filenames] elif isinstance(problem, MissingHaskellDependencies): - return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps] + return OneOfRequirement( + [HaskellPackageRequirement.from_string(dep) + for dep in problem.deps]) elif isinstance(problem, MissingMavenArtifacts): - return [ + return OneOfRequirement([ MavenArtifactRequirement.from_str(artifact) for artifact in problem.artifacts - ] - elif isinstance(problem, MissingCSharpCompiler): - return BinaryRequirement("msc") - elif isinstance(problem, GnomeCommonMissing): - return GnomeCommonRequirement() - elif isinstance(problem, MissingJDKFile): - return JDKFileRequirement(problem.jdk_path, problem.filename) - elif isinstance(problem, MissingJDK): - return JDKRequirement() - elif isinstance(problem, MissingJRE): - return JRERequirement() - elif isinstance(problem, MissingQt): - return QTRequirement() - elif isinstance(problem, MissingX11): - return X11Requirement() - elif isinstance(problem, MissingLibtool): - return LibtoolRequirement() - elif isinstance(problem, UnknownCertificateAuthority): - return CertificateAuthorityRequirement(problem.url) + ]) elif isinstance(problem, MissingPerlPredeclared): ret = PerlPreDeclaredRequirement(problem.name) try: @@ -210,36 +216,20 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return BinaryRequirement("glib-gettextize") else: logging.warning( - "No known command for gnome-common dependency %s", problem.package + "No known command for gnome-common dependency %s", + problem.package ) return None elif isinstance(problem, MissingXfceDependency): if problem.package == "gtk-doc": return BinaryRequirement("gtkdocize") else: - logging.warning("No known command for xfce dependency %s", problem.package) + logging.warning( + "No known command for xfce dependency %s", problem.package) return None - elif isinstance(problem, MissingPerlModule): - return PerlModuleRequirement( - module=problem.module, filename=problem.filename, inc=problem.inc - ) elif isinstance(problem, MissingPerlFile): return PerlFileRequirement(filename=problem.filename) - elif isinstance(problem, MissingAutoconfMacro): - return AutoconfMacroRequirement(problem.macro) - elif isinstance(problem, MissingPythonModule): - return PythonModuleRequirement( - problem.module, - python_version=problem.python_version, - minimum_version=problem.minimum_version, - ) - elif isinstance(problem, MissingPythonDistribution): - return PythonPackageRequirement( - problem.distribution, - python_version=problem.python_version, - minimum_version=problem.minimum_version, - ) - elif isinstance(problem, UnsatisfiedAptDependencies): + elif problem.kind == 'unsatisfied-apt-dependencies': from .resolver.apt import AptRequirement return AptRequirement(problem.relations) else: diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f3cd7a0..f508219 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,8 +20,7 @@ import logging import os import re -import shlex -from typing import Optional, Tuple +from typing import Optional, Tuple, Type, List, Iterable import warnings from . import shebang_binary, UnidentifiedError @@ -43,8 +42,9 @@ from .requirements import ( MavenArtifactRequirement, GoRequirement, GoPackageRequirement, + VagueDependencyRequirement, ) -from .fix_build import run_with_build_fixers +from .fix_build import run_with_build_fixers, run_detecting_problems from .session import which @@ -69,6 +69,14 @@ class InstallTarget(object): # TODO(jelmer): Add information about target directory, layout, etc. +def get_necessary_declared_requirements(resolver, requirements, stages): + missing = [] + for stage, req in requirements: + if stage in stages: + missing.append(req) + return missing + + class BuildSystem(object): """A particular buildsystem.""" @@ -78,20 +86,27 @@ class BuildSystem(object): return self.name def dist( - self, session, resolver, fixers, target_directory: str, quiet=False + self, session, resolver, target_directory: str, quiet=False ) -> str: raise NotImplementedError(self.dist) - def test(self, session, resolver, fixers): + def install_declared_requirements(self, stages, session, resolver, fixers): + from .buildlog import install_missing_reqs + declared_reqs = self.get_declared_dependencies(session, fixers) + relevant = get_necessary_declared_requirements( + resolver, declared_reqs, stages) + install_missing_reqs(session, resolver, relevant, explain=False) + + def test(self, session, resolver): raise NotImplementedError(self.test) - def build(self, session, resolver, fixers): + def build(self, session, resolver): raise NotImplementedError(self.build) - def clean(self, session, resolver, fixers): + def clean(self, session, resolver): raise NotImplementedError(self.clean) - def install(self, session, resolver, fixers, install_target): + def install(self, session, resolver, install_target): raise NotImplementedError(self.install) def get_declared_dependencies(self, session, fixers=None): @@ -101,7 +116,7 @@ class BuildSystem(object): raise NotImplementedError(self.get_declared_outputs) @classmethod - def probe(cls, path): + def probe(cls, path: str) -> Optional["BuildSystem"]: return None @@ -128,23 +143,33 @@ class Pear(BuildSystem): def __init__(self, path): self.path = path - def dist(self, session, resolver, fixers, target_directory: str, quiet=False): + def dist(self, session, resolver, target_directory: str, + quiet: bool = False): with DistCatcher([session.external_path(".")]) as dc: - run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "package"], fixers) + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "pear"), "package"]) return dc.copy_single(target_directory) - def test(self, session, resolver, fixers): - run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "run-tests"], fixers) + def test(self, session, resolver): + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "pear"), "run-tests"]) - def build(self, session, resolver, fixers): - run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "build", self.path], fixers) + def build(self, session, resolver): + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "pear"), "build", self.path]) - def clean(self, session, resolver, fixers): + def clean(self, session, resolver): self.setup(resolver) # TODO - def install(self, session, resolver, fixers, install_target): - run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "install", self.path], fixers) + def install(self, session, resolver, install_target): + run_detecting_problems( + session, + [guaranteed_which( + session, resolver, "pear"), "install", self.path]) def get_declared_dependencies(self, session, fixers=None): path = os.path.join(self.path, "package.xml") @@ -194,7 +219,9 @@ class Pear(BuildSystem): for ns in cls.PEAR_NAMESPACES: if tree.root.tag == '{%s}package' % ns: - logging.debug("Found package.xml with namespace %s, assuming pear package.") + logging.debug( + "Found package.xml with namespace %s, " + "assuming pear package.") return cls(path) @@ -203,6 +230,11 @@ class Pear(BuildSystem): def run_setup(script_name, script_args=None, stop_after="run"): + # Import setuptools, just in case it decides to replace distutils + try: + import setuptools # noqa: F401 + except ImportError: + pass from distutils import core import sys @@ -235,6 +267,10 @@ def run_setup(script_name, script_args=None, stop_after="run"): _setup_wrapper = """\ +try: + import setuptools +except ImportError: + pass import distutils from distutils import core import sys @@ -295,6 +331,9 @@ class SetupPy(BuildSystem): self.config = self.load_setup_cfg() except FileNotFoundError: self.config = None + except ModuleNotFoundError as e: + logging.warning('Error parsing setup.cfg: %s', e) + self.config = None try: self.pyproject = self.load_toml() @@ -313,7 +352,7 @@ class SetupPy(BuildSystem): return toml.load(pf) def load_setup_cfg(self): - from setuptools.config import read_configuration + from setuptools.config.setupcfg import read_configuration p = os.path.join(self.path, "setup.cfg") if os.path.exists(p): @@ -338,7 +377,8 @@ class SetupPy(BuildSystem): if d is None: logging.warning( "'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p) + "perhaps '%s' is not a Distutils setup script?", + os.path.basename(p) ) return None @@ -357,9 +397,7 @@ class SetupPy(BuildSystem): import tempfile import json - interpreter = shebang_binary(os.path.join(self.path, "setup.py")) - if interpreter is None: - interpreter = self.DEFAULT_PYTHON + interpreter = self._determine_interpreter() output_f = tempfile.NamedTemporaryFile( dir=os.path.join(session.location, "tmp"), mode="w+t" ) @@ -369,14 +407,16 @@ class SetupPy(BuildSystem): argv = [ interpreter, "-c", - _setup_wrapper.replace("%(script_name)s", '"setup.py"').replace( - "%(output_path)s", - '"/' + os.path.relpath(output_f.name, session.location) + '"', + _setup_wrapper.replace( + "%(script_name)s", '"setup.py"').replace( + "%(output_path)s", + '"/' + os.path.relpath(output_f.name, session.location) + + '"', ), ] try: if fixers is not None: - run_with_build_fixers(session, argv, fixers) + run_with_build_fixers(fixers, session, argv, quiet=True) else: session.check_call(argv, close_fds=False) except RuntimeError as e: @@ -388,31 +428,43 @@ class SetupPy(BuildSystem): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def test(self, session, resolver, fixers): + def test(self, session, resolver): if os.path.exists(os.path.join(self.path, "tox.ini")): - run_with_build_fixers(session, ["tox"], fixers) - elif self.pyproject: - run_with_build_fixers( - session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers - ) - elif self.has_setup_py: + run_detecting_problems( + session, ["tox", "--skip-missing-interpreters"]) + return + if self.pyproject: + run_detecting_problems( + session, ["python3", "-m", "pep517.check", "."]) + return + if 'tool:pytest' in self.config or 'pytest' in self.config: + run_detecting_problems(session, ['pytest']) + return + if self.has_setup_py: # Pre-emptively insall setuptools, since distutils doesn't provide # a 'test' subcommand and some packages fall back to distutils # if setuptools is not available. setuptools_req = PythonPackageRequirement("setuptools") if not setuptools_req.met(session): resolver.install([setuptools_req]) - self._run_setup(session, resolver, ["test"], fixers) - else: - raise NotImplementedError + try: + self._run_setup(session, resolver, ["test"]) + except UnidentifiedError as e: + if "error: invalid command 'test'" in e.lines: + pass + else: + raise + else: + return + raise NotImplementedError - def build(self, session, resolver, fixers): + def build(self, session, resolver): if self.has_setup_py: - self._run_setup(session, resolver, ["build"], fixers) + self._run_setup(session, resolver, ["build"]) else: raise NotImplementedError - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): # TODO(jelmer): Look at self.build_backend if self.has_setup_py: preargs = [] @@ -424,59 +476,68 @@ class SetupPy(BuildSystem): if not setuptools_req.met(session): resolver.install([setuptools_req]) with DistCatcher([session.external_path("dist")]) as dc: - self._run_setup(session, resolver, preargs + ["sdist"], fixers) + self._run_setup(session, resolver, preargs + ["sdist"]) return dc.copy_single(target_directory) elif self.pyproject: with DistCatcher([session.external_path("dist")]) as dc: - run_with_build_fixers( + run_detecting_problems( session, - [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], - fixers, + ["python3", "-m", "pep517.build", "--source", "."], ) return dc.copy_single(target_directory) raise AssertionError("no setup.py or pyproject.toml") - def clean(self, session, resolver, fixers): + def clean(self, session, resolver): if self.has_setup_py: - self._run_setup(session, resolver, ["clean"], fixers) + self._run_setup(session, resolver, ["clean"]) else: raise NotImplementedError - def install(self, session, resolver, fixers, install_target): + def install(self, session, resolver, install_target): if self.has_setup_py: extra_args = [] if install_target.user: extra_args.append("--user") if install_target.prefix: extra_args.append("--prefix=%s" % install_target.prefix) - self._run_setup(session, resolver, ["install"] + extra_args, fixers) + self._run_setup( + session, resolver, ["install"] + extra_args) else: raise NotImplementedError - def _run_setup(self, session, resolver, args, fixers): + def _determine_interpreter(self): + interpreter = None + if self.config: + python_requires = self.config.get( + 'options', {}).get('python_requires') + if python_requires: + if not python_requires.contains('2.7'): + interpreter = 'python3' + if interpreter is None: + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) + if interpreter is None: + interpreter = self.DEFAULT_PYTHON + return interpreter + + def _run_setup(self, session, resolver, args): from .buildlog import install_missing_reqs # Install the setup_requires beforehand, since otherwise # setuptools might fetch eggs instead of our preferred resolver. install_missing_reqs(session, resolver, list(self._setup_requires())) - interpreter = shebang_binary(os.path.join(self.path, "setup.py")) - if interpreter is None: - interpreter = self.DEFAULT_PYTHON + interpreter = self._determine_interpreter() argv = [interpreter, "./setup.py"] + args # TODO(jelmer): Perhaps this should be additive? env = dict(os.environ) - # Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment - if "SETUPTOOLS_SCM_PRETEND_VERSION" in os.environ: - env["SETUPTOOLS_SCM_PRETEND_VERSION"] = os.environ[ - "SETUPTOOLS_SCM_PRETEND_VERSION" - ] - run_with_build_fixers(session, argv, fixers, env=env) + run_detecting_problems(session, argv, env=env) def _setup_requires(self): if self.pyproject: if "build-system" in self.pyproject: - for require in self.pyproject["build-system"].get("requires", []): - yield PythonPackageRequirement.from_requirement_str(require) + requires = self.pyproject["build-system"].get("requires", []) + for require in requires: + yield PythonPackageRequirement.from_requirement_str( + require) if self.config: options = self.config.get("options", {}) for require in options.get("setup_requires", []): @@ -486,28 +547,35 @@ class SetupPy(BuildSystem): distribution = self._extract_setup(session, fixers) if distribution is not None: for require in distribution["requires"]: - yield "core", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str( + require) # Not present for distutils-only packages for require in distribution["setup_requires"]: - yield "build", PythonPackageRequirement.from_requirement_str(require) + yield "build", PythonPackageRequirement.from_requirement_str( + require) # Not present for distutils-only packages for require in distribution["install_requires"]: - yield "core", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str( + require) # Not present for distutils-only packages for require in distribution["tests_require"]: - yield "test", PythonPackageRequirement.from_requirement_str(require) + yield "test", PythonPackageRequirement.from_requirement_str( + require) if self.pyproject: if "build-system" in self.pyproject: - for require in self.pyproject["build-system"].get("requires", []): - yield "build", PythonPackageRequirement.from_requirement_str( - require - ) + requires = self.pyproject["build-system"].get("requires", []) + for require in requires: + yield ( + "build", + PythonPackageRequirement.from_requirement_str(require)) if self.config: options = self.config.get("options", {}) for require in options.get("setup_requires", []): - yield "build", PythonPackageRequirement.from_requirement_str(require) + yield "build", PythonPackageRequirement.from_requirement_str( + require) for require in options.get("install_requires", []): - yield "core", PythonPackageRequirement.from_requirement_str(require) + yield "core", PythonPackageRequirement.from_requirement_str( + require) def get_declared_outputs(self, session, fixers=None): distribution = self._extract_setup(session, fixers) @@ -515,7 +583,8 @@ class SetupPy(BuildSystem): if distribution is not None: for script in distribution["scripts"]: yield BinaryOutput(os.path.basename(script)) - for script in distribution["entry_points"].get("console_scripts", []): + for script in distribution["entry_points"].get( + "console_scripts", []): yield BinaryOutput(script.split("=")[0]) all_packages.update(distribution["packages"]) if self.config: @@ -523,7 +592,8 @@ class SetupPy(BuildSystem): all_packages.update(options.get("packages", [])) for script in options.get("scripts", []): yield BinaryOutput(os.path.basename(script)) - for script in options.get("entry_points", {}).get("console_scripts", []): + for script in options.get("entry_points", {}).get( + "console_scripts", []): yield BinaryOutput(script.split("=")[0]) packages = set() @@ -571,11 +641,49 @@ class Bazel(BuildSystem): logging.debug("Found BUILD, assuming bazel package.") return cls(path) - def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["bazel", "build", "//..."], fixers) + def build(self, session, resolver): + run_detecting_problems(session, ["bazel", "build", "//..."]) - def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["bazel", "test", "//..."], fixers) + def test(self, session, resolver): + run_detecting_problems(session, ["bazel", "test", "//..."]) + + +class GnomeShellExtension(BuildSystem): + + name = "gnome-shell-extension" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def exists(cls, path): + if not os.path.exists(os.path.join(path, "metadata.json")): + return False + return True + + @classmethod + def probe(cls, path): + if cls.exists(path): + logging.debug( + "Found metadata.json , assuming gnome-shell extension.") + return cls(path) + + def build(self, session, resolver): + pass + + def test(self, session, resolver): + pass + + def get_declared_dependencies(self, session, fixers=None): + import json + with open(os.path.join(self.path, 'metadata.json'), 'r') as f: + metadata = json.load(f) + if 'shell-version' in metadata: + # TODO(jelmer): Somehow represent supported versions + yield "core", VagueDependencyRequirement("gnome-shell") class Octave(BuildSystem): @@ -639,9 +747,8 @@ class Gradle(BuildSystem): @classmethod def exists(cls, path): - return os.path.exists(os.path.join(path, "build.gradle")) or os.path.exists( - os.path.join(path, "build.gradle.kts") - ) + return (os.path.exists(os.path.join(path, "build.gradle")) + or os.path.exists(os.path.join(path, "build.gradle.kts"))) @classmethod def from_path(cls, path): @@ -661,7 +768,7 @@ class Gradle(BuildSystem): if not binary_req.met(session): resolver.install([binary_req]) - def _run(self, session, resolver, task, args, fixers): + def _run(self, session, resolver, task, args): self.setup(session, resolver) argv = [] if self.executable.startswith("./") and ( @@ -671,12 +778,13 @@ class Gradle(BuildSystem): argv.extend([self.executable, task]) argv.extend(args) try: - run_with_build_fixers(session, argv, fixers) + run_detecting_problems(session, argv) except UnidentifiedError as e: if any( [ re.match( - r"Task '" + task + r"' not found in root project '.*'\.", line + r"Task '" + task + + r"' not found in root project '.*'\.", line ) for line in e.lines ] @@ -684,24 +792,24 @@ class Gradle(BuildSystem): raise NotImplementedError raise - def clean(self, session, resolver, fixers): - self._run(session, resolver, "clean", [], fixers) + def clean(self, session, resolver): + self._run(session, resolver, "clean", []) - def build(self, session, resolver, fixers): - self._run(session, resolver, "build", [], fixers) + def build(self, session, resolver): + self._run(session, resolver, "build", []) - def test(self, session, resolver, fixers): - self._run(session, resolver, "test", [], fixers) + def test(self, session, resolver): + self._run(session, resolver, "test", []) - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): with DistCatcher([session.external_path(".")]) as dc: - self._run(session, resolver, "distTar", [], fixers) + self._run(session, resolver, "distTar", []) return dc.copy_single(target_directory) - def install(self, session, resolver, fixers, install_target): + def install(self, session, resolver, install_target): raise NotImplementedError # TODO(jelmer): installDist just creates files under build/install/... - self._run(session, resolver, "installDist", [], fixers) + self._run(session, resolver, "installDist", []) class R(BuildSystem): @@ -716,31 +824,39 @@ class R(BuildSystem): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def build(self, session, resolver, fixers): + def build(self, session, resolver): pass - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): r_path = guaranteed_which(session, resolver, "R") with DistCatcher([session.external_path(".")]) as dc: - run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) + run_detecting_problems(session, [r_path, "CMD", "build", "."]) return dc.copy_single(target_directory) - def install(self, session, resolver, fixers, install_target): + def install(self, session, resolver, install_target): extra_args = [] if install_target.prefix: extra_args.append("--prefix=%s" % install_target.prefix) r_path = guaranteed_which(session, resolver, "R") - run_with_build_fixers(session, [r_path, "CMD", "INSTALL", "."] + extra_args, fixers) + run_detecting_problems( + session, [r_path, "CMD", "INSTALL", "."] + extra_args) - def test(self, session, resolver, fixers): + def test(self, session, resolver): r_path = guaranteed_which(session, resolver, "R") - run_with_build_fixers(session, [r_path, "CMD", "check", "."], fixers) + if session.exists("run_tests.sh"): + run_detecting_problems(session, ["./run_tests.sh"]) + elif session.exists("tests/testthat"): + run_detecting_problems( + session, [r_path, "-e", "testthat::test_dir('tests')"]) + + def lint(self, session, resolver): + r_path = guaranteed_which(session, resolver, "R") + run_detecting_problems(session, [r_path, "CMD", "check"]) @classmethod def probe(cls, path): - if os.path.exists(os.path.join(path, "DESCRIPTION")) and os.path.exists( - os.path.join(path, "NAMESPACE") - ): + if (os.path.exists(os.path.join(path, "DESCRIPTION")) + and os.path.exists(os.path.join(path, "NAMESPACE"))): return cls(path) def _read_description(self): @@ -785,37 +901,39 @@ class Meson(BuildSystem): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def _setup(self, session, fixers): + def _setup(self, session): if not session.exists("build"): session.mkdir("build") - run_with_build_fixers(session, ["meson", "setup", "build"], fixers) + run_detecting_problems(session, ["meson", "setup", "build"]) - def clean(self, session, resolver, fixers): - self._setup(session, fixers) - run_with_build_fixers(session, ["ninja", "-C", "build", "clean"], fixers) + def clean(self, session, resolver): + self._setup(session) + run_detecting_problems(session, ["ninja", "-C", "build", "clean"]) - def build(self, session, resolver, fixers): - self._setup(session, fixers) - run_with_build_fixers(session, ["ninja", "-C", "build"], fixers) + def build(self, session, resolver): + self._setup(session) + run_detecting_problems(session, ["ninja", "-C", "build"]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): - self._setup(session, fixers) + def dist(self, session, resolver, target_directory, quiet=False): + self._setup(session) with DistCatcher([session.external_path("build/meson-dist")]) as dc: try: - run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) + run_detecting_problems( + session, ["ninja", "-C", "build", "dist"]) except UnidentifiedError as e: - if "ninja: error: unknown target 'dist', did you mean 'dino'?" in e.lines: + if ("ninja: error: unknown target 'dist', did you mean 'dino'?" + in e.lines): raise NotImplementedError raise return dc.copy_single(target_directory) - def test(self, session, resolver, fixers): - self._setup(session, fixers) - run_with_build_fixers(session, ["ninja", "-C", "build", "test"], fixers) + def test(self, session, resolver): + self._setup(session) + run_detecting_problems(session, ["ninja", "-C", "build", "test"]) - def install(self, session, resolver, fixers, install_target): - self._setup(session, fixers) - run_with_build_fixers(session, ["ninja", "-C", "build", "install"], fixers) + def install(self, session, resolver, install_target): + self._setup(session) + run_detecting_problems(session, ["ninja", "-C", "build", "install"]) @classmethod def probe(cls, path): @@ -823,6 +941,40 @@ class Meson(BuildSystem): logging.debug("Found meson.build, assuming meson package.") return Meson(os.path.join(path, "meson.build")) + def _introspect(self, session, fixers, args): + ret = run_with_build_fixers( + fixers, + session, + ["meson", "introspect"] + args + ['./meson.build']) + import json + return json.loads(''.join(ret)) + + def get_declared_dependencies(self, session, fixers=None): + resp = self._introspect(session, fixers, ["--scan-dependencies"]) + for entry in resp: + version = entry.get('version', []) + minimum_version = None + if len(version) == 1 and version[0].startswith('>='): + minimum_version = version[0][2:] + elif len(version) > 1: + logging.warning( + 'Unable to parse version constraints: %r', version) + # TODO(jelmer): Include entry['required'] + yield ( + "core", + VagueDependencyRequirement( + entry['name'], minimum_version=minimum_version)) + + def get_declared_outputs(self, session, fixers=None): + resp = self._introspect(session, fixers, ["--targets"]) + for entry in resp: + if not entry['installed']: + continue + if entry['type'] == 'executable': + for name in entry['filename']: + yield BinaryOutput(os.path.basename(name)) + # TODO(jelmer): Handle other types + class Npm(BuildSystem): @@ -840,49 +992,49 @@ class Npm(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def get_declared_dependencies(self, session, fixers=None): - if "dependencies" in self.package: - for name, unused_version in self.package["dependencies"].items(): - # TODO(jelmer): Look at version - yield "core", NodePackageRequirement(name) - if "devDependencies" in self.package: - for name, unused_version in self.package["devDependencies"].items(): - # TODO(jelmer): Look at version - yield "build", NodePackageRequirement(name) + for name, unused_version in self.package.get( + "dependencies", {}).items(): + # TODO(jelmer): Look at version + yield "core", NodePackageRequirement(name) + for name, unused_version in self.package.get( + "devDependencies", {}).items(): + # TODO(jelmer): Look at version + yield "build", NodePackageRequirement(name) def setup(self, session, resolver): binary_req = BinaryRequirement("npm") if not binary_req.met(session): resolver.install([binary_req]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): self.setup(session, resolver) with DistCatcher([session.external_path(".")]) as dc: - run_with_build_fixers(session, ["npm", "pack"], fixers) + run_detecting_problems(session, ["npm", "pack"]) return dc.copy_single(target_directory) - def test(self, session, resolver, fixers): + def test(self, session, resolver): self.setup(session, resolver) - test_script = self.package["scripts"].get("test") + test_script = self.package.get("scripts", {}).get("test") if test_script: - run_with_build_fixers(session, shlex.split(test_script), fixers) + run_detecting_problems(session, ['bash', '-c', test_script]) else: - raise NotImplementedError + logging.info('No test command defined in package.json') - def build(self, session, resolver, fixers): + def build(self, session, resolver): self.setup(session, resolver) - build_script = self.package["scripts"].get("build") + build_script = self.package.get("scripts", {}).get("build") if build_script: - run_with_build_fixers(session, shlex.split(build_script), fixers) + run_detecting_problems(session, ['bash', '-c', build_script]) else: - raise NotImplementedError + logging.info('No build command defined in package.json') - def clean(self, session, resolver, fixers): + def clean(self, session, resolver): self.setup(session, resolver) - clean_script = self.package["scripts"].get("clean") + clean_script = self.package.get("scripts", {}).get("clean") if clean_script: - run_with_build_fixers(session, shlex.split(clean_script), fixers) + run_detecting_problems(session, ['bash', '-c', clean_script]) else: - raise NotImplementedError + logging.info('No clean command defined in package.json') @classmethod def probe(cls, path): @@ -898,20 +1050,31 @@ class Waf(BuildSystem): def __init__(self, path): self.path = path - def setup(self, session, resolver, fixers): + def setup(self, session, resolver): binary_req = BinaryRequirement("python3") if not binary_req.met(session): resolver.install([binary_req]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): - self.setup(session, resolver, fixers) + def build(self, session, resolver): + try: + run_detecting_problems(session, [self.path, 'build']) + except UnidentifiedError as e: + if ("The project was not configured: run \"waf configure\" first!" + in e.lines): + run_detecting_problems(session, [self.path, 'configure']) + run_detecting_problems(session, [self.path, 'build']) + else: + raise + + def dist(self, session, resolver, target_directory, quiet=False): + self.setup(session, resolver) with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers(session, ["./waf", "dist"], fixers) + run_detecting_problems(session, ["./waf", "dist"]) return dc.copy_single(target_directory) - def test(self, session, resolver, fixers): - self.setup(session, resolver, fixers) - run_with_build_fixers(session, ["./waf", "test"], fixers) + def test(self, session, resolver): + self.setup(session, resolver) + run_detecting_problems(session, ["./waf", "test"]) @classmethod def probe(cls, path): @@ -927,22 +1090,25 @@ class Gem(BuildSystem): def __init__(self, path): self.path = path - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): gemfiles = [ - entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") + entry.name for entry in session.scandir(".") + if entry.name.endswith(".gem") ] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers( + run_detecting_problems( session, - [guaranteed_which(session, resolver, "gem2tgz"), gemfiles[0]], fixers) + [guaranteed_which(session, resolver, "gem2tgz"), + gemfiles[0]]) return dc.copy_single(target_directory) @classmethod def probe(cls, path): gemfiles = [ - entry.path for entry in os.scandir(path) if entry.name.endswith(".gem") + entry.path for entry in os.scandir(path) + if entry.name.endswith(".gem") ] if gemfiles: return cls(gemfiles[0]) @@ -963,9 +1129,11 @@ class DistZilla(BuildSystem): (key, value) = line[2:].strip().split(b"=", 1) except ValueError: continue - if key.strip() == b"class" and value.strip().startswith(b"'Dist::Inkt"): + if (key.strip() == b"class" + and value.strip().startswith(b"'Dist::Inkt")): logging.debug( - "Found Dist::Inkt section in dist.ini, " "assuming distinkt." + "Found Dist::Inkt section in dist.ini, " + "assuming distinkt." ) self.name = "dist-inkt" self.dist_inkt_class = value.decode().strip("'") @@ -979,39 +1147,51 @@ class DistZilla(BuildSystem): ] ) - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): self.setup(resolver) if self.name == "dist-inkt": with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers(session, [guaranteed_which(session, resolver, "distinkt-dist")], fixers) + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "distinkt-dist")]) return dc.copy_single(target_directory) else: # Default to invoking Dist::Zilla with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "build", "--tgz"], fixers) + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "dzil"), + "build", "--tgz"]) return dc.copy_single(target_directory) - def test(self, session, resolver, fixers): + def test(self, session, resolver): + # see also + # https://perlmaven.com/how-to-run-the-tests-of-a-typical-perl-module self.setup(resolver) - run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "test"], fixers) + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "dzil"), "test"]) - def build(self, session, resolver, fixers): + def build(self, session, resolver): self.setup(resolver) - run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "build"], fixers) + run_detecting_problems( + session, + [guaranteed_which(session, resolver, "dzil"), "build"]) @classmethod def probe(cls, path): - if os.path.exists(os.path.join(path, "dist.ini")) and not os.path.exists( - os.path.join(path, "Makefile.PL") - ): + if (os.path.exists(os.path.join(path, "dist.ini")) + and not os.path.exists(os.path.join(path, "Makefile.PL"))): return cls(os.path.join(path, "dist.ini")) def get_declared_dependencies(self, session, fixers=None): if os.path.exists(os.path.join(self.path, "dist.ini")): - lines = run_with_build_fixers(session, ["dzil", "authordeps"], fixers) + lines = run_with_build_fixers( + fixers, session, ["dzil", "authordeps"]) for entry in lines: yield "build", PerlModuleRequirement(entry.strip()) - if os.path.exists(os.path.join(os.path.dirname(self.path), "cpanfile")): + if os.path.exists( + os.path.join(os.path.dirname(self.path), "cpanfile")): yield from _declared_deps_from_cpanfile(session, fixers) @@ -1030,22 +1210,24 @@ class RunTests(BuildSystem): if os.path.exists(os.path.join(path, "runtests.sh")): return cls(path) - def test(self, session, resolver, fixers): + def test(self, session, resolver): if shebang_binary(os.path.join(self.path, "runtests.sh")) is not None: - run_with_build_fixers(session, ["./runtests.sh"], fixers) + run_detecting_problems(session, ["./runtests.sh"]) else: - run_with_build_fixers(session, ["/bin/bash", "./runtests.sh"], fixers) + run_detecting_problems(session, ["/bin/bash", "./runtests.sh"]) def _read_cpanfile(session, args, kind, fixers): - for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers): + for line in run_with_build_fixers( + fixers, session, ["cpanfile-dump"] + args): line = line.strip() if line: yield kind, PerlModuleRequirement(line) def _declared_deps_from_cpanfile(session, fixers): - yield from _read_cpanfile(session, ["--configure", "--build"], "build", fixers) + yield from _read_cpanfile( + session, ["--configure", "--build"], "build", fixers) yield from _read_cpanfile(session, ["--test"], "test", fixers) @@ -1069,6 +1251,53 @@ def _declared_deps_from_meta_yml(f): # TODO(jelmer): recommends +class CMake(BuildSystem): + + name = "cmake" + + def __init__(self, path): + self.path = path + self.builddir = 'build' + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def setup(self, session, resolver): + if not session.exists(self.builddir): + session.mkdir(self.builddir) + try: + run_detecting_problems( + session, ["cmake", '.', '-B%s' % self.builddir]) + except Exception: + session.rmtree(self.builddir) + raise + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, 'CMakeLists.txt')): + return cls(path) + return None + + def build(self, session, resolver): + self.setup(session, resolver) + run_detecting_problems( + session, ["cmake", "--build", self.builddir]) + + def install(self, session, resolver, install_target): + self.setup(session, resolver) + run_detecting_problems( + session, ["cmake", "--install", self.builddir]) + + def clean(self, session, resolver): + self.setup(session, resolver) + run_detecting_problems( + session, + ["cmake", "--build %s" % self.builddir, ".", "--target", "clean"]) + + def test(self, session, resolver): + raise NotImplementedError(self.test) + + class Make(BuildSystem): def __init__(self, path): @@ -1080,67 +1309,72 @@ class Make(BuildSystem): elif any([os.path.exists(os.path.join(path, n)) for n in ['configure.ac', 'configure.in', 'autogen.sh']]): self.name = 'autoconf' - elif os.path.exists(os.path.join(path, "CMakeLists.txt")): - self.name = 'cmake' + elif any([n.name.endswith(".pro") for n in os.scandir(path)]): + self.name = 'qmake' else: self.name = "make" def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def setup(self, session, resolver, fixers, prefix=None): + def setup(self, session, resolver, prefix=None): def makefile_exists(): return any( - [session.exists(p) for p in ["Makefile", "GNUmakefile", "makefile"]] + [session.exists(p) + for p in ["Makefile", "GNUmakefile", "makefile"]] ) if session.exists("Makefile.PL") and not makefile_exists(): - run_with_build_fixers(session, ["perl", "Makefile.PL"], fixers) + run_detecting_problems(session, ["perl", "Makefile.PL"]) if not makefile_exists() and not session.exists("configure"): if session.exists("autogen.sh"): - if shebang_binary(os.path.join(self.path, "autogen.sh")) is None: - run_with_build_fixers(session, ["/bin/sh", "./autogen.sh"], fixers) + if shebang_binary( + os.path.join(self.path, "autogen.sh")) is None: + run_detecting_problems( + session, ["/bin/sh", "./autogen.sh"]) try: - run_with_build_fixers(session, ["./autogen.sh"], fixers) + run_detecting_problems(session, ["./autogen.sh"]) except UnidentifiedError as e: if ( "Gnulib not yet bootstrapped; " "run ./bootstrap instead." in e.lines ): - run_with_build_fixers(session, ["./bootstrap"], fixers) - run_with_build_fixers(session, ["./autogen.sh"], fixers) + run_detecting_problems(session, ["./bootstrap"]) + run_detecting_problems(session, ["./autogen.sh"]) else: raise - elif session.exists("configure.ac") or session.exists("configure.in"): - run_with_build_fixers(session, ["autoreconf", "-i"], fixers) + elif (session.exists("configure.ac") + or session.exists("configure.in")): + run_detecting_problems(session, ["autoreconf", "-i"]) if not makefile_exists() and session.exists("configure"): extra_args = [] if prefix is not None: extra_args.append('--prefix=%s' % prefix) - run_with_build_fixers(session, ["./configure"] + extra_args, fixers) + run_detecting_problems(session, ["./configure"] + extra_args) if not makefile_exists() and any( [n.name.endswith(".pro") for n in session.scandir(".")] ): - run_with_build_fixers(session, ["qmake"], fixers) + run_detecting_problems(session, ["qmake"]) - if not makefile_exists() and session.exists('CMakeLists.txt'): - if not session.exists("build"): - session.mkdir('build') - run_with_build_fixers(session, ["cmake", '..'], fixers, cwd='build') + def build(self, session, resolver): + self.setup(session, resolver) + if self.name == 'qmake': + default_target = None + else: + default_target = 'all' + self._run_make( + session, + [default_target] if default_target else []) - def build(self, session, resolver, fixers): - self.setup(session, resolver, fixers) - self._run_make(session, ["all"], fixers) + def clean(self, session, resolver): + self.setup(session, resolver) + self._run_make(session, ["clean"]) - def clean(self, session, resolver, fixers): - self.setup(session, resolver, fixers) - self._run_make(session, ["clean"], fixers) - - def _run_make(self, session, args, fixers, prefix=None): + def _run_make(self, session, args, prefix=None): def _wants_configure(line): if line.startswith("Run ./configure"): return True @@ -1150,96 +1384,116 @@ class Make(BuildSystem): return True if line.startswith("The project was not configured"): return True + if re.match( + r'Makefile:[0-9]+: \*\*\* ' + r'You need to run \.\/configure .*', line): + return True return False - if session.exists('build'): + if session.exists('build/Makefile'): cwd = 'build' else: cwd = None try: - run_with_build_fixers(session, ["make"] + args, fixers, cwd=cwd) + run_detecting_problems(session, ["make"] + args, cwd=cwd) except UnidentifiedError as e: - if len(e.lines) < 5 and any([_wants_configure(line) for line in e.lines]): + if len(e.lines) < 5 and any( + [_wants_configure(line) for line in e.lines]): extra_args = [] if prefix is not None: extra_args.append("--prefix=%s" % prefix) - run_with_build_fixers(session, ["./configure"] + extra_args, fixers) - run_with_build_fixers(session, ["make"] + args, fixers) + run_detecting_problems(session, ["./configure"] + extra_args) + run_detecting_problems(session, ["make"] + args) elif ( "Reconfigure the source tree " "(via './config' or 'perl Configure'), please." ) in e.lines: - run_with_build_fixers(session, ["./config"], fixers) - run_with_build_fixers(session, ["make"] + args, fixers) + run_detecting_problems(session, ["./config"]) + run_detecting_problems(session, ["make"] + args) else: raise - def test(self, session, resolver, fixers): - self.setup(session, resolver, fixers) - self._run_make(session, ["check"], fixers) + def test(self, session, resolver): + self.setup(session, resolver) + for target in ["check", "test"]: + try: + self._run_make(session, [target]) + except UnidentifiedError as e: + if (("make: *** No rule to make target '%s'. Stop." % target) + in e.lines): + pass + else: + raise + else: + break + else: + if os.path.isdir('t'): + # See + # https://perlmaven.com/how-to-run-the-tests-of-a-typical-perl-module + run_detecting_problems(session, ["prove", "-b", "t/"]) + else: + logging.warning('No test target found') - def install(self, session, resolver, fixers, install_target): - self.setup(session, resolver, fixers, prefix=install_target.prefix) - self._run_make(session, ["install"], fixers, prefix=install_target.prefix) + def install(self, session, resolver, install_target): + self.setup(session, resolver, prefix=install_target.prefix) + self._run_make(session, ["install"], prefix=install_target.prefix) - def dist(self, session, resolver, fixers, target_directory, quiet=False): - self.setup(session, resolver, fixers) + def dist(self, session, resolver, target_directory, quiet=False): + self.setup(session, resolver) with DistCatcher.default(session.external_path(".")) as dc: try: - self._run_make(session, ["dist"], fixers) + self._run_make(session, ["dist"]) except UnidentifiedError as e: - if "make: *** No rule to make target 'dist'. Stop." in e.lines: + if ("make: *** No rule to make target 'dist'. Stop." + in e.lines): raise NotImplementedError - elif "make[1]: *** No rule to make target 'dist'. Stop." in e.lines: + elif ("make[1]: *** No rule to make target 'dist'. Stop." + in e.lines): raise NotImplementedError - elif "ninja: error: unknown target 'dist', did you mean 'dino'?" in e.lines: + elif ("ninja: error: unknown target 'dist', " + "did you mean 'dino'?" in e.lines): raise NotImplementedError elif ( "Please try running 'make manifest' and then run " "'make dist' again." in e.lines ): - run_with_build_fixers(session, ["make", "manifest"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) + run_detecting_problems(session, ["make", "manifest"]) + run_detecting_problems(session, ["make", "dist"]) elif any( [ re.match( r"(Makefile|GNUmakefile|makefile):[0-9]+: " r"\*\*\* Missing \'Make.inc\' " - r"Run \'./configure \[options\]\' and retry. Stop.", - line, + r"Run \'./configure \[options\]\' and retry. " + r"Stop.", line, ) for line in e.lines ] ): - run_with_build_fixers(session, ["./configure"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) + run_detecting_problems(session, ["./configure"]) + run_detecting_problems(session, ["make", "dist"]) elif any( [ re.match( - r"Problem opening MANIFEST: No such file or directory " - r"at .* line [0-9]+\.", - line, + r"Problem opening MANIFEST: " + r"No such file or directory " + r"at .* line [0-9]+\.", line, ) for line in e.lines ] ): - run_with_build_fixers(session, ["make", "manifest"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) + run_detecting_problems(session, ["make", "manifest"]) + run_detecting_problems(session, ["make", "dist"]) else: raise return dc.copy_single(target_directory) def get_declared_dependencies(self, session, fixers=None): - something = False # TODO(jelmer): Split out the perl-specific stuff? if os.path.exists(os.path.join(self.path, "META.yml")): with open(os.path.join(self.path, "META.yml"), "rb") as f: yield from _declared_deps_from_meta_yml(f) - something = True if os.path.exists(os.path.join(self.path, "cpanfile")): yield from _declared_deps_from_cpanfile(session, fixers) - something = True - if not something: - raise NotImplementedError @classmethod def probe(cls, path): @@ -1251,7 +1505,6 @@ class Make(BuildSystem): "GNUmakefile", "makefile", "Makefile.PL", - "CMakeLists.txt", "autogen.sh", "configure.ac", "configure.in", @@ -1280,6 +1533,9 @@ class Cargo(BuildSystem): with open(path, "r") as f: self.cargo = load(f) + def install_declared_requirements(self, stages, session, resolver, fixers): + run_with_build_fixers(fixers, session, ["cargo", "fetch"]) + def get_declared_dependencies(self, session, fixers=None): if "dependencies" in self.cargo: for name, details in self.cargo["dependencies"].items(): @@ -1292,14 +1548,26 @@ class Cargo(BuildSystem): version=details.get("version"), ) - def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["cargo", "test"], fixers) + def test(self, session, resolver): + run_detecting_problems(session, ["cargo", "test"]) - def clean(self, session, resolver, fixers): - run_with_build_fixers(session, ["cargo", "clean"], fixers) + def clean(self, session, resolver): + run_detecting_problems(session, ["cargo", "clean"]) - def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["cargo", "build"], fixers) + def build(self, session, resolver): + try: + run_detecting_problems(session, ["cargo", "generate"]) + except UnidentifiedError as e: + if e.lines != ['error: no such subcommand: `generate`']: + raise + run_detecting_problems(session, ["cargo", "build"]) + + def install(self, session, resolver, install_target): + args = [] + if install_target.prefix: + args.append('-root=%s' % install_target.prefix) + run_detecting_problems( + session, ["cargo", "install", "--path=."] + args) @classmethod def probe(cls, path): @@ -1344,16 +1612,16 @@ class Golang(BuildSystem): def __repr__(self): return "%s()" % (type(self).__name__) - def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["go", "test", "./..."], fixers) + def test(self, session, resolver): + run_detecting_problems(session, ["go", "test", "./..."]) - def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["go", "build"], fixers) + def build(self, session, resolver): + run_detecting_problems(session, ["go", "build"]) - def install(self, session, resolver, fixers): - run_with_build_fixers(session, ["go", "install"], fixers) + def install(self, session, resolver, install_target): + run_detecting_problems(session, ["go", "install"]) - def clean(self, session, resolver, fixers): + def clean(self, session, resolver): session.check_call(["go", "clean"]) def get_declared_dependencies(self, session, fixers=None): @@ -1365,7 +1633,8 @@ class Golang(BuildSystem): yield "build", GoRequirement(parts[1]) elif parts[0] == "require": yield "build", GoPackageRequirement( - parts[1], parts[2].lstrip("v") if len(parts) > 2 else None + parts[1], + parts[2].lstrip("v") if len(parts) > 2 else None ) elif parts[0] == "exclude": pass # TODO(jelmer): Create conflicts? @@ -1374,7 +1643,8 @@ class Golang(BuildSystem): elif parts[0] == "module": pass else: - logging.warning("Unknown directive %s in go.mod", parts[0]) + logging.warning( + "Unknown directive %s in go.mod", parts[0]) @classmethod def probe(cls, path): @@ -1398,25 +1668,28 @@ class Maven(BuildSystem): def __init__(self, path): self.path = path + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, "pom.xml")): logging.debug("Found pom.xml, assuming maven package.") return cls(os.path.join(path, "pom.xml")) - def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["mvn", "test"], fixers) + def test(self, session, resolver): + run_detecting_problems(session, ["mvn", "test"]) - def clean(self, session, resolver, fixers): - run_with_build_fixers(session, ["mvn", "clean"], fixers) + def clean(self, session, resolver): + run_detecting_problems(session, ["mvn", "clean"]) - def install(self, session, resolver, fixers, install_target): - run_with_build_fixers(session, ["mvn", "install"], fixers) + def install(self, session, resolver, install_target): + run_detecting_problems(session, ["mvn", "install"]) - def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["mvn", "compile"], fixers) + def build(self, session, resolver): + run_detecting_problems(session, ["mvn", "compile"]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): # TODO(jelmer): 'mvn generate-sources' creates a jar in target/. # is that what we need? raise NotImplementedError @@ -1453,31 +1726,30 @@ class Cabal(BuildSystem): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def _run(self, session, args, fixers): + def _run(self, session, args): try: - run_with_build_fixers(session, ["runhaskell", "Setup.hs"] + args, fixers) + run_detecting_problems( + session, ["runhaskell", "Setup.hs"] + args) except UnidentifiedError as e: if "Run the 'configure' command first." in e.lines: - run_with_build_fixers( - session, ["runhaskell", "Setup.hs", "configure"], fixers - ) - run_with_build_fixers( - session, ["runhaskell", "Setup.hs"] + args, fixers - ) + run_detecting_problems( + session, ["runhaskell", "Setup.hs", "configure"]) + run_detecting_problems( + session, ["runhaskell", "Setup.hs"] + args) else: raise - def test(self, session, resolver, fixers): - self._run(session, ["test"], fixers) + def test(self, session, resolver): + self._run(session, ["test"]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): + def dist(self, session, resolver, target_directory, quiet=False): with DistCatcher( [ session.external_path("dist-newstyle/sdist"), session.external_path("dist"), ] ) as dc: - self._run(session, ["sdist"], fixers) + self._run(session, ["sdist"]) return dc.copy_single(target_directory) @classmethod @@ -1515,63 +1787,70 @@ class PerlBuildTiny(BuildSystem): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def setup(self, session, fixers): - run_with_build_fixers(session, ["perl", "Build.PL"], fixers) + def setup(self, session, fixers=None): + run_with_build_fixers(fixers, session, ["perl", "Build.PL"]) - def test(self, session, resolver, fixers): - self.setup(session, fixers) + def test(self, session, resolver): + self.setup(session) if self.minilla: - run_with_build_fixers(session, ["minil", "test"], fixers) + run_detecting_problems(session, ["minil", "test"]) else: - run_with_build_fixers(session, ["./Build", "test"], fixers) + run_detecting_problems(session, ["./Build", "test"]) - def build(self, session, resolver, fixers): - self.setup(session, fixers) - run_with_build_fixers(session, ["./Build", "build"], fixers) + def build(self, session, resolver): + self.setup(session) + run_detecting_problems(session, ["./Build", "build"]) - def clean(self, session, resolver, fixers): - self.setup(session, fixers) - run_with_build_fixers(session, ["./Build", "clean"], fixers) + def clean(self, session, resolver): + self.setup(session) + run_detecting_problems(session, ["./Build", "clean"]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): - self.setup(session, fixers) + def dist(self, session, resolver, target_directory, quiet=False): + self.setup(session) with DistCatcher([session.external_path('.')]) as dc: if self.minilla: - # minil seems to return 0 even if it didn't produce a tarball :( - run_with_build_fixers( - session, ["minil", "dist"], fixers, + # minil seems to return 0 even if it didn't produce a tarball + # :( + run_detecting_problems( + session, ["minil", "dist"], check_success=lambda retcode, lines: bool(dc.find_files())) else: try: - run_with_build_fixers(session, ["./Build", "dist"], fixers) + run_detecting_problems(session, ["./Build", "dist"]) except UnidentifiedError as e: - if "Can't find dist packages without a MANIFEST file" in e.lines: - run_with_build_fixers(session, ["./Build", "manifest"], fixers) - run_with_build_fixers(session, ["./Build", "dist"], fixers) + if ("Can't find dist packages without a MANIFEST file" + in e.lines): + run_detecting_problems( + session, ["./Build", "manifest"]) + run_detecting_problems(session, ["./Build", "dist"]) elif "No such action 'dist'" in e.lines: raise NotImplementedError else: raise return dc.copy_single(target_directory) - def install(self, session, resolver, fixers, install_target): - self.setup(session, fixers) + def install(self, session, resolver, install_target): + self.setup(session) if self.minilla: - run_with_build_fixers(session, ["minil", "install"], fixers) + run_detecting_problems(session, ["minil", "install"]) else: - run_with_build_fixers(session, ["./Build", "install"], fixers) + run_detecting_problems(session, ["./Build", "install"]) def get_declared_dependencies(self, session, fixers=None): self.setup(session, fixers) if self.minilla: - pass # Minilla doesn't seem to have a way to just regenerate the metadata :( + # Minilla doesn't seem to have a way to just regenerate the + # metadata :( + pass else: try: - run_with_build_fixers(session, ["./Build", "distmeta"], fixers) + run_with_build_fixers(fixers, session, ["./Build", "distmeta"]) except UnidentifiedError as e: if "No such action 'distmeta'" in e.lines: pass - if "Do not run distmeta. Install Minilla and `minil install` instead." in e.lines: + if ("Do not run distmeta. " + "Install Minilla and `minil install` instead." + in e.lines): self.minilla = True else: raise @@ -1584,11 +1863,12 @@ class PerlBuildTiny(BuildSystem): @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, "Build.PL")): - logging.debug("Found Build.PL, assuming Module::Build::Tiny package.") + logging.debug( + "Found Build.PL, assuming Module::Build::Tiny package.") return cls(path) -BUILDSYSTEM_CLSES = [ +BUILDSYSTEM_CLSES: List[Type[BuildSystem]] = [ Pear, SetupPy, Npm, @@ -1605,6 +1885,8 @@ BUILDSYSTEM_CLSES = [ R, Octave, Bazel, + CMake, + GnomeShellExtension, # Make is intentionally at the end of the list. Make, Composer, @@ -1612,21 +1894,30 @@ BUILDSYSTEM_CLSES = [ ] -def scan_buildsystems(path): +def lookup_buildsystem_cls(name: str) -> Type[BuildSystem]: + for bs in BUILDSYSTEM_CLSES: + if bs.name == name: + return bs + raise KeyError(name) + + +def scan_buildsystems(path: str) -> List[Tuple[str, BuildSystem]]: """Detect build systems.""" ret = [] - ret.extend([(".", bs) for bs in detect_buildsystems(path)]) + ret.extend([(".", bs) for bs in detect_buildsystems(path) if bs]) if not ret: # Nothing found. Try the next level? for entry in os.scandir(path): if entry.is_dir(): - ret.extend([(entry.name, bs) for bs in detect_buildsystems(entry.path)]) + ret.extend([ + (entry.name, bs) + for bs in detect_buildsystems(entry.path)]) return ret -def detect_buildsystems(path): +def detect_buildsystems(path: str) -> Iterable[BuildSystem]: for bs_cls in BUILDSYSTEM_CLSES: bs = bs_cls.probe(path) if bs is not None: diff --git a/ognibuild/clean.py b/ognibuild/clean.py index 6bbb3ee..2f76390 100644 --- a/ognibuild/clean.py +++ b/ognibuild/clean.py @@ -15,16 +15,25 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from functools import partial + +from .fix_build import iterate_with_build_fixers from .buildsystem import NoBuildToolsFound +from .logs import NoLogManager -def run_clean(session, buildsystems, resolver, fixers): +def run_clean(session, buildsystems, resolver, fixers, log_manager=None): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() + if log_manager is None: + log_manager = NoLogManager() + for buildsystem in buildsystems: - buildsystem.clean(session, resolver, fixers) + iterate_with_build_fixers( + fixers, log_manager.wrap( + partial(buildsystem.clean, session, resolver))) return raise NoBuildToolsFound() diff --git a/ognibuild/debian/__init__.py b/ognibuild/debian/__init__.py index 23a56a1..cf18419 100644 --- a/ognibuild/debian/__init__.py +++ b/ognibuild/debian/__init__.py @@ -29,7 +29,8 @@ def satisfy_build_deps(session: Session, tree, debian_path): deps.append(source[name].strip().strip(",")) except KeyError: pass - for name in ["Build-Conflicts", "Build-Conflicts-Indep", "Build-Conflicts-Arch"]: + for name in ["Build-Conflicts", "Build-Conflicts-Indep", + "Build-Conflicts-Arch"]: try: deps.append("Conflicts: " + source[name]) except KeyError: diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 1e704b0..52789e7 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -16,8 +16,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from debian.changelog import Version import logging -from typing import List, Optional +from typing import List, Optional, Iterable import os from buildlog_consultant.apt import ( @@ -37,7 +38,12 @@ from .file_search import ( def run_apt( session: Session, args: List[str], prefix: Optional[List[str]] = None ) -> None: - """Run apt.""" + """Run apt. + + Raises: + DetailedFailure: When a known error occurs + UnidentifiedError: If an unknown error occurs + """ if prefix is None: prefix = [] args = prefix = ["apt", "-y"] + args @@ -48,7 +54,7 @@ def run_apt( match, error = find_apt_get_failure(lines) if error is not None: raise DetailedFailure(retcode, args, error) - while lines and lines[-1] == "": + while lines and lines[-1].rstrip('\n') == "": lines.pop(-1) raise UnidentifiedError(retcode, args, lines, secondary=match) @@ -93,13 +99,18 @@ class AptManager(object): def package_exists(self, package): return package in self.apt_cache - def package_versions(self, package): - return list(self.apt_cache[package].versions) + def package_versions(self, package: str) -> Optional[Iterable[Version]]: + try: + return list(self.apt_cache[package].versions) + except KeyError: + return None - def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): + async def get_packages_for_paths( + self, paths, regex: bool = False, case_insensitive: bool = False): logging.debug("Searching for packages containing %r", paths) - return get_packages_for_paths( - paths, self.searchers(), regex=regex, case_insensitive=case_insensitive + return await get_packages_for_paths( + paths, self.searchers(), regex=regex, + case_insensitive=case_insensitive ) def missing(self, packages): diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 29d689e..eb655a5 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -17,6 +17,7 @@ __all__ = [ "get_build_architecture", + "version_add_suffix", "add_dummy_changelog_entry", "build", "DetailedDebianBuildFailure", @@ -24,20 +25,22 @@ __all__ = [ ] from datetime import datetime -from debmutate.changelog import ChangelogEditor import logging import os import re import shlex import subprocess import sys +from typing import Optional, List, Tuple -from debian.changelog import Changelog -from debmutate.changelog import get_maintainer +from debian.changelog import Changelog, Version, ChangeBlock +from debmutate.changelog import get_maintainer, ChangelogEditor +from debmutate.reformatting import GeneratedFile from breezy.mutabletree import MutableTree from breezy.plugins.debian.builder import BuildFailedError from breezy.tree import Tree +from breezy.workingtree import WorkingTree from buildlog_consultant.sbuild import ( worker_failure_from_sbuild_log, @@ -45,10 +48,18 @@ from buildlog_consultant.sbuild import ( from .. import DetailedFailure as DetailedFailure, UnidentifiedError +BUILD_LOG_FILENAME = 'build.log' DEFAULT_BUILDER = "sbuild --no-clean-source" +class ChangelogNotEditable(Exception): + """Changelog can not be edited.""" + + def __init__(self, path): + self.path = path + + class DetailedDebianBuildFailure(DetailedFailure): def __init__(self, stage, phase, retcode, argv, error, description): @@ -60,7 +71,8 @@ class DetailedDebianBuildFailure(DetailedFailure): class UnidentifiedDebianBuildError(UnidentifiedError): - def __init__(self, stage, phase, retcode, argv, lines, description, secondary=None): + def __init__(self, stage, phase, retcode, argv, lines, description, + secondary=None): super(UnidentifiedDebianBuildError, self).__init__( retcode, argv, lines, secondary) self.stage = stage @@ -75,11 +87,12 @@ class MissingChangesFile(Exception): self.filename = filename -def find_changes_files(path, package, version): - non_epoch_version = version.upstream_version +def find_changes_files(path: str, package: str, version: Version): + non_epoch_version = version.upstream_version or '' if version.debian_version is not None: non_epoch_version += "-%s" % version.debian_version - c = re.compile('%s_%s_(.*).changes' % (re.escape(package), re.escape(non_epoch_version))) + c = re.compile('%s_%s_(.*).changes' % ( + re.escape(package), re.escape(non_epoch_version))) for entry in os.scandir(path): m = c.match(entry.name) if m: @@ -109,15 +122,32 @@ def control_files_in_root(tree: Tree, subpath: str) -> bool: return False +def version_add_suffix(version: Version, suffix: str) -> Version: + version = Version(str(version)) + + def add_suffix(v): + m = re.fullmatch("(.*)(" + re.escape(suffix) + ")([0-9]+)", v) + if m: + return m.group(1) + m.group(2) + "%d" % (int(m.group(3)) + 1) + else: + return v + suffix + "1" + if version.debian_revision: + version.debian_revision = add_suffix(version.debian_revision) + else: + version.upstream_version = add_suffix(version.upstream_version) + return version + + def add_dummy_changelog_entry( tree: MutableTree, subpath: str, suffix: str, suite: str, message: str, - timestamp=None, - maintainer=None, -): + timestamp: Optional[datetime] = None, + maintainer: Tuple[Optional[str], Optional[str]] = None, + allow_reformatting: bool = True, +) -> Version: """Add a dummy changelog entry to a package. Args: @@ -125,18 +155,10 @@ def add_dummy_changelog_entry( suffix: Suffix for the version suite: Debian suite message: Changelog message + Returns: + version of the newly added entry """ - def add_suffix(v, suffix): - m = re.fullmatch( - "(.*)(" + re.escape(suffix) + ")([0-9]+)", - v, - ) - if m: - return m.group(1) + m.group(2) + "%d" % (int(m.group(3)) + 1) - else: - return v + suffix + "1" - if control_files_in_root(tree, subpath): path = os.path.join(subpath, "changelog") else: @@ -145,38 +167,38 @@ def add_dummy_changelog_entry( maintainer = get_maintainer() if timestamp is None: timestamp = datetime.now() - with ChangelogEditor(tree.abspath(os.path.join(path))) as editor: - version = editor[0].version - if version.debian_revision: - version.debian_revision = add_suffix(version.debian_revision, suffix) - else: - version.upstream_version = add_suffix(version.upstream_version, suffix) - editor.auto_version(version, timestamp=timestamp) - editor.add_entry( - summary=[message], maintainer=maintainer, timestamp=timestamp, urgency='low') - editor[0].distributions = suite + try: + with ChangelogEditor( + tree.abspath(path), # type: ignore + allow_reformatting=allow_reformatting) as editor: + version = version_add_suffix(editor[0].version, suffix) + editor.auto_version(version, timestamp=timestamp) + editor.add_entry( + summary=[message], maintainer=maintainer, timestamp=timestamp, + urgency='low') + editor[0].distributions = suite + return version + except GeneratedFile as e: + raise ChangelogNotEditable(path) from e -def get_latest_changelog_entry(local_tree, subpath=""): +def get_latest_changelog_entry( + local_tree: WorkingTree, subpath: str = "") -> ChangeBlock: if control_files_in_root(local_tree, subpath): path = os.path.join(subpath, "changelog") else: path = os.path.join(subpath, "debian", "changelog") with local_tree.get_file(path) as f: cl = Changelog(f, max_blocks=1) - return cl.package, cl.version + return cl[0] -def build( - local_tree, - outf, - build_command=DEFAULT_BUILDER, - result_dir=None, - distribution=None, - subpath="", - source_date_epoch=None, - extra_repositories=None, -): +def _builddeb_command( + build_command: str = DEFAULT_BUILDER, + result_dir: Optional[str] = None, + apt_repository: Optional[str] = None, + apt_repository_key: Optional[str] = None, + extra_repositories: Optional[List[str]] = None): for repo in extra_repositories or []: build_command += " --extra-repository=" + shlex.quote(repo) args = [ @@ -187,8 +209,34 @@ def build( "--guess-upstream-branch-url", "--builder=%s" % build_command, ] + if apt_repository: + args.append("--apt-repository=%s" % apt_repository) + if apt_repository_key: + args.append("--apt-repository-key=%s" % apt_repository_key) if result_dir: args.append("--result-dir=%s" % result_dir) + return args + + +def build( + local_tree: WorkingTree, + outf, + build_command: str = DEFAULT_BUILDER, + result_dir: Optional[str] = None, + distribution: Optional[str] = None, + subpath: str = "", + source_date_epoch: Optional[int] = None, + apt_repository: Optional[str] = None, + apt_repository_key: Optional[str] = None, + extra_repositories: Optional[List[str]] = None, +): + args = _builddeb_command( + build_command=build_command, + result_dir=result_dir, + apt_repository=apt_repository, + apt_repository_key=apt_repository_key, + extra_repositories=extra_repositories) + outf.write("Running %r\n" % (build_command,)) outf.flush() env = dict(os.environ.items()) @@ -199,22 +247,25 @@ def build( logging.info("Building debian packages, running %r.", build_command) try: subprocess.check_call( - args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf, env=env + args, cwd=local_tree.abspath(subpath), stdout=outf, stderr=outf, + env=env ) except subprocess.CalledProcessError: raise BuildFailedError() def build_once( - local_tree, - build_suite, - output_directory, - build_command, - subpath="", - source_date_epoch=None, - extra_repositories=None + local_tree: WorkingTree, + build_suite: str, + output_directory: str, + build_command: str, + subpath: str = "", + source_date_epoch: Optional[int] = None, + apt_repository: Optional[str] = None, + apt_repository_key: Optional[str] = None, + extra_repositories: Optional[List[str]] = None ): - build_log_path = os.path.join(output_directory, "build.log") + build_log_path = os.path.join(output_directory, BUILD_LOG_FILENAME) logging.debug("Writing build log to %s", build_log_path) try: with open(build_log_path, "w") as f: @@ -226,6 +277,8 @@ def build_once( distribution=build_suite, subpath=subpath, source_date_epoch=source_date_epoch, + apt_repository=apt_repository, + apt_repository_key=apt_repository_key, extra_repositories=extra_repositories, ) except BuildFailedError as e: @@ -247,27 +300,39 @@ def build_once( [], sbuild_failure.description) cl_entry = get_latest_changelog_entry(local_tree, subpath) + if cl_entry.package is None: + raise Exception('missing package in changelog entry') changes_names = [] - for kind, entry in find_changes_files(output_directory, cl_entry.package, cl_entry.version): + for kind, entry in find_changes_files( + output_directory, cl_entry.package, cl_entry.version): changes_names.append((entry.name)) return (changes_names, cl_entry) +class GitBuildpackageMissing(Exception): + """git-buildpackage is not installed""" + + def gbp_dch(path): - subprocess.check_call(["gbp", "dch", "--ignore-branch"], cwd=path) + try: + subprocess.check_call(["gbp", "dch", "--ignore-branch"], cwd=path) + except FileNotFoundError: + raise GitBuildpackageMissing() def attempt_build( - local_tree, - suffix, - build_suite, - output_directory, - build_command, - build_changelog_entry=None, - subpath="", - source_date_epoch=None, - run_gbp_dch=False, - extra_repositories=None + local_tree: WorkingTree, + suffix: str, + build_suite: str, + output_directory: str, + build_command: str, + build_changelog_entry: Optional[str] = None, + subpath: str = "", + source_date_epoch: Optional[int] = None, + run_gbp_dch: bool = False, + apt_repository: Optional[str] = None, + apt_repository_key: Optional[str] = None, + extra_repositories: Optional[List[str]] = None ): """Attempt a build, with a custom distribution set. @@ -282,7 +347,7 @@ def attempt_build( source_date_epoch: Source date epoch to set Returns: Tuple with (changes_name, cl_version) """ - if run_gbp_dch and not subpath: + if run_gbp_dch and not subpath and hasattr(local_tree.controldir, '_git'): gbp_dch(local_tree.abspath(subpath)) if build_changelog_entry is not None: add_dummy_changelog_entry( @@ -295,5 +360,7 @@ def attempt_build( build_command, subpath, source_date_epoch=source_date_epoch, + apt_repository=apt_repository, + apt_repository_key=apt_repository_key, extra_repositories=extra_repositories, ) diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index 246a058..0241f9f 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -18,43 +18,46 @@ """Tie breaking by build deps.""" +from debian.deb822 import PkgRelation import logging +from breezy.plugins.debian.apt_repo import LocalApt, NoAptSources + class BuildDependencyTieBreaker(object): - def __init__(self, rootdir): - self.rootdir = rootdir + def __init__(self, apt): + self.apt = apt self._counts = None def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.rootdir) + return "%s(%r)" % (type(self).__name__, self.apt) @classmethod def from_session(cls, session): - return cls(session.location) + return cls(LocalApt(session.location)) def _count(self): counts = {} - import apt_pkg - - apt_pkg.init() - apt_pkg.config.set("Dir", self.rootdir) - apt_cache = apt_pkg.SourceRecords() - apt_cache.restart() - while apt_cache.step(): - try: - for d in apt_cache.build_depends.values(): - for o in d: - for p in o: - counts.setdefault(p[0], 0) - counts[p[0]] += 1 - except AttributeError: - pass + with self.apt: + for source in self.apt.iter_sources(): + for field in ['Build-Depends', 'Build-Depends-Indep', + 'Build-Depends-Arch']: + for r in PkgRelation.parse_relations( + source.get(field, '')): + for p in r: + counts.setdefault(p['name'], 0) + counts[p['name']] += 1 return counts def __call__(self, reqs): if self._counts is None: - self._counts = self._count() + try: + self._counts = self._count() + except NoAptSources: + logging.warning( + "No 'deb-src' in sources.list, " + "unable to break build-depends") + return None by_count = {} for req in reqs: try: @@ -80,5 +83,5 @@ if __name__ == "__main__": parser.add_argument("req", nargs="+") args = parser.parse_args() reqs = [AptRequirement.from_str(req) for req in args.req] - tie_breaker = BuildDependencyTieBreaker("/") + tie_breaker = BuildDependencyTieBreaker(LocalApt()) print(tie_breaker(reqs)) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 5276835..f9422c7 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -17,12 +17,13 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import apt_pkg +import asyncio from datetime import datetime from debian.deb822 import Release import os import re import subprocess -from typing import Iterator, List +from typing import List, AsyncIterator import logging @@ -32,11 +33,15 @@ from ..session import Session class FileSearcher(object): def search_files( - self, path: str, regex: bool = False, case_insensitive: bool = False - ) -> Iterator[str]: + self, path: str, regex: bool = False, + case_insensitive: bool = False) -> AsyncIterator[str]: raise NotImplementedError(self.search_files) +class AptFileAccessError(Exception): + """Apt file access error.""" + + class ContentsFileNotFound(Exception): """The contents file was not found.""" @@ -71,7 +76,8 @@ def contents_urls_from_sources_entry(source, arches, load_url): response = load_url(release_url) except FileNotFoundError as e: logging.warning( - "Unable to download %s or %s: %s", inrelease_url, release_url, e + "Unable to download %s or %s: %s", inrelease_url, + release_url, e ) return @@ -118,7 +124,7 @@ def _unwrap(f, ext): def load_direct_url(url): - from urllib.error import HTTPError + from urllib.error import HTTPError, URLError from urllib.request import urlopen, Request for ext in [".xz", ".gz", ""]: @@ -128,7 +134,11 @@ def load_direct_url(url): except HTTPError as e: if e.status == 404: continue - raise + raise AptFileAccessError( + 'Unable to access apt URL %s: %s' % (url + ext, e)) + except URLError as e: + raise AptFileAccessError( + 'Unable to access apt URL %s: %s' % (url + ext, e)) break else: raise FileNotFoundError(url) @@ -187,7 +197,7 @@ class AptFileFileSearcher(FileSearcher): @classmethod def from_session(cls, session): - logging.info('Using apt-file to search apt contents') + logging.debug('Using apt-file to search apt contents') if not os.path.exists(session.external_path(cls.CACHE_IS_EMPTY_PATH)): from .apt import AptManager AptManager.from_session(session).install(['apt-file']) @@ -195,7 +205,7 @@ class AptFileFileSearcher(FileSearcher): session.check_call(['apt-file', 'update'], user='root') return cls(session) - def search_files(self, path, regex=False, case_insensitive=False): + async def search_files(self, path, regex=False, case_insensitive=False): args = [] if regex: args.append('-x') @@ -204,15 +214,17 @@ class AptFileFileSearcher(FileSearcher): if case_insensitive: args.append('-i') args.append(path) - try: - output = self.session.check_output(['/usr/bin/apt-file', 'search'] + args) - except subprocess.CalledProcessError as e: - if e.returncode == 1: - # No results - return - if e.returncode == 3: - raise Exception('apt-file cache is empty') - raise + process = await asyncio.create_subprocess_exec( + '/usr/bin/apt-file', 'search', *args, + stdout=asyncio.subprocess.PIPE) + (output, error) = await process.communicate(input=None) + if process.returncode == 1: + # No results + return + elif process.returncode == 3: + raise Exception('apt-file cache is empty') + elif process.returncode != 0: + raise Exception("unexpected return code %d" % process.returncode) for line in output.splitlines(False): pkg, path = line.split(b': ') @@ -253,7 +265,8 @@ class RemoteContentsFileSearcher(FileSearcher): return load_url_with_cache(url, cache_dirs) urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture(), load_url) + contents_urls_from_sourceslist( + sl, get_build_architecture(), load_url) ) self._load_urls(urls, cache_dirs, load_url) @@ -277,8 +290,8 @@ class RemoteContentsFileSearcher(FileSearcher): return load_url_with_cache(url, cache_dirs) urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture(), load_url) - ) + contents_urls_from_sourceslist( + sl, get_build_architecture(), load_url)) self._load_urls(urls, cache_dirs, load_url) def _load_urls(self, urls, cache_dirs, load_url): @@ -286,13 +299,16 @@ class RemoteContentsFileSearcher(FileSearcher): try: f = load_url(url) self.load_file(f, url) + except ConnectionResetError: + logging.warning("Connection reset error retrieving %s", url) + # TODO(jelmer): Retry? except ContentsFileNotFound: logging.warning("Unable to fetch contents file %s", url) def __setitem__(self, path, package): self._db[path] = package - def search_files(self, path, regex=False, case_insensitive=False): + async def search_files(self, path, regex=False, case_insensitive=False): path = path.lstrip("/").encode("utf-8", "surrogateescape") if case_insensitive and not regex: regex = True @@ -338,9 +354,9 @@ class GeneratedFileSearcher(FileSearcher): (path, pkg) = line.strip().split(None, 1) self._db.append(path, pkg) - def search_files( - self, path: str, regex: bool = False, case_insensitive: bool = False - ) -> Iterator[str]: + async def search_files( + self, path: str, regex: bool = False, + case_insensitive: bool = False): for p, pkg in self._db: if regex: flags = 0 @@ -371,16 +387,17 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( ) -def get_packages_for_paths( +async def get_packages_for_paths( paths: List[str], searchers: List[FileSearcher], regex: bool = False, case_insensitive: bool = False, ) -> List[str]: candidates: List[str] = list() + # TODO(jelmer): Combine these, perhaps by creating one gigantic regex? for path in paths: for searcher in searchers: - for pkg in searcher.search_files( + async for pkg in searcher.search_files( path, regex=regex, case_insensitive=case_insensitive ): if pkg not in candidates: @@ -393,8 +410,10 @@ def main(argv): from ..session.plain import PlainSession parser = argparse.ArgumentParser() - parser.add_argument("path", help="Path to search for.", type=str, nargs="*") - parser.add_argument("--regex", "-x", help="Search for regex.", action="store_true") + parser.add_argument( + "path", help="Path to search for.", type=str, nargs="*") + parser.add_argument( + "--regex", "-x", help="Search for regex.", action="store_true") parser.add_argument("--debug", action="store_true") args = parser.parse_args() @@ -403,13 +422,14 @@ def main(argv): else: logging.basicConfig(level=logging.INFO) - main_searcher = get_apt_contents_file_searcher(PlainSession()) - main_searcher.load_local() - searchers = [main_searcher, GENERATED_FILE_SEARCHER] + with PlainSession() as session: + main_searcher = get_apt_contents_file_searcher(session) + searchers = [main_searcher, GENERATED_FILE_SEARCHER] - packages = get_packages_for_paths(args.path, searchers=searchers, regex=args.regex) - for package in packages: - print(package) + packages = asyncio.run(get_packages_for_paths( + args.path, searchers=searchers, regex=args.regex)) + for package in packages: + print(package) if __name__ == "__main__": diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index ffb064d..8e8a76b 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -22,10 +22,10 @@ __all__ = [ from functools import partial import logging import os -import re import shutil import sys -from typing import List, Set, Optional, Type +import time +from typing import List, Set, Optional, Type, Tuple from debian.deb822 import ( Deb822, @@ -34,6 +34,8 @@ from debian.deb822 import ( from breezy.commit import PointlessCommit, NullCommitReporter from breezy.tree import Tree +from breezy.workingtree import WorkingTree + from debmutate.changelog import ChangelogEditor from debmutate.control import ( ensure_relation, @@ -50,49 +52,7 @@ from debmutate.reformatting import ( GeneratedFile, ) -try: - from breezy.workspace import reset_tree -except ImportError: # breezy < 3.2 - - def delete_items(deletables, dry_run=False): - """Delete files in the deletables iterable""" - import errno - import shutil - - def onerror(function, path, excinfo): - """Show warning for errors seen by rmtree.""" - # Handle only permission error while removing files. - # Other errors are re-raised. - if function is not os.remove or excinfo[1].errno != errno.EACCES: - raise - logging.warning("unable to remove %s" % path) - - for path, subp in deletables: - if os.path.isdir(path): - shutil.rmtree(path, onerror=onerror) - else: - try: - os.unlink(path) - except OSError as e: - # We handle only permission error here - if e.errno != errno.EACCES: - raise e - logging.warning('unable to remove "%s": %s.', path, e.strerror) - - def reset_tree(local_tree, subpath=""): - from breezy.transform import revert - from breezy.clean_tree import iter_deletables - - revert( - local_tree, - local_tree.branch.basis_tree(), - [subpath] if subpath not in (".", "") else None, - ) - deletables = list( - iter_deletables(local_tree, unknown=True, ignored=False, detritus=False) - ) - delete_items(deletables) - +from breezy.workspace import reset_tree from debmutate._rules import ( dh_invoke_add_with, @@ -113,18 +73,21 @@ from buildlog_consultant.common import ( ) from buildlog_consultant.sbuild import ( DebcargoUnacceptablePredicate, + DebcargoUnacceptableComparator, ) from .build import ( DetailedDebianBuildFailure, UnidentifiedDebianBuildError, ) +from ..logs import rotate_logfile from ..buildlog import problem_to_upstream_requirement from ..fix_build import BuildFixer, resolve_error from ..resolver.apt import ( AptRequirement, ) -from .build import attempt_build, DEFAULT_BUILDER +from .apt import AptManager +from .build import attempt_build, DEFAULT_BUILDER, BUILD_LOG_FILENAME DEFAULT_MAX_ITERATIONS = 10 @@ -150,7 +113,9 @@ class DebianPackagingContext(object): def abspath(self, *parts): return self.tree.abspath(os.path.join(self.subpath, *parts)) - def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool: + def commit( + self, summary: str, + update_changelog: Optional[bool] = None) -> bool: if update_changelog is None: update_changelog = self.update_changelog with self.tree.lock_write(): @@ -214,6 +179,11 @@ def add_dependency(context, phase, requirement: AptRequirement): return add_test_dependency(context, phase[1], requirement) elif phase[0] == "build": return add_build_dependency(context, requirement) + elif phase[0] == "buildenv": + # TODO(jelmer): Actually, we probably just want to install it on the + # host system? + logging.warning("Unknown phase %r", phase) + return False else: logging.warning("Unknown phase %r", phase) return False @@ -231,16 +201,19 @@ def add_build_dependency(context, requirement: AptRequirement): raise CircularDependency(binary["Package"]) for rel in requirement.relations: updater.source["Build-Depends"] = ensure_relation( - updater.source.get("Build-Depends", ""), PkgRelation.str([rel]) + updater.source.get("Build-Depends", ""), + PkgRelation.str([rel]) ) except FormattingUnpreservable as e: - logging.info("Unable to edit %s in a way that preserves formatting.", e.path) + logging.info( + "Unable to edit %s in a way that preserves formatting.", e.path) return False desc = requirement.pkg_relation_str() if not updater.changed: - logging.info("Giving up; dependency %s was already present.", desc) + logging.info( + "Giving up; build dependency %s was already present.", desc) return False logging.info("Adding build dependency: %s", desc) @@ -272,13 +245,18 @@ def add_test_dependency(context, testname, requirement): control.get("Depends", ""), PkgRelation.str([rel]) ) except FormattingUnpreservable as e: - logging.info("Unable to edit %s in a way that preserves formatting.", e.path) - return False - if not updater.changed: + logging.info( + "Unable to edit %s in a way that preserves formatting.", e.path) return False desc = requirement.pkg_relation_str() + if not updater.changed: + logging.info( + "Giving up; dependency %s for test %s was already present.", + desc, testname) + return False + logging.info("Adding dependency to test %s: %s", testname, desc) return context.commit( "Add missing dependency for test %s on %s." % (testname, desc), @@ -288,7 +266,8 @@ def add_test_dependency(context, testname, requirement): def targeted_python_versions(tree: Tree, subpath: str) -> List[str]: with tree.get_file(os.path.join(subpath, "debian/control")) as f: control = Deb822(f) - build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) + build_depends = PkgRelation.parse_relations( + control.get("Build-Depends", "")) all_build_deps: Set[str] = set() for or_deps in build_depends: all_build_deps.update(or_dep["name"] for or_dep in or_deps) @@ -312,7 +291,7 @@ def python_tie_breaker(tree, subpath, reqs): return True if pkg.startswith("lib%s-" % python_version): return True - if re.match(r'lib%s\.[0-9]-dev' % python_version, pkg): + if pkg == r'lib%s-dev' % python_version: return True return False @@ -337,7 +316,8 @@ def retry_apt_failure(error, phase, apt, context): def enable_dh_autoreconf(context, phase): # Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by # default. - debhelper_compat_version = get_debhelper_compat_level(context.tree.abspath(".")) + debhelper_compat_version = get_debhelper_compat_level( + context.tree.abspath(".")) if debhelper_compat_version is not None and debhelper_compat_version < 10: def add_with_autoreconf(line, target): @@ -356,9 +336,8 @@ def enable_dh_autoreconf(context, phase): def fix_missing_configure(error, phase, context): - if not context.tree.has_filename("configure.ac") and not context.tree.has_filename( - "configure.in" - ): + if (not context.tree.has_filename("configure.ac") + and not context.tree.has_filename("configure.in")): return False return enable_dh_autoreconf(context, phase) @@ -433,7 +412,7 @@ def fix_missing_makefile_pl(error, phase, context): return False -def coerce_unacceptable_predicate(error, phase, context): +def debcargo_coerce_unacceptable_prerelease(error, phase, context): from debmutate.debcargo import DebcargoEditor with DebcargoEditor(context.abspath('debian/debcargo.toml')) as editor: editor['allow_prerelease_deps'] = True @@ -461,7 +440,8 @@ class SimpleBuildFixer(BuildFixer): class DependencyBuildFixer(BuildFixer): - def __init__(self, packaging_context, apt_resolver, problem_cls: Type[Problem], fn): + def __init__(self, packaging_context, apt_resolver, + problem_cls: Type[Problem], fn): self.context = packaging_context self.apt_resolver = apt_resolver self._problem_cls = problem_cls @@ -481,32 +461,47 @@ class DependencyBuildFixer(BuildFixer): return self._fn(problem, phase, self.apt_resolver, self.context) -def versioned_package_fixers(session, packaging_context, apt): +def versioned_package_fixers(session, packaging_context, apt: AptManager): return [ PgBuildExtOutOfDateControlFixer(packaging_context, session, apt), - SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure), + SimpleBuildFixer( + packaging_context, MissingConfigure, fix_missing_configure), SimpleBuildFixer( packaging_context, MissingAutomakeInput, fix_missing_automake_input ), SimpleBuildFixer( - packaging_context, MissingConfigStatusInput, fix_missing_config_status_input + packaging_context, MissingConfigStatusInput, + fix_missing_config_status_input ), - SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl), - SimpleBuildFixer(packaging_context, DebcargoUnacceptablePredicate, coerce_unacceptable_predicate), + SimpleBuildFixer( + packaging_context, MissingPerlFile, fix_missing_makefile_pl), + SimpleBuildFixer( + packaging_context, DebcargoUnacceptablePredicate, + debcargo_coerce_unacceptable_prerelease), + SimpleBuildFixer( + packaging_context, DebcargoUnacceptableComparator, + debcargo_coerce_unacceptable_prerelease), ] -def apt_fixers(apt, packaging_context) -> List[BuildFixer]: +def apt_fixers(apt: AptManager, packaging_context, + dep_server_url: Optional[str] = None) -> List[BuildFixer]: from ..resolver.apt import AptResolver from .udd import popcon_tie_breaker from .build_deps import BuildDependencyTieBreaker apt_tie_breakers = [ - partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), + partial(python_tie_breaker, packaging_context.tree, + packaging_context.subpath), BuildDependencyTieBreaker.from_session(apt.session), popcon_tie_breaker, ] - resolver = AptResolver(apt, apt_tie_breakers) + resolver: AptResolver + if dep_server_url: + from ..resolver.dep_server import DepServerAptResolver + resolver = DepServerAptResolver(apt, dep_server_url, apt_tie_breakers) + else: + resolver = AptResolver(apt, apt_tie_breakers) return [ DependencyBuildFixer( packaging_context, apt, AptFetchFailure, retry_apt_failure @@ -515,38 +510,49 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]: ] -def default_fixers(local_tree, subpath, apt, committer=None, update_changelog=None): +def default_fixers( + local_tree: WorkingTree, + subpath: str, apt: AptManager, + committer: Optional[str] = None, + update_changelog: Optional[bool] = None, + dep_server_url: Optional[str] = None): packaging_context = DebianPackagingContext( local_tree, subpath, committer, update_changelog, commit_reporter=NullCommitReporter() ) - return versioned_package_fixers(apt.session, packaging_context, apt) + apt_fixers( - apt, packaging_context - ) + return (versioned_package_fixers(apt.session, packaging_context, apt) + + apt_fixers(apt, packaging_context, dep_server_url)) def build_incrementally( - local_tree, - apt, - suffix, - build_suite, - output_directory, - build_command, + local_tree: WorkingTree, + apt: AptManager, + suffix: str, + build_suite: str, + output_directory: str, + build_command: str, build_changelog_entry, - committer=None, - max_iterations=DEFAULT_MAX_ITERATIONS, - subpath="", + committer: Optional[str] = None, + max_iterations: int = DEFAULT_MAX_ITERATIONS, + subpath: str = "", source_date_epoch=None, - update_changelog=True, - extra_repositories=None, - fixers=None + update_changelog: bool = True, + apt_repository: Optional[str] = None, + apt_repository_key: Optional[str] = None, + extra_repositories: Optional[List[str]] = None, + fixers: Optional[List[BuildFixer]] = None, + run_gbp_dch: Optional[bool] = None, + dep_server_url: Optional[str] = None, ): - fixed_errors = [] + fixed_errors: List[Tuple[Problem, str]] = [] if fixers is None: fixers = default_fixers( local_tree, subpath, apt, committer=committer, - update_changelog=update_changelog) + update_changelog=update_changelog, + dep_server_url=dep_server_url) logging.info("Using fixers: %r", fixers) + if run_gbp_dch is None: + run_gbp_dch = (update_changelog is False) while True: try: return attempt_build( @@ -558,7 +564,9 @@ def build_incrementally( build_changelog_entry, subpath=subpath, source_date_epoch=source_date_epoch, - run_gbp_dch=(update_changelog is False), + run_gbp_dch=run_gbp_dch, + apt_repository=apt_repository, + apt_repository_key=apt_repository_key, extra_repositories=extra_repositories, ) except UnidentifiedDebianBuildError: @@ -569,15 +577,19 @@ def build_incrementally( logging.info("No relevant context, not making any changes.") raise if (e.error, e.phase) in fixed_errors: - logging.warning("Error was still not fixed on second try. Giving up.") + logging.warning( + "Error was still not fixed on second try. Giving up.") raise - if max_iterations is not None and len(fixed_errors) > max_iterations: - logging.warning("Last fix did not address the issue. Giving up.") + if (max_iterations is not None + and len(fixed_errors) > max_iterations): + logging.warning( + "Last fix did not address the issue. Giving up.") raise reset_tree(local_tree, subpath=subpath) try: if not resolve_error(e.error, e.phase, fixers): - logging.warning("Failed to resolve error %r. Giving up.", e.error) + logging.warning( + "Failed to resolve error %r. Giving up.", e.error) raise except GeneratedFile: logging.warning( @@ -588,71 +600,71 @@ def build_incrementally( raise e except CircularDependency: logging.warning( - "Unable to fix %r; it would introduce a circular " "dependency.", + "Unable to fix %r; it would introduce a circular " + "dependency.", e.error, ) raise e fixed_errors.append((e.error, e.phase)) - if os.path.exists(os.path.join(output_directory, "build.log")): - i = 1 - while os.path.exists( - os.path.join(output_directory, "build.log.%d" % i) - ): - i += 1 - target_path = os.path.join(output_directory, "build.log.%d" % i) - os.rename(os.path.join(output_directory, "build.log"), target_path) - logging.debug("Storing build log at %s", target_path) + rotate_logfile(os.path.join(output_directory, BUILD_LOG_FILENAME)) def main(argv=None): import argparse parser = argparse.ArgumentParser("ognibuild.debian.fix_build") - parser.add_argument( - "--suffix", type=str, help="Suffix to use for test builds.", default="fixbuild1" + modifications = parser.add_argument_group('Modifications') + modifications.add_argument( + "--suffix", type=str, help="Suffix to use for test builds.", + default="fixbuild1" ) - parser.add_argument( + modifications.add_argument( "--suite", type=str, help="Suite to target.", default="unstable" ) - parser.add_argument( - "--output-directory", type=str, help="Output directory.", default=None + modifications.add_argument( + "--committer", type=str, help="Committer string (name and email)", + default=None ) - parser.add_argument( - "--committer", type=str, help="Committer string (name and email)", default=None - ) - parser.add_argument( - "--build-command", - type=str, - help="Build command", - default=(DEFAULT_BUILDER + " -A -s -v"), - ) - parser.add_argument( + modifications.add_argument( "--no-update-changelog", action="store_false", default=None, dest="update_changelog", help="do not update the changelog", ) - parser.add_argument( - '--max-iterations', - type=int, - default=DEFAULT_MAX_ITERATIONS, - help='Maximum number of issues to attempt to fix before giving up.') - parser.add_argument( + modifications.add_argument( "--update-changelog", action="store_true", dest="update_changelog", help="force updating of the changelog", default=None, ) - parser.add_argument("--schroot", type=str, help="chroot to use.") + build_behaviour = parser.add_argument_group('Build Behaviour') + build_behaviour.add_argument( + "--output-directory", type=str, help="Output directory.", default=None + ) + build_behaviour.add_argument( + "--build-command", + type=str, + help="Build command", + default=(DEFAULT_BUILDER + " -A -s -v"), + ) + + build_behaviour.add_argument( + '--max-iterations', + type=int, + default=DEFAULT_MAX_ITERATIONS, + help='Maximum number of issues to attempt to fix before giving up.') + build_behaviour.add_argument("--schroot", type=str, help="chroot to use.") + parser.add_argument( + "--dep-server-url", type=str, + help="ognibuild dep server to use", + default=os.environ.get('OGNIBUILD_DEPS')) parser.add_argument("--verbose", action="store_true", help="Be verbose") args = parser.parse_args() - from breezy.workingtree import WorkingTree import breezy.git # noqa: F401 import breezy.bzr # noqa: F401 - from .apt import AptManager from ..session.plain import PlainSession from ..session.schroot import SchrootSession import tempfile @@ -669,6 +681,10 @@ def main(argv=None): logging.info("Using output directory %s", output_directory) else: output_directory = args.output_directory + if not os.path.isdir(output_directory): + parser.error( + 'output directory %s is not a directory' + % output_directory) tree = WorkingTree.open(".") if args.schroot: @@ -692,6 +708,7 @@ def main(argv=None): committer=args.committer, update_changelog=args.update_changelog, max_iterations=args.max_iterations, + dep_server_url=args.dep_server_url, ) except DetailedDebianBuildFailure as e: if e.phase is None: @@ -701,6 +718,21 @@ def main(argv=None): else: phase = "%s (%s)" % (e.phase[0], e.phase[1]) logging.fatal("Error during %s: %s", phase, e.error) + if not args.output_directory: + xdg_cache_dir = os.environ.get( + 'XDG_CACHE_HOME', os.path.expanduser('~/.cache')) + buildlogs_dir = os.path.join( + xdg_cache_dir, 'ognibuild', 'buildlogs') + os.makedirs(buildlogs_dir, exist_ok=True) + target_log_file = os.path.join( + buildlogs_dir, + '%s-%s.log' % ( + os.path.basename(getattr(tree, 'basedir', 'build')), + time.strftime('%Y-%m-%d_%H%M%s'))) + shutil.copy( + os.path.join(output_directory, 'build.log'), + target_log_file) + logging.info('Build log available in %s', target_log_file) return 1 except UnidentifiedDebianBuildError as e: if e.phase is None: diff --git a/ognibuild/debian/udd.py b/ognibuild/debian/udd.py index 0c73818..3789585 100644 --- a/ognibuild/debian/udd.py +++ b/ognibuild/debian/udd.py @@ -35,7 +35,8 @@ class UDD(object): def get_most_popular(self, packages): cursor = self._conn.cursor() cursor.execute( - "SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1", + "SELECT package FROM popcon " + "WHERE package IN %s ORDER BY insts DESC LIMIT 1", (tuple(packages),), ) return cursor.fetchone()[0] @@ -54,7 +55,8 @@ def popcon_tie_breaker(candidates): names = {list(c.package_names())[0]: c for c in candidates} winner = udd.get_most_popular(list(names.keys())) if winner is None: - logging.warning("No relevant popcon information found, not ranking by popcon") + logging.warning( + "No relevant popcon information found, not ranking by popcon") return None logging.info("Picked winner using popcon") return names[winner] diff --git a/ognibuild/dep_server.py b/ognibuild/dep_server.py new file mode 100644 index 0000000..86ece6f --- /dev/null +++ b/ognibuild/dep_server.py @@ -0,0 +1,126 @@ +#!/usr/bin/python +# Copyright (C) 2022 Jelmer Vernooij +# encoding: utf-8 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import logging +import sys + +from aiohttp import web +from aiohttp_openmetrics import setup_metrics + +from . import Requirement, UnknownRequirementFamily +from .debian.apt import AptManager +from .resolver.apt import resolve_requirement_apt + +SUPPORTED_RELEASES = ['unstable', 'sid'] + + +routes = web.RouteTableDef() + + +@routes.get('/health', name='health') +async def handle_health(request): + return web.Response(text='ok') + + +@routes.get('/families', name='families') +async def handle_families(request): + return web.json_response(list(Requirement._JSON_DESERIALIZERS.keys())) + + +@routes.post('/resolve-apt', name='resolve-apt') +async def handle_apt(request): + js = await request.json() + try: + req_js = js['requirement'] + except KeyError: + raise web.HTTPBadRequest(text="json missing 'requirement' key") + release = js.get('release') + if release and release not in SUPPORTED_RELEASES: + return web.json_response( + {"reason": "unsupported-release", "release": release}, + status=404) + try: + req = Requirement.from_json(req_js) + except UnknownRequirementFamily as e: + return web.json_response( + {"reason": "family-unknown", "family": e.family}, status=404) + apt_reqs = await resolve_requirement_apt(request.app['apt_mgr'], req) + return web.json_response([r.pkg_relation_str() for r in apt_reqs]) + + +@routes.get('/resolve-apt/{release}/{family}:{arg}', name='resolve-apt-simple') +async def handle_apt_simple(request): + if request.match_info['release'] not in SUPPORTED_RELEASES: + return web.json_response( + {"reason": "unsupported-release", + "release": request.match_info['release']}, + status=404) + try: + req = Requirement.from_json( + (request.match_info['family'], request.match_info['arg'])) + except UnknownRequirementFamily as e: + return web.json_response( + {"reason": "family-unknown", "family": e.family}, status=404) + apt_reqs = await resolve_requirement_apt(request.app['apt_mgr'], req) + return web.json_response([r.pkg_relation_str() for r in apt_reqs]) + + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--listen-address', type=str, help='Listen address') + parser.add_argument('--schroot', type=str, help='Schroot session to use') + parser.add_argument('--port', type=str, help='Listen port', default=9934) + parser.add_argument('--debug', action='store_true') + parser.add_argument( + "--gcp-logging", action='store_true', help='Use Google cloud logging.') + args = parser.parse_args() + + if args.gcp_logging: + import google.cloud.logging + client = google.cloud.logging.Client() + client.get_default_handler() + client.setup_logging() + else: + if args.debug: + log_level = logging.DEBUG + else: + log_level = logging.INFO + + logging.basicConfig( + level=log_level, + format="[%(asctime)s] %(message)s", + datefmt="%Y-%m-%d %H:%M:%S") + + if args.schroot: + from .session.schroot import SchrootSession + session = SchrootSession(args.schroot) + else: + from .session.plain import PlainSession + session = PlainSession() + with session: + app = web.Application() + app.router.add_routes(routes) + app['apt_mgr'] = AptManager.from_session(session) + setup_metrics(app) + + web.run_app(app, host=args.listen_address, port=args.port) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 8206920..0eeb8f9 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -18,18 +18,19 @@ __all__ = [ "UnidentifiedError", "DetailedFailure", - "create_dist", + "run_dist", "create_dist_schroot", + "create_dist", + "dist", ] import errno +from functools import partial import logging import os import sys from typing import Optional, List -from debian.deb822 import Deb822 - from breezy.tree import Tree from breezy.workingtree import WorkingTree @@ -37,16 +38,78 @@ from buildlog_consultant.common import ( NoSpaceOnDevice, ) +from debian.deb822 import Deb822 + from . import DetailedFailure, UnidentifiedError from .dist_catcher import DistNoTarball +from .fix_build import iterate_with_build_fixers +from .logs import LogManager, NoLogManager from .buildsystem import NoBuildToolsFound from .resolver import auto_resolver from .session import Session from .session.schroot import SchrootSession -def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=False): +DIST_LOG_FILENAME = 'dist.log' + + +def run_dist(session, buildsystems, resolver, fixers, target_directory, + quiet=False, log_manager=None): + # Some things want to write to the user's home directory, + # e.g. pip caches in ~/.cache + session.create_home() + + logging.info('Using dependency resolver: %s', resolver) + + if log_manager is None: + log_manager = NoLogManager() + + for buildsystem in buildsystems: + filename = iterate_with_build_fixers(fixers, log_manager.wrap( + partial(buildsystem.dist, session, resolver, target_directory, + quiet=quiet))) + return filename + + raise NoBuildToolsFound() + + +def dist(session, export_directory, reldir, target_dir, log_manager, *, + version: Optional[str] = None, quiet=False): + from .fix_build import BuildFixer + from .buildsystem import detect_buildsystems + from .buildlog import InstallFixer + from .fixers import ( + GitIdentityFixer, + MissingGoSumEntryFixer, + SecretGpgKeyFixer, + UnexpandedAutoconfMacroFixer, + GnulibDirectoryFixer, + ) + + if version: + # TODO(jelmer): Shouldn't include backend-specific code here + os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = version + + # TODO(jelmer): use scan_buildsystems to also look in subdirectories + buildsystems = list(detect_buildsystems(export_directory)) + resolver = auto_resolver(session) + fixers: List[BuildFixer] = [ + UnexpandedAutoconfMacroFixer(session, resolver), + GnulibDirectoryFixer(session), + MissingGoSumEntryFixer(session)] + + fixers.append(InstallFixer(resolver)) + + if session.is_temporary: + # Only muck about with temporary sessions + fixers.extend([ + GitIdentityFixer(session), + SecretGpgKeyFixer(session), + ]) + + session.chdir(reldir) + # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() @@ -54,31 +117,34 @@ def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=Fa logging.info('Using dependency resolver: %s', resolver) for buildsystem in buildsystems: - filename = buildsystem.dist( - session, resolver, fixers, target_directory, quiet=quiet - ) + filename = iterate_with_build_fixers(fixers, log_manager.wrap( + partial( + buildsystem.dist, session, resolver, target_dir, + quiet=quiet))) return filename raise NoBuildToolsFound() +# This is the function used by debianize() def create_dist( session: Session, tree: Tree, target_dir: str, include_controldir: bool = True, subdir: Optional[str] = None, - cleanup: bool = False, + log_manager: Optional[LogManager] = None, + version: Optional[str] = None, ) -> Optional[str]: - from .buildsystem import detect_buildsystems - from .buildlog import InstallFixer - from .fix_build import BuildFixer - from .fixers import ( - GitIdentityFixer, - SecretGpgKeyFixer, - UnexpandedAutoconfMacroFixer, - ) + """Create a dist tarball for a tree. + Args: + session: session to run it + tree: Tree object to work in + target_dir: Directory to write tarball into + include_controldir: Whether to include the version control directory + subdir: subdirectory in the tree to operate in + """ if subdir is None: subdir = "package" try: @@ -90,19 +156,11 @@ def create_dist( raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) raise - # TODO(jelmer): use scan_buildsystems to also look in subdirectories - buildsystems = list(detect_buildsystems(export_directory)) - resolver = auto_resolver(session) - fixers: List[BuildFixer] = [UnexpandedAutoconfMacroFixer(session, resolver)] + if log_manager is None: + log_manager = NoLogManager() - fixers.append(InstallFixer(resolver)) - - if session.is_temporary: - # Only muck about with temporary sessions - fixers.extend([GitIdentityFixer(session), SecretGpgKeyFixer(session)]) - - session.chdir(reldir) - return run_dist(session, buildsystems, resolver, fixers, target_dir) + return dist(session, export_directory, reldir, target_dir, + log_manager=log_manager, version=version) def create_dist_schroot( @@ -113,30 +171,35 @@ def create_dist_schroot( packaging_subpath: Optional[str] = None, include_controldir: bool = True, subdir: Optional[str] = None, - cleanup: bool = False, + log_manager: Optional[LogManager] = None, ) -> Optional[str]: + """Create a dist tarball for a tree. + + Args: + session: session to run it + tree: Tree object to work in + target_dir: Directory to write tarball into + include_controldir: Whether to include the version control directory + subdir: subdirectory in the tree to operate in + """ with SchrootSession(chroot) as session: if packaging_tree is not None: from .debian import satisfy_build_deps satisfy_build_deps(session, packaging_tree, packaging_subpath) return create_dist( - session, - tree, - target_dir, - include_controldir=include_controldir, - subdir=subdir, - cleanup=cleanup, - ) + session, tree, target_dir, + include_controldir=include_controldir, subdir=subdir, + log_manager=log_manager) -if __name__ == "__main__": +def main(argv=None): import argparse import breezy.bzr # noqa: F401 import breezy.git # noqa: F401 from breezy.export import export - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(argv) parser.add_argument( "--chroot", default="unstable-amd64-sbuild", @@ -157,8 +220,12 @@ if __name__ == "__main__": "--target-directory", type=str, default="..", help="Target directory" ) parser.add_argument("--verbose", action="store_true", help="Be verbose") + parser.add_argument("--mode", choices=["auto", "vcs", "buildsystem"], + type=str, + help="Mechanism to use to create buildsystem") parser.add_argument( - "--include-controldir", action="store_true", help="Clone rather than export." + "--include-controldir", action="store_true", + help="Clone rather than export." ) args = parser.parse_args() @@ -169,6 +236,10 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format="%(message)s") tree = WorkingTree.open(args.directory) + + packaging_tree: Optional[WorkingTree] + subdir: Optional[str] + if args.packaging_directory: packaging_tree = WorkingTree.open(args.packaging_directory) with packaging_tree.lock_read(): @@ -179,30 +250,47 @@ if __name__ == "__main__": packaging_tree = None subdir = None - try: - ret = create_dist_schroot( - tree, - subdir=subdir, - target_dir=os.path.abspath(args.target_directory), - packaging_tree=packaging_tree, - chroot=args.chroot, - include_controldir=args.include_controldir, - ) - except (NoBuildToolsFound, NotImplementedError): - logging.info("No build tools found, falling back to simple export.") + if args.mode == 'vcs': export(tree, "dist.tar.gz", "tgz", None) - except NotImplementedError: - logging.info( - "Build system does not support dist tarball creation, " - "falling back to simple export." - ) - export(tree, "dist.tar.gz", "tgz", None) - except UnidentifiedError as e: - logging.fatal("Unidentified error: %r", e.lines) - except DetailedFailure as e: - logging.fatal("Identified error during dist creation: %s", e.error) - except DistNoTarball: - logging.fatal("dist operation did not create a tarball") - else: - logging.info("Created %s", ret) - sys.exit(0) + elif args.mode in ('auto', 'buildsystem'): + try: + ret = create_dist_schroot( + tree, + subdir=subdir, + target_dir=os.path.abspath(args.target_directory), + packaging_tree=packaging_tree, + chroot=args.chroot, + include_controldir=args.include_controldir, + ) + except NoBuildToolsFound: + if args.mode == 'buildsystem': + logging.fatal('No build tools found, unable to create tarball') + return 1 + logging.info( + "No build tools found, falling back to simple export.") + export(tree, "dist.tar.gz", "tgz", None) + except NotImplementedError: + if args.mode == 'buildsystem': + logging.fatal('Unable to ask buildsystem for tarball') + return 1 + logging.info( + "Build system does not support dist tarball creation, " + "falling back to simple export." + ) + export(tree, "dist.tar.gz", "tgz", None) + except UnidentifiedError as e: + logging.fatal("Unidentified error: %r", e.lines) + return 1 + except DetailedFailure as e: + logging.fatal("Identified error during dist creation: %s", e.error) + return 1 + except DistNoTarball: + logging.fatal("dist operation did not create a tarball") + return 1 + else: + logging.info("Created %s", ret) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/ognibuild/dist_catcher.py b/ognibuild/dist_catcher.py index b2546a1..646d819 100644 --- a/ognibuild/dist_catcher.py +++ b/ognibuild/dist_catcher.py @@ -54,7 +54,8 @@ class DistCatcher(object): @classmethod def default(cls, directory): return cls( - [os.path.join(directory, "dist"), directory, os.path.join(directory, "..")] + [os.path.join(directory, "dist"), directory, + os.path.join(directory, "..")] ) def __enter__(self): @@ -87,19 +88,23 @@ class DistCatcher(object): continue if len(possible_new) == 1: entry = possible_new[0] - logging.info("Found new tarball %s in %s.", entry.name, directory) + logging.info( + "Found new tarball %s in %s.", entry.name, directory) self.files.append(entry.path) return entry.name elif len(possible_new) > 1: logging.warning( - "Found multiple tarballs %r in %s.", possible_new, directory + "Found multiple tarballs %r in %s.", possible_new, + directory ) self.files.extend([entry.path for entry in possible_new]) return possible_new[0].name if len(possible_updated) == 1: entry = possible_updated[0] - logging.info("Found updated tarball %s in %s.", entry.name, directory) + logging.info( + "Found updated tarball %s in %s.", entry.name, + directory) self.files.append(entry.path) return entry.name diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 4863375..527273c 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -17,7 +17,7 @@ from functools import partial import logging -from typing import List, Tuple, Callable, Any, Optional +from typing import List, Tuple, Callable, Optional, TypeVar from buildlog_consultant import Problem from buildlog_consultant.common import ( @@ -29,6 +29,14 @@ from . import DetailedFailure, UnidentifiedError from .session import Session, run_with_tee +# Number of attempts to fix a build before giving up. +DEFAULT_LIMIT = 200 + + +class FixerLimitReached(Exception): + """The maximum number of fixes has been reached.""" + + class BuildFixer(object): """Build fixer.""" @@ -44,7 +52,11 @@ class BuildFixer(object): return self._fix(problem, phase) -def run_detecting_problems(session: Session, args: List[str], check_success=None, **kwargs): +def run_detecting_problems( + session: Session, args: List[str], check_success=None, + quiet=False, **kwargs) -> List[str]: + if not quiet: + logging.info('Running %r', args) if check_success is None: def check_success(retcode, contents): return (retcode == 0) @@ -63,17 +75,26 @@ def run_detecting_problems(session: Session, args: List[str], check_success=None logging.warning("Build failed with unidentified error:") logging.warning("%s", match.line.rstrip("\n")) else: - logging.warning("Build failed and unable to find cause. Giving up.") + logging.warning( + "Build failed and unable to find cause. Giving up.") raise UnidentifiedError(retcode, args, lines, secondary=match) raise DetailedFailure(retcode, args, error) -def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): +T = TypeVar('T') + + +def iterate_with_build_fixers( + fixers: List[BuildFixer], + cb: Callable[[], T], limit=DEFAULT_LIMIT) -> T: """Call cb() until there are no more DetailedFailures we can fix. Args: fixers: List of fixers to use to resolve issues + cb: Callable to run the build + limit: Maximum number of fixing attempts before giving up """ + attempts = 0 fixed_errors = [] while True: to_resolve = [] @@ -86,9 +107,13 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): logging.info("Identified error: %r", f.error) if f.error in fixed_errors: logging.warning( - "Failed to resolve error %r, it persisted. Giving up.", f.error + "Failed to resolve error %r, it persisted. Giving up.", + f.error ) raise f + attempts += 1 + if limit is not None and limit <= attempts: + raise FixerLimitReached(limit) try: resolved = resolve_error(f.error, None, fixers=fixers) except DetailedFailure as n: @@ -100,23 +125,25 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): else: if not resolved: logging.warning( - "Failed to find resolution for error %r. Giving up.", f.error + "Failed to find resolution for error %r. Giving up.", + f.error ) raise f fixed_errors.append(f.error) def run_with_build_fixers( - session: Session, args: List[str], fixers: Optional[List[BuildFixer]], **kwargs -): + fixers: Optional[List[BuildFixer]], session: Session, args: List[str], + quiet: bool = False, **kwargs +) -> List[str]: if fixers is None: fixers = [] return iterate_with_build_fixers( - fixers, partial(run_detecting_problems, session, args, **kwargs) - ) + fixers, + partial(run_detecting_problems, session, args, quiet=quiet, **kwargs)) -def resolve_error(error, phase, fixers): +def resolve_error(error, phase, fixers) -> bool: relevant_fixers = [] for fixer in fixers: if fixer.can_fix(error): diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index c413f20..b3337db 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -21,8 +21,10 @@ from typing import Tuple from buildlog_consultant import Problem from buildlog_consultant.common import ( MissingGitIdentity, + MissingGoSumEntry, MissingSecretGpgKey, MissingAutoconfMacro, + MissingGnulibDirectory, ) from ognibuild.requirements import AutoconfMacroRequirement from ognibuild.resolver import UnsatisfiedRequirements @@ -30,6 +32,18 @@ from ognibuild.resolver import UnsatisfiedRequirements from .fix_build import BuildFixer +class GnulibDirectoryFixer(BuildFixer): + def __init__(self, session): + self.session = session + + def can_fix(self, problem: Problem): + return isinstance(problem, MissingGnulibDirectory) + + def _fix(self, problem: Problem, phase: Tuple[str, ...]): + self.session.check_call(["./gnulib.sh"]) + return True + + class GitIdentityFixer(BuildFixer): def __init__(self, session): self.session = session @@ -77,6 +91,26 @@ Passphrase: "" return False +class MissingGoSumEntryFixer(BuildFixer): + def __init__(self, session): + self.session = session + + def __repr__(self): + return "%s()" % (type(self).__name__) + + def __str__(self): + return "missing go.sum entry fixer" + + def can_fix(self, error): + return isinstance(error, MissingGoSumEntry) + + def _fix(self, error, phase): + from .fix_build import run_detecting_problems + run_detecting_problems( + self.session, ["go", "mod", "download", error.package]) + return True + + class UnexpandedAutoconfMacroFixer(BuildFixer): def __init__(self, session, resolver): self.session = session diff --git a/ognibuild/info.py b/ognibuild/info.py index c0bc425..ae349bc 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -21,11 +21,13 @@ def run_info(session, buildsystems, fixers=None): print("%r:" % buildsystem) deps = {} try: - for kind, dep in buildsystem.get_declared_dependencies(session, fixers=fixers): + for kind, dep in buildsystem.get_declared_dependencies( + session, fixers=fixers): deps.setdefault(kind, []).append(dep) except NotImplementedError: print( - "\tUnable to detect declared dependencies for this type of build system" + "\tUnable to detect declared dependencies for this type of " + "build system" ) if deps: print("\tDeclared dependencies:") @@ -35,9 +37,11 @@ def run_info(session, buildsystems, fixers=None): print("\t\t\t%s" % dep) print("") try: - outputs = list(buildsystem.get_declared_outputs(session, fixers=fixers)) + outputs = list(buildsystem.get_declared_outputs( + session, fixers=fixers)) except NotImplementedError: - print("\tUnable to detect declared outputs for this type of build system") + print("\tUnable to detect declared outputs for this type of " + "build system") outputs = [] if outputs: print("\tDeclared outputs:") diff --git a/ognibuild/install.py b/ognibuild/install.py index d242ad7..be71846 100644 --- a/ognibuild/install.py +++ b/ognibuild/install.py @@ -15,21 +15,34 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from .buildsystem import NoBuildToolsFound, InstallTarget +from functools import partial from typing import Optional +from .buildsystem import NoBuildToolsFound, InstallTarget +from .fix_build import iterate_with_build_fixers +from .logs import NoLogManager -def run_install(session, buildsystems, resolver, fixers, user: bool = False, prefix: Optional[str] = None): + +def run_install( + session, buildsystems, resolver, fixers, *, user: bool = False, + prefix: Optional[str] = None, log_manager=None): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() + if log_manager is None: + log_manager = NoLogManager() + install_target = InstallTarget() install_target.user = user install_target.prefix = prefix for buildsystem in buildsystems: - buildsystem.install(session, resolver, fixers, install_target) + iterate_with_build_fixers( + fixers, + log_manager.wrap( + partial(buildsystem.install, session, resolver, + install_target))) return raise NoBuildToolsFound() diff --git a/ognibuild/logs.py b/ognibuild/logs.py new file mode 100644 index 0000000..dd0d412 --- /dev/null +++ b/ognibuild/logs.py @@ -0,0 +1,105 @@ +#!/usr/bin/python +# Copyright (C) 2018 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from contextlib import contextmanager +import subprocess +import logging +import os +import sys + + +@contextmanager +def copy_output(output_log: str, tee: bool = False): + old_stdout = os.dup(sys.stdout.fileno()) + old_stderr = os.dup(sys.stderr.fileno()) + if tee: + p = subprocess.Popen(["tee", output_log], stdin=subprocess.PIPE) + newfd = p.stdin + else: + newfd = open(output_log, 'wb') + os.dup2(newfd.fileno(), sys.stdout.fileno()) # type: ignore + os.dup2(newfd.fileno(), sys.stderr.fileno()) # type: ignore + try: + yield + finally: + sys.stdout.flush() + sys.stderr.flush() + os.dup2(old_stdout, sys.stdout.fileno()) + os.dup2(old_stderr, sys.stderr.fileno()) + if newfd is not None: + newfd.close() + + +@contextmanager +def redirect_output(to_file): + sys.stdout.flush() + sys.stderr.flush() + old_stdout = os.dup(sys.stdout.fileno()) + old_stderr = os.dup(sys.stderr.fileno()) + os.dup2(to_file.fileno(), sys.stdout.fileno()) # type: ignore + os.dup2(to_file.fileno(), sys.stderr.fileno()) # type: ignore + try: + yield + finally: + sys.stdout.flush() + sys.stderr.flush() + os.dup2(old_stdout, sys.stdout.fileno()) + os.dup2(old_stderr, sys.stderr.fileno()) + + +def rotate_logfile(source_path: str) -> None: + if os.path.exists(source_path): + (directory_path, name) = os.path.split(source_path) + i = 1 + while os.path.exists( + os.path.join(directory_path, "%s.%d" % (name, i))): + i += 1 + target_path = os.path.join(directory_path, "%s.%d" % (name, i)) + os.rename(source_path, target_path) + logging.debug("Storing previous build log at %s", target_path) + + +class LogManager(object): + + def wrap(self, fn): + raise NotImplementedError(self.wrap) + + +class DirectoryLogManager(LogManager): + + def __init__(self, path, mode): + self.path = path + self.mode = mode + + def wrap(self, fn): + def _run(*args, **kwargs): + rotate_logfile(self.path) + if self.mode == 'copy': + with copy_output(self.path, tee=True): + return fn(*args, **kwargs) + elif self.mode == 'redirect': + with copy_output(self.path, tee=False): + return fn(*args, **kwargs) + else: + raise NotImplementedError(self.mode) + return _run + + +class NoLogManager(LogManager): + + def wrap(self, fn): + return fn diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 29475b9..31e81ec 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -26,16 +26,19 @@ from . import Requirement class PythonPackageRequirement(Requirement): + family = "python-package" + package: str - def __init__(self, package, python_version=None, specs=None, minimum_version=None): - super(PythonPackageRequirement, self).__init__("python-package") + def __init__( + self, package, python_version=None, specs=None, + minimum_version=None): self.package = package self.python_version = python_version - if minimum_version is not None: - specs = [(">=", minimum_version)] if specs is None: specs = [] + if minimum_version is not None: + specs.append((">=", minimum_version)) self.specs = specs def __repr__(self): @@ -53,11 +56,29 @@ class PythonPackageRequirement(Requirement): return "python package: %s" % (self.package,) @classmethod - def from_requirement_str(cls, text): + def from_requirement_str(cls, text, python_version=None): from requirements.requirement import Requirement req = Requirement.parse(text) - return cls(package=req.name, specs=req.specs) + return cls( + package=req.name, specs=req.specs, python_version=python_version) + + def requirement_str(self): + if self.specs: + return '%s;%s' % ( + self.package, ','.join([''.join(s) for s in self.specs])) + return self.package + + @classmethod + def _from_json(cls, js): + if isinstance(js, str): + return cls.from_requirement_str(js) + return cls.from_requirement_str(js[0], python_version=js[1]) + + def _json(self): + if self.python_version: + return [self.requirement_str(), self.python_version] + return self.requirement_str() def met(self, session): if self.python_version == "cpython3": @@ -74,7 +95,8 @@ class PythonPackageRequirement(Requirement): raise NotImplementedError text = self.package + ",".join(["".join(spec) for spec in self.specs]) p = session.Popen( - [cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text], + [cmd, "-c", + "import pkg_resources; pkg_resources.require(%r)" % text], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) @@ -82,16 +104,33 @@ class PythonPackageRequirement(Requirement): return p.returncode == 0 +Requirement.register_json(PythonPackageRequirement) + + class LatexPackageRequirement(Requirement): + family = "latex-package" + def __init__(self, package: str): self.package = package def __repr__(self): return "%s(%r)" % (type(self).__name__, self.package) + def _json(self): + return self.package + + def _from_json(cls, package): + return cls(package) + + +Requirement.register_json(LatexPackageRequirement) + class PhpPackageRequirement(Requirement): + + family = "php-package" + def __init__( self, package: str, @@ -104,6 +143,13 @@ class PhpPackageRequirement(Requirement): self.min_version = min_version self.max_version = max_version + def _json(self): + return (self.package, self.channel, self.min_version, self.max_version) + + @classmethod + def _from_json(cls, js): + return cls(*js) + def __repr__(self): return "%s(%r, %r, %r, %r)" % ( type(self).__name__, @@ -114,14 +160,24 @@ class PhpPackageRequirement(Requirement): ) +Requirement.register_json(PhpPackageRequirement) + + class BinaryRequirement(Requirement): + family = "binary" binary_name: str def __init__(self, binary_name): - super(BinaryRequirement, self).__init__("binary") self.binary_name = binary_name + def _json(self): + return self.binary_name + + @classmethod + def _from_json(cls, js): + return cls(js) + def __repr__(self): return "%s(%r)" % (type(self).__name__, self.binary_name) @@ -135,14 +191,54 @@ class BinaryRequirement(Requirement): return p.returncode == 0 +Requirement.register_json(BinaryRequirement) + + +class PHPExtensionRequirement(Requirement): + + family = "php-extension" + extension: str + + def __init__(self, extension: str): + self.extension = extension + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.extension) + + +class PytestPluginRequirement(Requirement): + + family = "pytest-plugin" + + plugin: str + + def __init__(self, plugin: str): + self.plugin = plugin + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.plugin) + + +class VcsControlDirectoryAccessRequirement(Requirement): + + vcs: List[str] + family = "vcs-access" + + def __init__(self, vcs): + self.vcs = vcs + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.vcs) + + class PerlModuleRequirement(Requirement): module: str filename: Optional[str] inc: Optional[List[str]] + family = "perl-module" def __init__(self, module, filename=None, inc=None): - super(PerlModuleRequirement, self).__init__("perl-module") self.module = module self.filename = filename self.inc = inc @@ -158,10 +254,10 @@ class PerlModuleRequirement(Requirement): class VagueDependencyRequirement(Requirement): name: str + family = "vague" minimum_version: Optional[str] = None def __init__(self, name, minimum_version=None): - super(VagueDependencyRequirement, self).__init__("vague") self.name = name self.minimum_version = minimum_version @@ -169,19 +265,26 @@ class VagueDependencyRequirement(Requirement): if " " not in self.name: yield BinaryRequirement(self.name) yield LibraryRequirement(self.name) - yield PkgConfigRequirement(self.name, minimum_version=self.minimum_version) + yield PkgConfigRequirement( + self.name, minimum_version=self.minimum_version) if self.name.lower() != self.name: yield BinaryRequirement(self.name.lower()) yield LibraryRequirement(self.name.lower()) - yield PkgConfigRequirement(self.name.lower(), minimum_version=self.minimum_version) - from .resolver.apt import AptRequirement - - yield AptRequirement.simple(self.name.lower(), minimum_version=self.minimum_version) - if self.name.lower().startswith('lib'): - devname = '%s-dev' % self.name.lower() + yield PkgConfigRequirement( + self.name.lower(), minimum_version=self.minimum_version) + try: + from .resolver.apt import AptRequirement + except ModuleNotFoundError: + pass else: - devname = 'lib%s-dev' % self.name.lower() - yield AptRequirement.simple(devname, minimum_version=self.minimum_version) + yield AptRequirement.simple( + self.name.lower(), minimum_version=self.minimum_version) + if self.name.lower().startswith('lib'): + devname = '%s-dev' % self.name.lower() + else: + devname = 'lib%s-dev' % self.name.lower() + yield AptRequirement.simple( + devname, minimum_version=self.minimum_version) def met(self, session): for x in self.expand(): @@ -192,19 +295,36 @@ class VagueDependencyRequirement(Requirement): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.name) + def __str__(self): + if self.minimum_version: + return "%s >= %s" % (self.name, self.minimum_version) + return self.name + class NodePackageRequirement(Requirement): package: str + family = "npm-package" def __init__(self, package): - super(NodePackageRequirement, self).__init__("npm-package") self.package = package def __repr__(self): return "%s(%r)" % (type(self).__name__, self.package) +class LuaModuleRequirement(Requirement): + + module: str + family = "lua-module" + + def __init__(self, module): + self.module = module + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.module) + + class PerlPreDeclaredRequirement(Requirement): name: str @@ -227,8 +347,9 @@ class PerlPreDeclaredRequirement(Requirement): 'auto_set_bugtracker': 'Module::Install::Bugtracker', } + family = "perl-predeclared" + def __init__(self, name): - super(PerlPreDeclaredRequirement, self).__init__("perl-predeclared") self.name = name def lookup_module(self): @@ -242,9 +363,9 @@ class PerlPreDeclaredRequirement(Requirement): class NodeModuleRequirement(Requirement): module: str + family = "npm-module" def __init__(self, module): - super(NodeModuleRequirement, self).__init__("npm-module") self.module = module def __repr__(self): @@ -255,41 +376,45 @@ class CargoCrateRequirement(Requirement): crate: str features: Set[str] - version: Optional[str] + api_version: Optional[str] + minimum_version: Optional[str] + family = "cargo-crate" - def __init__(self, crate, features=None, version=None): - super(CargoCrateRequirement, self).__init__("cargo-crate") + def __init__(self, crate, features=None, api_version=None, + minimum_version=None): self.crate = crate if features is None: features = set() self.features = features - self.version = version + self.api_version = api_version + self.minimum_version = minimum_version def __repr__(self): - return "%s(%r, features=%r, version=%r)" % ( + return "%s(%r, features=%r, api_version=%r, minimum_version=%r)" % ( type(self).__name__, self.crate, self.features, - self.version, + self.api_version, + self.minimum_version, ) def __str__(self): + ret = "cargo crate: %s %s" % ( + self.crate, + self.api_version or "") if self.features: - return "cargo crate: %s %s (%s)" % ( - self.crate, - self.version or "", - ", ".join(sorted(self.features)), - ) - else: - return "cargo crate: %s %s" % (self.crate, self.version or "") + ret += " (%s)" % (", ".join(sorted(self.features))) + if self.minimum_version: + ret += " (>= %s)" % self.minimum_version + return ret class PkgConfigRequirement(Requirement): module: str + family = "pkg-config" def __init__(self, module, minimum_version=None): - super(PkgConfigRequirement, self).__init__("pkg-config") self.module = module self.minimum_version = minimum_version @@ -301,9 +426,9 @@ class PkgConfigRequirement(Requirement): class PathRequirement(Requirement): path: str + family = "path" def __init__(self, path): - super(PathRequirement, self).__init__("path") self.path = path def __repr__(self): @@ -313,9 +438,9 @@ class PathRequirement(Requirement): class CHeaderRequirement(Requirement): header: str + family = "c-header" def __init__(self, header): - super(CHeaderRequirement, self).__init__("c-header") self.header = header def __repr__(self): @@ -323,16 +448,15 @@ class CHeaderRequirement(Requirement): class JavaScriptRuntimeRequirement(Requirement): - def __init__(self): - super(JavaScriptRuntimeRequirement, self).__init__("javascript-runtime") + family = "javascript-runtime" class ValaPackageRequirement(Requirement): package: str + family = "vala" def __init__(self, package: str): - super(ValaPackageRequirement, self).__init__("vala") self.package = package @@ -340,9 +464,9 @@ class RubyGemRequirement(Requirement): gem: str minimum_version: Optional[str] + family = "gem" def __init__(self, gem: str, minimum_version: Optional[str]): - super(RubyGemRequirement, self).__init__("gem") self.gem = gem self.minimum_version = minimum_version @@ -351,12 +475,16 @@ class GoPackageRequirement(Requirement): package: str version: Optional[str] + family = "go-package" def __init__(self, package: str, version: Optional[str] = None): - super(GoPackageRequirement, self).__init__("go-package") self.package = package self.version = version + def __repr__(self): + return "%s(%r, version=%r)" % ( + type(self).__name__, self.package, self.version) + def __str__(self): if self.version: return "go package: %s (= %s)" % (self.package, self.version) @@ -366,9 +494,9 @@ class GoPackageRequirement(Requirement): class GoRequirement(Requirement): version: Optional[str] + family = "go" def __init__(self, version: Optional[str] = None): - super(GoRequirement, self).__init__("go") self.version = version def __str__(self): @@ -378,18 +506,18 @@ class GoRequirement(Requirement): class DhAddonRequirement(Requirement): path: str + family = "dh-addon" def __init__(self, path: str): - super(DhAddonRequirement, self).__init__("dh-addon") self.path = path class PhpClassRequirement(Requirement): php_class: str + family = "php-class" def __init__(self, php_class: str): - super(PhpClassRequirement, self).__init__("php-class") self.php_class = php_class @@ -397,9 +525,9 @@ class RPackageRequirement(Requirement): package: str minimum_version: Optional[str] + family = "r-package" def __init__(self, package: str, minimum_version: Optional[str] = None): - super(RPackageRequirement, self).__init__("r-package") self.package = package self.minimum_version = minimum_version @@ -412,7 +540,8 @@ class RPackageRequirement(Requirement): def __str__(self): if self.minimum_version: - return "R package: %s (>= %s)" % (self.package, self.minimum_version) + return "R package: %s (>= %s)" % ( + self.package, self.minimum_version) else: return "R package: %s" % (self.package,) @@ -432,9 +561,9 @@ class OctavePackageRequirement(Requirement): package: str minimum_version: Optional[str] + family = "octave-package" def __init__(self, package: str, minimum_version: Optional[str] = None): - super(OctavePackageRequirement, self).__init__("octave-package") self.package = package self.minimum_version = minimum_version @@ -447,7 +576,8 @@ class OctavePackageRequirement(Requirement): def __str__(self): if self.minimum_version: - return "Octave package: %s (>= %s)" % (self.package, self.minimum_version) + return "Octave package: %s (>= %s)" % ( + self.package, self.minimum_version) else: return "Octave package: %s" % (self.package,) @@ -466,9 +596,9 @@ class OctavePackageRequirement(Requirement): class LibraryRequirement(Requirement): library: str + family = "lib" def __init__(self, library: str): - super(LibraryRequirement, self).__init__("lib") self.library = library @@ -476,9 +606,9 @@ class StaticLibraryRequirement(Requirement): library: str filename: str + family = "static-lib" def __init__(self, library: str, filename: str): - super(StaticLibraryRequirement, self).__init__("static-lib") self.library = library self.filename = filename @@ -486,18 +616,18 @@ class StaticLibraryRequirement(Requirement): class RubyFileRequirement(Requirement): filename: str + family = "ruby-file" def __init__(self, filename: str): - super(RubyFileRequirement, self).__init__("ruby-file") self.filename = filename class XmlEntityRequirement(Requirement): url: str + family = "xml-entity" def __init__(self, url: str): - super(XmlEntityRequirement, self).__init__("xml-entity") self.url = url @@ -505,9 +635,9 @@ class SprocketsFileRequirement(Requirement): content_type: str name: str + family = "sprockets-file" def __init__(self, content_type: str, name: str): - super(SprocketsFileRequirement, self).__init__("sprockets-file") self.content_type = content_type self.name = name @@ -515,27 +645,29 @@ class SprocketsFileRequirement(Requirement): class JavaClassRequirement(Requirement): classname: str + family = "java-class" def __init__(self, classname: str): - super(JavaClassRequirement, self).__init__("java-class") self.classname = classname class CMakefileRequirement(Requirement): filename: str + version: Optional[str] + family = "cmake-file" - def __init__(self, filename: str): - super(CMakefileRequirement, self).__init__("cmake-file") + def __init__(self, filename: str, version=None): self.filename = filename + self.version = version class HaskellPackageRequirement(Requirement): package: str + family = "haskell-package" def __init__(self, package: str, specs=None): - super(HaskellPackageRequirement, self).__init__("haskell-package") self.package = package self.specs = specs @@ -551,9 +683,9 @@ class MavenArtifactRequirement(Requirement): artifact_id: str version: Optional[str] kind: Optional[str] + family = "maven-artifact" def __init__(self, group_id, artifact_id, version=None, kind=None): - super(MavenArtifactRequirement, self).__init__("maven-artifact") self.group_id = group_id self.artifact_id = artifact_id self.version = version @@ -566,6 +698,11 @@ class MavenArtifactRequirement(Requirement): self.version, ) + def __repr__(self): + return "%s(group_id=%r, artifact_id=%r, version=%r, kind=%r)" % ( + type(self).__name__, self.group_id, self.artifact_id, + self.version, self.kind) + @classmethod def from_str(cls, text): return cls.from_tuple(text.split(":")) @@ -587,17 +724,16 @@ class MavenArtifactRequirement(Requirement): class GnomeCommonRequirement(Requirement): - def __init__(self): - super(GnomeCommonRequirement, self).__init__("gnome-common") + family = "gnome-common" class JDKFileRequirement(Requirement): jdk_path: str filename: str + family = "jdk-file" def __init__(self, jdk_path: str, filename: str): - super(JDKFileRequirement, self).__init__("jdk-file") self.jdk_path = jdk_path self.filename = filename @@ -607,55 +743,70 @@ class JDKFileRequirement(Requirement): class JDKRequirement(Requirement): - def __init__(self): - super(JDKRequirement, self).__init__("jdk") + family = "jdk" class JRERequirement(Requirement): - def __init__(self): - super(JRERequirement, self).__init__("jre") + family = "jre" + + +class QtModuleRequirement(Requirement): + family = "qt-module" + + def __init__(self, module): + self.module = module class QTRequirement(Requirement): - def __init__(self): - super(QTRequirement, self).__init__("qt") + family = "qt" class X11Requirement(Requirement): - def __init__(self): - super(X11Requirement, self).__init__("x11") + family = "x11" class CertificateAuthorityRequirement(Requirement): + family = "ca-cert" + def __init__(self, url): - super(CertificateAuthorityRequirement, self).__init__("ca-cert") self.url = url class PerlFileRequirement(Requirement): filename: str + family = "perl-file" def __init__(self, filename: str): - super(PerlFileRequirement, self).__init__("perl-file") self.filename = filename class AutoconfMacroRequirement(Requirement): + family = "autoconf-macro" macro: str def __init__(self, macro: str): - super(AutoconfMacroRequirement, self).__init__("autoconf-macro") self.macro = macro + def _json(self): + return self.macro + + @classmethod + def _from_json(cls, macro): + return cls(macro) + + +Requirement.register_json(AutoconfMacroRequirement) + class LibtoolRequirement(Requirement): - def __init__(self): - super(LibtoolRequirement, self).__init__("libtool") + family = "libtool" class IntrospectionTypelibRequirement(Requirement): + family = "introspection-type-lib" + def __init__(self, library): self.library = library @@ -665,9 +816,9 @@ class PythonModuleRequirement(Requirement): module: str python_version: Optional[str] minimum_version: Optional[str] + family = "python-module" def __init__(self, module, python_version=None, minimum_version=None): - super(PythonModuleRequirement, self).__init__("python-module") self.module = module self.python_version = python_version self.minimum_version = minimum_version @@ -702,7 +853,25 @@ class PythonModuleRequirement(Requirement): class BoostComponentRequirement(Requirement): name: str + family = "boost-component" def __init__(self, name): - super(BoostComponentRequirement, self).__init__("boost-component") self.name = name + + +class KF5ComponentRequirement(Requirement): + + name: str + family = "kf5-component" + + def __init__(self, name): + self.name = name + + +class GnulibDirectoryRequirement(Requirement): + + directory: str + family = "gnulib" + + def __init__(self, directory): + self.directory = directory diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 7af26d5..958ff2b 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -18,8 +18,11 @@ import logging import subprocess -from .. import UnidentifiedError +from typing import Optional, List, Type + +from .. import UnidentifiedError, Requirement from ..fix_build import run_detecting_problems +from ..session import Session class UnsatisfiedRequirements(Exception): @@ -28,13 +31,22 @@ class UnsatisfiedRequirements(Exception): class Resolver(object): - def install(self, requirements): + + name: str + + def __init__(self, session, user_local): + raise NotImplementedError(self.__init__) + + def install(self, requirements: List[Requirement]): raise NotImplementedError(self.install) - def resolve(self, requirement): + def resolve(self, requirement: Requirement) -> Optional[Requirement]: raise NotImplementedError(self.resolve) - def explain(self, requirements): + def resolve_all(self, requirement: Requirement) -> List[Requirement]: + raise NotImplementedError(self.resolve_all) + + def explain(self, requirements: List[Requirement]): raise NotImplementedError(self.explain) def env(self): @@ -42,13 +54,15 @@ class Resolver(object): class CPANResolver(Resolver): + name = "cpan" + def __init__(self, session, user_local=False, skip_tests=True): self.session = session self.user_local = user_local self.skip_tests = skip_tests def __str__(self): - return "cpan" + return self.name def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) @@ -109,7 +123,8 @@ class TlmgrResolver(Resolver): self.repository = repository def __str__(self): - if self.repository.startswith('http://') or self.repository.startswith('https://'): + if (self.repository.startswith('http://') + or self.repository.startswith('https://')): return 'tlmgr(%r)' % self.repository else: return self.repository @@ -154,7 +169,8 @@ class TlmgrResolver(Resolver): try: run_detecting_problems(self.session, cmd, user=user) except UnidentifiedError as e: - if "tlmgr: user mode not initialized, please read the documentation!" in e.lines: + if ("tlmgr: user mode not initialized, " + "please read the documentation!") in e.lines: self.session.check_call(['tlmgr', 'init-usertree']) else: raise @@ -163,6 +179,7 @@ class TlmgrResolver(Resolver): class CTANResolver(TlmgrResolver): + name = "ctan" def __init__(self, session, user_local=False): super(CTANResolver, self).__init__( @@ -170,13 +187,16 @@ class CTANResolver(TlmgrResolver): class RResolver(Resolver): + + name: str + def __init__(self, session, repos, user_local=False): self.session = session self.repos = repos self.user_local = user_local def __str__(self): - return "cran" + return self.name def __repr__(self): return "%s(%r, %r)" % (type(self).__name__, self.session, self.repos) @@ -221,12 +241,14 @@ class RResolver(Resolver): class OctaveForgeResolver(Resolver): + name = "octave-forge" + def __init__(self, session, user_local=False): self.session = session self.user_local = user_local def __str__(self): - return "octave-forge" + return self.name def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) @@ -267,6 +289,8 @@ class OctaveForgeResolver(Resolver): class CRANResolver(RResolver): + name = "cran" + def __init__(self, session, user_local=False): super(CRANResolver, self).__init__( session, "http://cran.r-project.org", user_local=user_local @@ -274,19 +298,25 @@ class CRANResolver(RResolver): class BioconductorResolver(RResolver): + name = "bioconductor" + def __init__(self, session, user_local=False): super(BioconductorResolver, self).__init__( - session, "https://hedgehog.fhcrc.org/bioconductor", user_local=user_local + session, "https://hedgehog.fhcrc.org/bioconductor", + user_local=user_local ) class HackageResolver(Resolver): + + name = "hackage" + def __init__(self, session, user_local=False): self.session = session self.user_local = user_local def __str__(self): - return "hackage" + return self.name def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) @@ -295,7 +325,8 @@ class HackageResolver(Resolver): extra_args = [] if self.user_local: extra_args.append("--user") - return ["cabal", "install"] + extra_args + [req.package for req in reqs] + return (["cabal", "install"] + extra_args + + [req.package for req in reqs]) def install(self, requirements): from ..requirements import HaskellPackageRequirement @@ -329,12 +360,15 @@ class HackageResolver(Resolver): class PypiResolver(Resolver): + + name = "pypi" + def __init__(self, session, user_local=False): self.session = session self.user_local = user_local def __str__(self): - return "pypi" + return self.name def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) @@ -380,12 +414,15 @@ class PypiResolver(Resolver): class GoResolver(Resolver): + + name = "go" + def __init__(self, session, user_local): self.session = session self.user_local = user_local def __str__(self): - return "go" + return self.name def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) @@ -426,17 +463,26 @@ NPM_COMMAND_PACKAGES = { "del-cli": "del-cli", "husky": "husky", "cross-env": "cross-env", + "xo": "xo", + "standard": "standard", + "jshint": "jshint", + "if-node-version": "if-node-version", + "babel-cli": "babel", + "c8": "c8", + "prettier-standard": "prettier-standard", } class NpmResolver(Resolver): + name = "npm" + def __init__(self, session, user_local=False): self.session = session self.user_local = user_local # TODO(jelmer): Handle user_local def __str__(self): - return "npm" + return self.name def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) @@ -472,7 +518,10 @@ class NpmResolver(Resolver): if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - cmd = ["npm", "-g", "install", requirement.package] + cmd = ["npm", "install"] + if not self.user_local: + cmd.append('-g') + cmd.append(requirement.package) logging.info("npm: running %r", cmd) run_detecting_problems(self.session, cmd, user=user) if missing: @@ -529,7 +578,7 @@ class StackedResolver(Resolver): raise UnsatisfiedRequirements(requirements) -NATIVE_RESOLVER_CLS = [ +NATIVE_RESOLVER_CLS: List[Type[Resolver]] = [ CPANResolver, CTANResolver, PypiResolver, @@ -543,24 +592,70 @@ NATIVE_RESOLVER_CLS = [ def native_resolvers(session, user_local): - return StackedResolver([kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) + return StackedResolver( + [kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) -def auto_resolver(session, explain=False): +def select_resolvers(session, user_local, resolvers, + dep_server_url=None) -> Optional[Resolver]: + selected = [] + for resolver in resolvers: + for kls in NATIVE_RESOLVER_CLS: + if kls.name == resolver: + selected.append(kls(session, user_local)) + break + else: + if resolver == 'native': + selected.extend([ + kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) + elif resolver == 'apt': + if user_local: + raise NotImplementedError( + 'user local not supported for apt') + if dep_server_url: + from .dep_server import DepServerAptResolver + selected.append(DepServerAptResolver.from_session( + session, dep_server_url)) + else: + from .apt import AptResolver + selected.append(AptResolver.from_session(session)) + else: + raise KeyError(resolver) + if len(selected) == 0: + return None + if len(selected) == 1: + return selected[0] + return StackedResolver(selected) + + +def auto_resolver(session: Session, explain: bool = False, + system_wide: Optional[bool] = None, + dep_server_url: Optional[str] = None): # if session is SchrootSession or if we're root, use apt - from .apt import AptResolver from ..session.schroot import SchrootSession from ..session import get_user user = get_user(session) resolvers = [] - # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if - # present? - if isinstance(session, SchrootSession) or user == "root" or explain: - user_local = False - else: - user_local = True - if not user_local: - resolvers.append(AptResolver.from_session(session)) - resolvers.extend([kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) + if system_wide is None: + # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if + # present? + if isinstance(session, SchrootSession) or user == "root" or explain: + system_wide = True + else: + system_wide = False + if system_wide: + try: + from .apt import AptResolver + except ModuleNotFoundError: + pass + else: + if dep_server_url: + from .dep_server import DepServerAptResolver + resolvers.append( + DepServerAptResolver.from_session(session, dep_server_url)) + else: + resolvers.append(AptResolver.from_session(session)) + resolvers.extend([kls(session, not system_wide) + for kls in NATIVE_RESOLVER_CLS]) return StackedResolver(resolvers) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 82ef627..ae88a49 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -15,12 +15,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import asyncio from itertools import chain import logging import os import posixpath import re -from typing import Optional, List +from typing import Optional, List, Tuple, Callable, Type, Awaitable from debian.changelog import Version from debian.deb822 import PkgRelation @@ -28,6 +29,7 @@ from debian.deb822 import PkgRelation from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements +from .. import OneOfRequirement from ..requirements import ( Requirement, CargoCrateRequirement, @@ -48,9 +50,11 @@ from ..requirements import ( NodePackageRequirement, LibraryRequirement, BoostComponentRequirement, + KF5ComponentRequirement, StaticLibraryRequirement, RubyFileRequirement, XmlEntityRequirement, + OctavePackageRequirement, SprocketsFileRequirement, JavaClassRequirement, CMakefileRequirement, @@ -61,6 +65,7 @@ from ..requirements import ( JDKRequirement, JRERequirement, QTRequirement, + QtModuleRequirement, X11Requirement, PerlModuleRequirement, PerlFileRequirement, @@ -72,12 +77,17 @@ from ..requirements import ( VagueDependencyRequirement, PerlPreDeclaredRequirement, IntrospectionTypelibRequirement, + PHPExtensionRequirement, + VcsControlDirectoryAccessRequirement, ) class AptRequirement(Requirement): + + family = "apt" + def __init__(self, relations): - super(AptRequirement, self).__init__("apt") + super(AptRequirement, self).__init__() if not isinstance(relations, list): raise TypeError(relations) self.relations = relations @@ -96,17 +106,26 @@ class AptRequirement(Requirement): def pkg_relation_str(self): return PkgRelation.str(self.relations) + def _json(self): + return self.pkg_relation_str() + + @classmethod + def _from_json(cls, text): + return cls.from_str(text) + def __hash__(self): return hash((type(self), self.pkg_relation_str())) def __eq__(self, other): - return isinstance(self, type(other)) and self.relations == other.relations + return (isinstance(self, type(other)) + and self.relations == other.relations) def __str__(self): return "apt requirement: %s" % self.pkg_relation_str() def __repr__(self): - return "%s.from_str(%r)" % (type(self).__name__, self.pkg_relation_str()) + return "%s.from_str(%r)" % ( + type(self).__name__, self.pkg_relation_str()) def package_names(self): for rel in self.relations: @@ -133,32 +152,34 @@ class AptRequirement(Requirement): for rel in self.relations: for entry in rel: - if any(binary_pkg_matches(entry, binary) for binary in binaries): + if any(binary_pkg_matches(entry, binary) + for binary in binaries): break else: return False return True -def resolve_perl_predeclared_req(apt_mgr, req): +async def resolve_perl_predeclared_req(apt_mgr, req): try: req = req.lookup_module() except KeyError: logging.warning( 'Unable to map predeclared function %s to a perl module', req.name) return None - return resolve_perl_module_req(apt_mgr, req) + return await resolve_perl_module_req(apt_mgr, req) -def find_package_names( - apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False +async def find_package_names( + apt_mgr: AptManager, paths: List[str], regex: bool = False, + case_insensitive: bool = False ) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) - return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) + return await apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) -def find_reqs_simple( +async def find_reqs_simple( apt_mgr: AptManager, paths: List[str], regex: bool = False, @@ -169,7 +190,8 @@ def find_reqs_simple( raise TypeError(paths) return [ AptRequirement.simple(package, minimum_version=minimum_version) - for package in find_package_names(apt_mgr, paths, regex, case_insensitive) + for package in await find_package_names( + apt_mgr, paths, regex, case_insensitive) ] @@ -190,8 +212,10 @@ def python_spec_to_apt_rels(pkg_name, specs): next_maj_deb_version = Version(".".join(parts)) deb_version = Version(spec[1]) rels.extend( - [[{"name": pkg_name, "version": (">=", deb_version)}], - [{"name": pkg_name, "version": ("<<", next_maj_deb_version)}]]) + [[{"name": pkg_name, + "version": (">=", deb_version)}], + [{"name": pkg_name, + "version": ("<<", next_maj_deb_version)}]]) elif spec[0] == "!=": deb_version = Version(spec[1]) rels.extend([ @@ -203,28 +227,36 @@ def python_spec_to_apt_rels(pkg_name, specs): n = list(s) n[-1] = str(int(n[-1]) + 1) rels.extend( - [[{"name": pkg_name, "version": (">=", Version(".".join(s)))}], - [{"name": pkg_name, "version": ("<<", Version(".".join(n)))}]]) + [[{"name": pkg_name, + "version": (">=", Version(".".join(s)))}], + [{"name": pkg_name, + "version": ("<<", Version(".".join(n)))}]]) else: - c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] + c = { + ">=": ">=", + "<=": "<=", + "<": "<<", + ">": ">>", + "==": "=", + }[spec[0]] deb_version = Version(spec[1]) rels.append([{"name": pkg_name, "version": (c, deb_version)}]) return rels -def get_package_for_python_package( +async def get_package_for_python_package( apt_mgr, package, python_version: Optional[str], specs=None ): - pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape( - package.replace("-", "_") - ) + pypy_regex = ( + "/usr/lib/pypy/dist\\-packages/%s-.*\\.(dist|egg)\\-info" + % re.escape(package.replace("-", "_"))) cpython2_regex = ( - "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" + "/usr/lib/python2\\.[0-9]/dist\\-packages/%s-.*\\.(dist|egg)\\-info" % re.escape(package.replace("-", "_")) ) - cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape( - package.replace("-", "_") - ) + cpython3_regex = ( + "/usr/lib/python3/dist\\-packages/%s-.*\\.(dist|egg)\\-info" + % re.escape(package.replace("-", "_"))) if python_version == "pypy": paths = [pypy_regex] elif python_version == "cpython2": @@ -234,109 +266,174 @@ def get_package_for_python_package( elif python_version is None: paths = [cpython3_regex, cpython2_regex, pypy_regex] else: - raise NotImplementedError("unsupported python version %s" % python_version) - names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True) - return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] + raise NotImplementedError( + "unsupported python version %s" % python_version) + names = await find_package_names( + apt_mgr, paths, regex=True, case_insensitive=True) + return [AptRequirement( + python_spec_to_apt_rels(name, specs)) for name in names] -def get_package_for_python_module(apt_mgr, module, python_version, specs): - cpython3_regexes = [ - posixpath.join( - "/usr/lib/python3/dist-packages", - re.escape(module.replace(".", "/")), - "__init__.py", - ), - posixpath.join( - "/usr/lib/python3/dist-packages", - re.escape(module.replace(".", "/")) + ".py", - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/lib-dynload", - re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/", - re.escape(module.replace(".", "/")), - "__init__.py", - ), - ] - cpython2_regexes = [ - posixpath.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - re.escape(module.replace(".", "/")), - "__init__.py", - ), - posixpath.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - re.escape(module.replace(".", "/")) + ".py", - ), - posixpath.join( - "/usr/lib/python2.\\.[0-9]/lib-dynload", - re.escape(module.replace(".", "/")) + ".so", - ), - ] - pypy_regexes = [ - posixpath.join( - "/usr/lib/pypy/dist-packages", - re.escape(module.replace(".", "/")), - "__init__.py", - ), - posixpath.join( - "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" - ), - posixpath.join( - "/usr/lib/pypy/dist-packages", - re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", - ), - ] +def get_possible_python3_paths_for_python_object(object_path): + cpython3_regexes = [] + while True: + cpython3_regexes.extend([ + posixpath.join( + re.escape("/usr/lib/python3/dist-packages"), + re.escape(object_path.replace(".", "/")), + re.escape("__init__.py"), + ), + posixpath.join( + re.escape("/usr/lib/python3/dist-packages"), + re.escape(object_path.replace(".", "/")) + re.escape(".py"), + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/lib\\-dynload", + re.escape(object_path.replace(".", "/")) + + ".cpython\\-.*\\.so", + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", + re.escape(object_path.replace(".", "/")) + "\\.py" + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", + re.escape(object_path.replace(".", "/")), + "__init__\\.py", + ), + ]) + try: + object_path, discarded = object_path.rsplit('.', 1) + except ValueError: + break + return cpython3_regexes + + +def get_possible_pypy_paths_for_python_object(object_path): + pypy_regexes = [] + while True: + pypy_regexes.extend([ + posixpath.join( + "/usr/lib/pypy/dist\\-packages", + re.escape(object_path.replace(".", "/")), + "__init__\\.py", + ), + posixpath.join( + "/usr/lib/pypy/dist\\-packages", + re.escape(object_path.replace(".", "/")) + "\\.py" + ), + posixpath.join( + "/usr/lib/pypy/dist\\-packages", + re.escape(object_path.replace(".", "/")) + "\\.pypy-.*\\.so", + ), + ]) + try: + object_path, discarded = object_path.rsplit('.', 1) + except ValueError: + break + return pypy_regexes + + +def get_possible_python2_paths_for_python_object(object_path): + cpython2_regexes = [] + while True: + cpython2_regexes.extend([ + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist\\-packages", + re.escape(object_path.replace(".", "/")), + "__init__\\.py", + ), + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist\\-packages", + re.escape(object_path.replace(".", "/")) + "\\.py", + ), + posixpath.join( + "/usr/lib/python2.\\.[0-9]/lib\\-dynload", + re.escape(object_path.replace(".", "/")) + "\\.so", + ), + ]) + try: + object_path, discarded = object_path.rsplit('.', 1) + except ValueError: + break + return cpython2_regexes + + +async def get_package_for_python_object_path( + apt_mgr, object_path, python_version, specs): + # Try to find the most specific file if python_version == "cpython3": - paths = cpython3_regexes + paths = get_possible_python3_paths_for_python_object(object_path) elif python_version == "cpython2": - paths = cpython2_regexes + paths = get_possible_python2_paths_for_python_object(object_path) elif python_version == "pypy": - paths = pypy_regexes + paths = get_possible_pypy_paths_for_python_object(object_path) elif python_version is None: - paths = cpython3_regexes + cpython2_regexes + pypy_regexes + paths = (get_possible_python3_paths_for_python_object(object_path) + + get_possible_python2_paths_for_python_object(object_path) + + get_possible_pypy_paths_for_python_object(object_path)) else: raise AssertionError("unknown python version %r" % python_version) - names = find_package_names(apt_mgr, paths, regex=True) - return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] + names = await find_package_names(apt_mgr, paths, regex=True) + return [AptRequirement(python_spec_to_apt_rels(name, specs)) + for name in names] vague_map = { "the Gnu Scientific Library": "libgsl-dev", "the required FreeType library": "libfreetype-dev", "the Boost C++ libraries": "libboost-dev", + "the sndfile library": "libsndfile-dev", # TODO(jelmer): Support resolving virtual packages "PythonLibs": "libpython3-dev", + "PythonInterp": "python3", "ZLIB": "libz3-dev", "Osmium": "libosmium2-dev", "glib": "libglib2.0-dev", + "OpenGL": "libgl-dev", - # TODO(jelmer): For Python, check minimum_version and map to python 2 or python 3 + # TODO(jelmer): For Python, check minimum_version and map to python 2 or + # python 3 "Python": "libpython3-dev", "Lua": "liblua5.4-dev", } -def resolve_vague_dep_req(apt_mgr, req): +async def resolve_vague_dep_req(apt_mgr, req): name = req.name options = [] + if ' or ' in name: + for entry in name.split(' or '): + options.extend(await resolve_vague_dep_req( + apt_mgr, VagueDependencyRequirement(entry))) + if name in vague_map: - options.append(AptRequirement.simple(vague_map[name], minimum_version=req.minimum_version)) + options.append(AptRequirement.simple( + vague_map[name], minimum_version=req.minimum_version)) for x in req.expand(): - options.extend(resolve_requirement_apt(apt_mgr, x)) + options.extend(await resolve_requirement_apt(apt_mgr, x)) + + if name.startswith('GNU '): + options.extend(await resolve_vague_dep_req( + apt_mgr, VagueDependencyRequirement(name[4:]))) + + if name.startswith('py') or name.endswith('py'): + # TODO(jelmer): Try harder to determine whether this is a python + # package + options.append(await resolve_requirement_apt( + apt_mgr, PythonPackageRequirement(name))) + # Try even harder if not options: - options.extend(find_reqs_simple( + options.extend(await find_reqs_simple( apt_mgr, [ - posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.name) + "-.*\\.pc"), - posixpath.join("/usr/lib/pkgconfig", re.escape(req.name) + "-.*\\.pc") + posixpath.join( + "/usr/lib", ".*", "pkgconfig", + re.escape(req.name) + "-.*\\.pc"), + posixpath.join( + "/usr/lib/pkgconfig", re.escape(req.name) + "\\-.*\\.pc") ], regex=True, case_insensitive=True, @@ -346,18 +443,29 @@ def resolve_vague_dep_req(apt_mgr, req): return options -def resolve_binary_req(apt_mgr, req): +async def resolve_php_extension_req(apt_mgr, req): + return [AptRequirement.simple("php-%s" % req.extension)] + + +async def resolve_octave_pkg_req(apt_mgr, req): + return [AptRequirement.simple( + "octave-%s" % req.package, minimum_version=req.minimum_version)] + + +async def resolve_binary_req(apt_mgr, req): if posixpath.isabs(req.binary_name): paths = [req.binary_name] else: paths = [ - posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] + posixpath.join(dirname, req.binary_name) + for dirname in ["/usr/bin", "/bin"] ] - return find_reqs_simple(apt_mgr, paths) + # TODO(jelmer): Check for binaries which use alternatives + return await find_reqs_simple(apt_mgr, paths) -def resolve_pkg_config_req(apt_mgr, req): - names = find_package_names( +async def resolve_pkg_config_req(apt_mgr, req): + names = await find_package_names( apt_mgr, [ posixpath.join( @@ -367,7 +475,7 @@ def resolve_pkg_config_req(apt_mgr, req): regex=True, ) if not names: - names = find_package_names( + names = await find_package_names( apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")] ) return [ @@ -376,16 +484,16 @@ def resolve_pkg_config_req(apt_mgr, req): ] -def resolve_path_req(apt_mgr, req): - return find_reqs_simple(apt_mgr, [req.path]) +async def resolve_path_req(apt_mgr, req): + return await find_reqs_simple(apt_mgr, [req.path]) -def resolve_c_header_req(apt_mgr, req): - reqs = find_reqs_simple( +async def resolve_c_header_req(apt_mgr, req): + reqs = await find_reqs_simple( apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False ) if not reqs: - reqs = find_reqs_simple( + reqs = await find_reqs_simple( apt_mgr, [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True, @@ -393,140 +501,149 @@ def resolve_c_header_req(apt_mgr, req): return reqs -def resolve_js_runtime_req(apt_mgr, req): - return find_reqs_simple(apt_mgr, ["/usr/bin/node", "/usr/bin/duk"]) +async def resolve_js_runtime_req(apt_mgr, req): + return await find_reqs_simple(apt_mgr, ["/usr/bin/node", "/usr/bin/duk"]) -def resolve_vala_package_req(apt_mgr, req): +async def resolve_vala_package_req(apt_mgr, req): path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package) - return find_reqs_simple(apt_mgr, [path], regex=True) + return await find_reqs_simple(apt_mgr, [path], regex=True) -def resolve_ruby_gem_req(apt_mgr, req): +async def resolve_ruby_gem_req(apt_mgr, req): paths = [ posixpath.join( "/usr/share/rubygems-integration/all/" "specifications/%s-.*\\.gemspec" % re.escape(req.gem) ) ] - return find_reqs_simple( + return await find_reqs_simple( apt_mgr, paths, regex=True, minimum_version=req.minimum_version ) -def resolve_go_package_req(apt_mgr, req): - return find_reqs_simple( +async def resolve_go_package_req(apt_mgr, req): + return await find_reqs_simple( apt_mgr, - [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], + [posixpath.join( + "/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True, ) -def resolve_go_req(apt_mgr, req): - return [AptRequirement.simple("golang-go", minimum_version="2:%s" % req.version)] - - -def resolve_dh_addon_req(apt_mgr, req): - paths = [posixpath.join("/usr/share/perl5", req.path)] - return find_reqs_simple(apt_mgr, paths) - - -def resolve_php_class_req(apt_mgr, req): - path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") - return find_reqs_simple(apt_mgr, [path]) - - -def resolve_php_package_req(apt_mgr, req): +async def resolve_go_req(apt_mgr, req): return [ - AptRequirement.simple("php-%s" % req.package, minimum_version=req.min_version) + AptRequirement.simple( + "golang-go", minimum_version="2:%s~" % req.version)] + + +async def resolve_dh_addon_req(apt_mgr, req): + paths = [posixpath.join("/usr/share/perl5", req.path)] + return await find_reqs_simple(apt_mgr, paths) + + +async def resolve_php_class_req(apt_mgr, req): + path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") + return await find_reqs_simple(apt_mgr, [path]) + + +async def resolve_php_package_req(apt_mgr, req): + return [ + AptRequirement.simple( + "php-%s" % req.package, minimum_version=req.min_version) ] -def resolve_r_package_req(apt_mgr, req): +async def resolve_r_package_req(apt_mgr, req): paths = [ posixpath.join("/usr/lib/R/site-library", req.package, "DESCRIPTION") ] - return find_reqs_simple(apt_mgr, paths, minimum_version=req.minimum_version) + return await find_reqs_simple( + apt_mgr, paths, minimum_version=req.minimum_version) -def resolve_node_module_req(apt_mgr, req): +async def resolve_node_module_req(apt_mgr, req): paths = [ - "/usr/share/nodejs/.*/node_modules/%s/index.js" % re.escape(req.module), - "/usr/lib/nodejs/%s/index.js" % re.escape(req.module), - "/usr/share/nodejs/%s/index.js" % re.escape(req.module), + "/usr/share/nodejs/.*/node_modules/%s/index\\.js" + % re.escape(req.module), + "/usr/lib/nodejs/%s/index\\.js" % re.escape(req.module), + "/usr/share/nodejs/%s/index\\.js" % re.escape(req.module), ] - return find_reqs_simple(apt_mgr, paths, regex=True) + return await find_reqs_simple(apt_mgr, paths, regex=True) -def resolve_node_package_req(apt_mgr, req): +async def resolve_node_package_req(apt_mgr, req): paths = [ - "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % re.escape(req.package), + "/usr/share/nodejs/.*/node_modules/%s/package\\.json" + % re.escape(req.package), "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), ] - return find_reqs_simple(apt_mgr, paths, regex=True) + return await find_reqs_simple(apt_mgr, paths, regex=True) -def resolve_library_req(apt_mgr, req): +async def resolve_library_req(apt_mgr, req): paths = [ - posixpath.join("/usr/lib/lib%s.so$" % re.escape(req.library)), - posixpath.join("/usr/lib/.*/lib%s.so$" % re.escape(req.library)), - posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), - posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), + posixpath.join("/usr/lib/lib%s\\.so$" % re.escape(req.library)), + posixpath.join("/usr/lib/.*/lib%s\\.so$" % re.escape(req.library)), + posixpath.join("/usr/lib/lib%s\\.a$" % re.escape(req.library)), + posixpath.join("/usr/lib/.*/lib%s\\.a$" % re.escape(req.library)), ] - return find_reqs_simple(apt_mgr, paths, regex=True) + return await find_reqs_simple(apt_mgr, paths, regex=True) -def resolve_static_library_req(apt_mgr, req): +async def resolve_static_library_req(apt_mgr, req): paths = [ posixpath.join("/usr/lib/%s$" % re.escape(req.filename)), posixpath.join("/usr/lib/.*/%s$" % re.escape(req.filename)), ] - return find_reqs_simple(apt_mgr, paths, regex=True) + return await find_reqs_simple(apt_mgr, paths, regex=True) -def resolve_ruby_file_req(apt_mgr, req): +async def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] - reqs = find_reqs_simple(apt_mgr, paths, regex=False) + reqs = await find_reqs_simple(apt_mgr, paths, regex=False) if reqs: return reqs paths = [ posixpath.join( - r"/usr/share/rubygems-integration/all/gems/([^/]+)/" + r"/usr/share/rubygems\-integration/all/gems/([^/]+)/" "lib/%s\\.rb" % re.escape(req.filename) ) ] - return find_reqs_simple(apt_mgr, paths, regex=True) + return await find_reqs_simple(apt_mgr, paths, regex=True) -def resolve_xml_entity_req(apt_mgr, req): +async def resolve_xml_entity_req(apt_mgr, req): # Ideally we should be using the XML catalog for this, but hardcoding # a few URLs will do for now.. URL_MAP = { - "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/" + "http://www.oasis-open.org/docbook/xml/": + "/usr/share/xml/docbook/schema/dtd/" } for url, path in URL_MAP.items(): if req.url.startswith(url): - search_path = posixpath.join(path, req.url[len(url) :]) + search_path = posixpath.join(path, req.url[len(url):]) break else: return None - return find_reqs_simple(apt_mgr, [search_path], regex=False) + return await find_reqs_simple(apt_mgr, [search_path], regex=False) -def resolve_sprockets_file_req(apt_mgr, req): +async def resolve_sprockets_file_req(apt_mgr, req): if req.content_type == "application/javascript": - path = "/usr/share/.*/app/assets/javascripts/%s\\.js$" % re.escape(req.name) + path = ("/usr/share/.*/app/assets/javascripts/%s\\.js$" + % re.escape(req.name)) else: - logging.warning("unable to handle content type %s", req.content_type) + logging.warning( + "unable to handle content type %s", req.content_type) return None - return find_reqs_simple(apt_mgr, [path], regex=True) + return await find_reqs_simple(apt_mgr, [path], regex=True) -def resolve_java_class_req(apt_mgr, req): - # Unfortunately this only finds classes in jars installed on the host - # system :( +async def resolve_java_class_req(apt_mgr, req): + apt_mgr.satisfy(["java-propose-classpath"]) output = apt_mgr.session.check_output( ["java-propose-classpath", "-c" + req.classname] ) @@ -535,20 +652,22 @@ def resolve_java_class_req(apt_mgr, req): logging.warning("unable to find classpath for %s", req.classname) return False logging.info("Classpath for %s: %r", req.classname, classpath) - return find_reqs_simple(apt_mgr, [classpath]) + return await find_reqs_simple(apt_mgr, [classpath]) -def resolve_cmake_file_req(apt_mgr, req): - paths = ['/usr/lib/.*/cmake/.*/%s' % re.escape(req.filename)] - return find_reqs_simple(apt_mgr, paths, regex=True) +async def resolve_cmake_file_req(apt_mgr, req): + paths = ['/usr/lib/.*/cmake/.*/%s' % re.escape(req.filename), + '/usr/share/.*/cmake/%s' % re.escape(req.filename)] + return await find_reqs_simple(apt_mgr, paths, regex=True) -def resolve_haskell_package_req(apt_mgr, req): - path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.package) - return find_reqs_simple(apt_mgr, [path], regex=True) +async def resolve_haskell_package_req(apt_mgr, req): + path = ("/var/lib/ghc/package\\.conf\\.d/%s\\-.*\\.conf" + % re.escape(req.package)) + return await find_reqs_simple(apt_mgr, [path], regex=True) -def resolve_maven_artifact_req(apt_mgr, req): +async def resolve_maven_artifact_req(apt_mgr, req): if req.version is None: version = ".*" regex = True @@ -566,50 +685,66 @@ def resolve_maven_artifact_req(apt_mgr, req): escape(req.group_id.replace(".", "/")), escape(req.artifact_id), version, - escape("%s-") + version + escape("." + kind), + escape("%s-" % req.artifact_id) + version + escape("." + kind), ) - return find_reqs_simple(apt_mgr, [path], regex=regex) + return await find_reqs_simple(apt_mgr, [path], regex=regex) -def resolve_gnome_common_req(apt_mgr, req): +async def resolve_gnome_common_req(apt_mgr, req): return [AptRequirement.simple("gnome-common")] -def resolve_jdk_file_req(apt_mgr, req): +async def resolve_jdk_file_req(apt_mgr, req): path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename) - return find_reqs_simple(apt_mgr, [path], regex=True) + return await find_reqs_simple(apt_mgr, [path], regex=True) -def resolve_jdk_req(apt_mgr, req): +async def resolve_jdk_req(apt_mgr, req): return [AptRequirement.simple("default-jdk")] -def resolve_jre_req(apt_mgr, req): +async def resolve_jre_req(apt_mgr, req): return [AptRequirement.simple("default-jre")] -def resolve_x11_req(apt_mgr, req): +async def resolve_x11_req(apt_mgr, req): return [AptRequirement.simple("libx11-dev")] -def resolve_qt_req(apt_mgr, req): - return find_reqs_simple(apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) +async def resolve_qt_req(apt_mgr, req): + return await find_reqs_simple( + apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) -def resolve_libtool_req(apt_mgr, req): +async def resolve_qt_module_req(apt_mgr, req): + return await find_reqs_simple( + apt_mgr, + ["/usr/lib/.*/qt5/mkspecs/modules/qt_lib_%s\\.pri" + % re.escape(req.module)], + regex=True) + + +async def resolve_libtool_req(apt_mgr, req): return [AptRequirement.simple("libtool")] -def resolve_perl_module_req(apt_mgr, req): - DEFAULT_PERL_PATHS = ["/usr/share/perl5", "/usr/lib/.*/perl5/.*", "/usr/lib/.*/perl-base"] +async def resolve_perl_module_req(apt_mgr, req): + DEFAULT_PERL_PATHS = [ + "/usr/share/perl5", "/usr/lib/.*/perl5/.*", "/usr/lib/.*/perl-base", + "/usr/lib/.*/perl/[^/]+", "/usr/share/perl/[^/]+"] if req.inc is None: if req.filename is None: - paths = [posixpath.join(inc, re.escape(req.module.replace('::', '/') + '.pm')) for inc in DEFAULT_PERL_PATHS] + paths = [ + posixpath.join( + inc, re.escape(req.module.replace('::', '/') + '.pm')) + for inc in DEFAULT_PERL_PATHS] regex = True elif not posixpath.isabs(req.filename): - paths = [posixpath.join(inc, re.escape(req.filename)) for inc in DEFAULT_PERL_PATHS] + paths = [ + posixpath.join(inc, re.escape(req.filename)) + for inc in DEFAULT_PERL_PATHS] regex = True else: paths = [req.filename] @@ -617,79 +752,92 @@ def resolve_perl_module_req(apt_mgr, req): else: regex = False paths = [posixpath.join(inc, req.filename) for inc in req.inc] - return find_reqs_simple(apt_mgr, paths, regex=regex) + return await find_reqs_simple(apt_mgr, paths, regex=regex) -def resolve_perl_file_req(apt_mgr, req): - return find_reqs_simple(apt_mgr, [req.filename], regex=False) +async def resolve_perl_file_req(apt_mgr, req): + return await find_reqs_simple(apt_mgr, [req.filename], regex=False) -def _find_aclocal_fun(macro): - # TODO(jelmer): Use the API for codesearch.debian.net instead? - defun_prefix = b"AC_DEFUN([%s]," % macro.encode("ascii") - au_alias_prefix = b"AU_ALIAS([%s]," % macro.encode("ascii") - prefixes = [defun_prefix, au_alias_prefix] +def _m4_macro_regex(macro): + defun_prefix = re.escape("AC_DEFUN([%s]," % macro) + au_alias_prefix = re.escape("AU_ALIAS([%s]," % macro) + m4_copy = r"m4_copy\(.*,\s*\[%s\]\)" % re.escape(macro) + return "(" + "|".join([defun_prefix, au_alias_prefix, m4_copy]) + ")" + + +def _find_local_m4_macro(macro): + # TODO(jelmer): Query some external service that can search all binary + # packages? + p = re.compile(_m4_macro_regex(macro).encode('ascii')) for entry in os.scandir("/usr/share/aclocal"): if not entry.is_file(): continue with open(entry.path, "rb") as f: for line in f: - if any([line.startswith(prefix) for prefix in prefixes]): + if any(p.finditer(line)): return entry.path raise KeyError -def resolve_autoconf_macro_req(apt_mgr, req): +async def resolve_autoconf_macro_req(apt_mgr, req): try: - path = _find_aclocal_fun(req.macro) + path = _find_local_m4_macro(req.macro) except KeyError: logging.info("No local m4 file found defining %s", req.macro) return None - return find_reqs_simple(apt_mgr, [path]) + return await find_reqs_simple(apt_mgr, [path]) -def resolve_python_module_req(apt_mgr, req): +async def resolve_python_module_req(apt_mgr, req): if req.minimum_version: specs = [(">=", req.minimum_version)] else: specs = [] if req.python_version == 2: - return get_package_for_python_module(apt_mgr, req.module, "cpython2", specs) + return await get_package_for_python_object_path( + apt_mgr, req.module, "cpython2", specs) elif req.python_version in (None, 3): - return get_package_for_python_module(apt_mgr, req.module, "cpython3", specs) + return await get_package_for_python_object_path( + apt_mgr, req.module, "cpython3", specs) else: return None -def resolve_python_package_req(apt_mgr, req): +async def resolve_python_package_req(apt_mgr, req): if req.python_version == 2: - return get_package_for_python_package( + return await get_package_for_python_package( apt_mgr, req.package, "cpython2", req.specs ) elif req.python_version in (None, 3): - return get_package_for_python_package( + return await get_package_for_python_package( apt_mgr, req.package, "cpython3", req.specs ) else: return None -def resolve_cargo_crate_req(apt_mgr, req): - paths = ["/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml" % re.escape(req.crate)] - return find_reqs_simple(apt_mgr, paths, regex=True) +async def resolve_cargo_crate_req(apt_mgr, req): + paths = [ + "/usr/share/cargo/registry/%s\\-[0-9]+.*/Cargo\\.toml" + % re.escape(req.crate)] + return await find_reqs_simple( + apt_mgr, paths, regex=True, minimum_version=req.minimum_version) -def resolve_ca_req(apt_mgr, req): +async def resolve_ca_req(apt_mgr, req): return [AptRequirement.simple("ca-certificates")] -def resolve_introspection_typelib_req(apt_mgr, req): - return find_reqs_simple( - apt_mgr, [r'/usr/lib/.*/girepository-.*/%s-.*\.typelib' % re.escape(req.library)], +async def resolve_introspection_typelib_req(apt_mgr, req): + return await find_reqs_simple( + apt_mgr, + [r'/usr/lib/.*/girepository\\-.*/%s\\-.*\.typelib' + % re.escape(req.library)], regex=True) -def resolve_apt_req(apt_mgr, req): +async def resolve_apt_req(apt_mgr, req): # TODO(jelmer): This should be checking whether versions match as well. for package_name in req.package_names(): if not apt_mgr.package_exists(package_name): @@ -697,13 +845,47 @@ def resolve_apt_req(apt_mgr, req): return [req] -def resolve_boost_component_req(apt_mgr, req): - return find_reqs_simple( +async def resolve_boost_component_req(apt_mgr, req): + return await find_reqs_simple( apt_mgr, ["/usr/lib/.*/libboost_%s" % re.escape(req.name)], regex=True) -APT_REQUIREMENT_RESOLVERS = [ +async def resolve_kf5_component_req(apt_mgr, req): + return await find_reqs_simple( + apt_mgr, ["/usr/lib/.*/cmake/KF5%s/KF5%sConfig\\.cmake" % ( + re.escape(req.name), re.escape(req.name))], + regex=True) + + +async def resolve_vcs_access_req(apt_mgr, req): + PKG_MAP = { + 'hg': 'mercurial', + 'svn': 'subversion', + 'git': 'git', + 'bzr': 'bzr', + } + ret = [] + for vcs in req.vcs: + try: + ret.append(PKG_MAP[vcs]) + except KeyError: + logging.debug('Unable to map VCS %s to package', vcs) + return [AptRequirement.from_str(','.join(ret))] + + +async def resolve_oneof_req(apt_mgr, req): + options = await asyncio.gather( + *[resolve_requirement_apt(apt_mgr, req) for req in req.elements]) + for option in options: + if not option: + continue + return option + + +APT_REQUIREMENT_RESOLVERS: List[Tuple[ + Type[Requirement], Callable[ + [AptManager, Requirement], Awaitable[List[AptRequirement]]]]] = [ (AptRequirement, resolve_apt_req), (BinaryRequirement, resolve_binary_req), (VagueDependencyRequirement, resolve_vague_dep_req), @@ -736,6 +918,7 @@ APT_REQUIREMENT_RESOLVERS = [ (JDKRequirement, resolve_jdk_req), (JRERequirement, resolve_jre_req), (QTRequirement, resolve_qt_req), + (QtModuleRequirement, resolve_qt_module_req), (X11Requirement, resolve_x11_req), (LibtoolRequirement, resolve_libtool_req), (PerlModuleRequirement, resolve_perl_module_req), @@ -747,13 +930,19 @@ APT_REQUIREMENT_RESOLVERS = [ (CargoCrateRequirement, resolve_cargo_crate_req), (IntrospectionTypelibRequirement, resolve_introspection_typelib_req), (BoostComponentRequirement, resolve_boost_component_req), + (KF5ComponentRequirement, resolve_kf5_component_req), + (PHPExtensionRequirement, resolve_php_extension_req), + (OctavePackageRequirement, resolve_octave_pkg_req), + (VcsControlDirectoryAccessRequirement, resolve_vcs_access_req), + (OneOfRequirement, resolve_oneof_req), ] -def resolve_requirement_apt(apt_mgr, req: Requirement) -> List[AptRequirement]: +async def resolve_requirement_apt( + apt_mgr, req: Requirement) -> List[AptRequirement]: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): - ret = rr_fn(apt_mgr, req) + ret = await rr_fn(apt_mgr, req) if not ret: return [] if not isinstance(ret, list): @@ -782,7 +971,8 @@ class AptResolver(Resolver): return "apt" def __repr__(self): - return "%s(%r, %r)" % (type(self).__name__, self.apt, self.tie_breakers) + return "%s(%r, %r)" % ( + type(self).__name__, self.apt, self.tie_breakers) @classmethod def from_session(cls, session, tie_breakers=None): @@ -808,8 +998,8 @@ class AptResolver(Resolver): apt_requirements.append(apt_req) if apt_requirements: self.apt.satisfy( - [PkgRelation.str(chain(*[r.relations for r in apt_requirements]))] - ) + [PkgRelation.str( + chain(*[r.relations for r in apt_requirements]))]) if still_missing: raise UnsatisfiedRequirements(still_missing) @@ -831,18 +1021,23 @@ class AptResolver(Resolver): [o for o, r in apt_requirements], ) + def resolve_all(self, req: Requirement): + return asyncio.run(resolve_requirement_apt(self.apt, req)) + def resolve(self, req: Requirement): - ret = resolve_requirement_apt(self.apt, req) + ret = self.resolve_all(req) if not ret: return None if len(ret) == 1: return ret[0] - logging.info("Need to break tie between %r with %r", ret, self.tie_breakers) + logging.info( + "Need to break tie between %r with %r", ret, self.tie_breakers) for tie_breaker in self.tie_breakers: winner = tie_breaker(ret) if winner is not None: if not isinstance(winner, AptRequirement): raise TypeError(winner) return winner - logging.info("Unable to break tie over %r, picking first: %r", ret, ret[0]) + logging.info( + "Unable to break tie over %r, picking first: %r", ret, ret[0]) return ret[0] diff --git a/ognibuild/resolver/dep_server.py b/ognibuild/resolver/dep_server.py new file mode 100644 index 0000000..cd80385 --- /dev/null +++ b/ognibuild/resolver/dep_server.py @@ -0,0 +1,88 @@ +#!/usr/bin/python3 +# Copyright (C) 2022 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import asyncio +import logging +from typing import List + +from aiohttp import ( + ClientSession, + ClientConnectorError, + ClientResponseError, + ServerDisconnectedError, +) +from yarl import URL + + +from .. import Requirement, USER_AGENT +from ..debian.apt import AptManager +from .apt import AptRequirement, AptResolver + + +class DepServerError(Exception): + + def __init__(self, inner): + self.inner = inner + + +async def resolve_apt_requirement_dep_server( + url: str, req: Requirement) -> List[AptRequirement]: + """Resolve a requirement to an APT requirement with a dep server. + + Args: + url: Dep server URL + req: Requirement to resolve + Returns: + List of Apt requirements. + """ + async with ClientSession() as session: + try: + async with session.post(URL(url) / "resolve-apt", headers={ + 'User-Agent': USER_AGENT}, + json={'requirement': req.json()}, + raise_for_status=True) as resp: + return [ + AptRequirement._from_json(e) for e in await resp.json()] + except (ClientConnectorError, ClientResponseError, + ServerDisconnectedError) as e: + logging.warning('Unable to contact dep server: %r', e) + raise DepServerError(e) + + +class DepServerAptResolver(AptResolver): + def __init__(self, apt, dep_server_url, tie_breakers=None): + super(DepServerAptResolver, self).__init__( + apt, tie_breakers=tie_breakers) + self.dep_server_url = dep_server_url + + @classmethod + def from_session(cls, session, dep_server_url, tie_breakers=None): + return cls( + AptManager.from_session(session), dep_server_url, + tie_breakers=tie_breakers) + + def resolve_all(self, req: Requirement): + try: + req.json() + except NotImplementedError: + return super(DepServerAptResolver, self).resolve_all(req) + try: + return asyncio.run( + resolve_apt_requirement_dep_server(self.dep_server_url, req)) + except DepServerError: + logging.warning('Falling back to resolving error locally') + return super(DepServerAptResolver, self).resolve_all(req) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 464381d..0ef6a1a 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -69,12 +69,14 @@ class Session(object): raise NotImplementedError(self.check_output) def Popen( - self, argv, cwd: Optional[str] = None, user: Optional[str] = None, **kwargs + self, argv, cwd: Optional[str] = None, user: Optional[str] = None, + **kwargs ): raise NotImplementedError(self.Popen) def call( - self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None + self, argv: List[str], cwd: Optional[str] = None, + user: Optional[str] = None ): raise NotImplementedError(self.call) @@ -100,17 +102,26 @@ class Session(object): def external_path(self, path: str) -> str: raise NotImplementedError + def rmtree(self, path: str) -> str: + raise NotImplementedError + is_temporary: bool class SessionSetupFailure(Exception): """Session failed to be set up.""" + def __init__(self, reason, errlines=None): + self.reason = reason + self.errlines = errlines -def run_with_tee(session: Session, args: List[str], **kwargs): + +def run_with_tee(session: Session, + args: List[str], **kwargs) -> Tuple[int, List[str]]: if "stdin" not in kwargs: kwargs["stdin"] = subprocess.DEVNULL - p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) + p = session.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) contents = [] while p.poll() is None: line = p.stdout.readline() @@ -121,7 +132,8 @@ def run_with_tee(session: Session, args: List[str], **kwargs): def get_user(session): - return session.check_output(["echo", "$USER"], cwd="/").decode().strip() + return session.check_output( + ["sh", "-c", "echo $USER"], cwd="/").decode().strip() def which(session, name): diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index df4b1a2..ab08e6c 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -20,6 +20,7 @@ from . import Session, NoSessionOpen, SessionAlreadyOpen import contextlib import os +import shutil import subprocess import tempfile from typing import Optional, Dict, List @@ -72,7 +73,8 @@ class PlainSession(Session): close_fds: bool = True, ): argv = self._prepend_user(user, argv) - return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds) + return subprocess.check_call( + argv, cwd=cwd, env=env, close_fds=close_fds) def check_output( self, @@ -84,13 +86,19 @@ class PlainSession(Session): argv = self._prepend_user(user, argv) return subprocess.check_output(argv, cwd=cwd, env=env) - def Popen(self, args, stdout=None, stderr=None, stdin=None, user=None, cwd=None, env=None): + def Popen( + self, args, stdout=None, stderr=None, stdin=None, user=None, + cwd=None, env=None): args = self._prepend_user(user, args) - return subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, cwd=cwd, env=env) + return subprocess.Popen( + args, stdout=stdout, stderr=stderr, stdin=stdin, cwd=cwd, env=env) def exists(self, path): return os.path.exists(path) + def rmtree(self, path): + return shutil.rmtree(path) + def scandir(self, path): return os.scandir(path) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index bc9bcd1..7a485f2 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -66,25 +66,38 @@ class SchrootSession(Session): if line.startswith(b"E: "): logging.error("%s", line[3:].decode(errors="replace")) logging.warning( - "Failed to close schroot session %s, leaving stray.", self.session_id + "Failed to close schroot session %s, leaving stray.", + self.session_id ) self.session_id = None return False self.session_id = None + self._location = None return True def __enter__(self) -> "Session": if self.session_id is not None: raise SessionAlreadyOpen(self) + stderr = tempfile.TemporaryFile() try: self.session_id = ( - subprocess.check_output(["schroot", "-c", self.chroot, "-b"]) + subprocess.check_output( + ["schroot", "-c", self.chroot, "-b"], stderr=stderr) .strip() .decode() ) except subprocess.CalledProcessError: - # TODO(jelmer): Capture stderr and forward in SessionSetupFailure - raise SessionSetupFailure() + stderr.seek(0) + errlines = stderr.readlines() + if len(errlines) == 1: + raise SessionSetupFailure( + errlines[0].rstrip().decode(), errlines=errlines) + elif len(errlines) == 0: + raise SessionSetupFailure( + "No output from schroot", errlines=errlines) + else: + raise SessionSetupFailure( + errlines[-1].decode(), errlines=errlines) logging.info( "Opened schroot session %s (from %s)", self.session_id, self.chroot ) @@ -156,24 +169,28 @@ class SchrootSession(Session): env: Optional[Dict[str, str]] = None, ) -> bytes: try: - return subprocess.check_output(self._run_argv(argv, cwd, user, env=env)) + return subprocess.check_output( + self._run_argv(argv, cwd, user, env=env)) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv) def Popen( - self, argv, cwd: Optional[str] = None, user: Optional[str] = None, **kwargs + self, argv, cwd: Optional[str] = None, user: Optional[str] = None, + **kwargs ): return subprocess.Popen(self._run_argv(argv, cwd, user), **kwargs) def call( - self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None + self, argv: List[str], cwd: Optional[str] = None, + user: Optional[str] = None ): return subprocess.call(self._run_argv(argv, cwd, user)) def create_home(self) -> None: """Create the user's home directory.""" home = ( - self.check_output(["sh", "-c", "echo $HOME"], cwd="/").decode().rstrip("\n") + self.check_output( + ["sh", "-c", "echo $HOME"], cwd="/").decode().rstrip("\n") ) user = ( self.check_output(["sh", "-c", "echo $LOGNAME"], cwd="/") @@ -189,7 +206,8 @@ class SchrootSession(Session): return os.path.join(self.location, path.lstrip("/")) if self._cwd is None: raise ValueError("no cwd set") - return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/")) + return os.path.join( + self.location, os.path.join(self._cwd, path).lstrip("/")) def exists(self, path: str) -> bool: fullpath = self.external_path(path) @@ -203,13 +221,17 @@ class SchrootSession(Session): fullpath = self.external_path(path) return os.mkdir(fullpath) + def rmtree(self, path: str): + import shutil + fullpath = self.external_path(path) + return shutil.rmtree(fullpath) + def setup_from_vcs( self, tree, include_controldir: Optional[bool] = None, subdir="package" ): from ..vcs import dupe_vcs_tree, export_vcs_tree build_dir = os.path.join(self.location, "build") - directory = tempfile.mkdtemp(dir=build_dir) reldir = "/" + os.path.relpath(directory, self.location) @@ -228,7 +250,7 @@ class SchrootSession(Session): directory = tempfile.mkdtemp(dir=build_dir) reldir = "/" + os.path.relpath(directory, self.location) export_directory = os.path.join(directory, subdir) - shutil.copytree(path, export_directory, dirs_exist_ok=True) + shutil.copytree(path, export_directory, symlinks=True) return export_directory, os.path.join(reldir, subdir) is_temporary = True diff --git a/ognibuild/test.py b/ognibuild/test.py index 750143f..2ab8261 100644 --- a/ognibuild/test.py +++ b/ognibuild/test.py @@ -15,16 +15,25 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from functools import partial + from .buildsystem import NoBuildToolsFound +from .fix_build import iterate_with_build_fixers +from .logs import NoLogManager -def run_test(session, buildsystems, resolver, fixers): +def run_test(session, buildsystems, resolver, fixers, log_manager=None): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() + if log_manager is None: + log_manager = NoLogManager() + for buildsystem in buildsystems: - buildsystem.test(session, resolver, fixers) + iterate_with_build_fixers( + fixers, log_manager.wrap( + partial(buildsystem.test, session, resolver))) return raise NoBuildToolsFound() diff --git a/ognibuild/upstream.py b/ognibuild/upstream.py new file mode 100644 index 0000000..8a08c90 --- /dev/null +++ b/ognibuild/upstream.py @@ -0,0 +1,253 @@ +#!/usr/bin/python3 +# Copyright (C) 2020-2021 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from dataclasses import dataclass, field +from typing import Optional, Dict, Any +from debian.changelog import Version +import logging +import re + +from . import Requirement +from .requirements import ( + CargoCrateRequirement, + GoPackageRequirement, + PythonPackageRequirement, +) +from .resolver.apt import AptRequirement, OneOfRequirement + + +@dataclass +class UpstreamInfo: + name: Optional[str] + buildsystem: Optional[str] = None + branch_url: Optional[str] = None + branch_subpath: Optional[str] = None + tarball_url: Optional[str] = None + version: Optional[str] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + def json(self): + return { + 'name': self.name, + 'buildsystem': self.buildsystem, + 'branch_url': self.branch_url, + 'branch_subpath': self.branch_subpath, + 'tarball_url': self.tarball_url, + 'version': self.version + } + + +def go_base_name(package): + (hostname, path) = package.split('/', 1) + if hostname == "github.com": + hostname = "github" + if hostname == "gopkg.in": + hostname = "gopkg" + path = path.rstrip('/').replace("/", "-") + if path.endswith('.git'): + path = path[:-4] + return (hostname + path).replace("_", "-").lower() + + +def load_crate_info(crate): + import urllib.error + from urllib.request import urlopen, Request + import json + http_url = 'https://crates.io/api/v1/crates/%s' % crate + headers = {'User-Agent': 'debianize', 'Accept': 'application/json'} + http_contents = urlopen(Request(http_url, headers=headers)).read() + try: + return json.loads(http_contents) + except urllib.error.HTTPError as e: + if e.code == 404: + logging.warning('No crate %r', crate) + return None + raise + + +def find_python_package_upstream(requirement): + import urllib.error + from urllib.request import urlopen, Request + import json + http_url = 'https://pypi.org/pypi/%s/json' % requirement.package + headers = {'User-Agent': 'ognibuild', 'Accept': 'application/json'} + try: + http_contents = urlopen( + Request(http_url, headers=headers)).read() + except urllib.error.HTTPError as e: + if e.code == 404: + logging.warning('No pypi project %r', requirement.package) + return None + raise + pypi_data = json.loads(http_contents) + upstream_branch = None + for name, url in pypi_data['info']['project_urls'].items(): + if name.lower() in ('github', 'repository'): + upstream_branch = url + tarball_url = None + for url_data in pypi_data['urls']: + if url_data.get('package_type') == 'sdist': + tarball_url = url_data['url'] + return UpstreamInfo( + branch_url=upstream_branch, branch_subpath='', + name='python-%s' % pypi_data['info']['name'], + tarball_url=tarball_url) + + +def find_go_package_upstream(requirement): + if requirement.package.startswith('github.com/'): + return UpstreamInfo( + name='golang-%s' % go_base_name(requirement.package), + branch_url='https://%s' % '/'.join( + requirement.package.split('/')[:3]), + branch_subpath='') + + +def find_cargo_crate_upstream(requirement): + import semver + from debmutate.debcargo import semver_pair + data = load_crate_info(requirement.crate) + if data is None: + return None + upstream_branch = data['crate']['repository'] + name = 'rust-' + data['crate']['name'].replace('_', '-') + version = None + if requirement.api_version is not None: + for version_info in data['versions']: + if (not version_info['num'].startswith( + requirement.api_version + '.') + and not version_info['num'] == requirement.api_version): + continue + if version is None: + version = semver.VersionInfo.parse(version_info['num']) + else: + version = semver.max_ver( + version, semver.VersionInfo.parse(version_info['num'])) + if version is None: + logging.warning( + 'Unable to find version of crate %s ' + 'that matches API version %s', + name, requirement.api_version) + else: + name += '-' + semver_pair(str(version)) + return UpstreamInfo( + branch_url=upstream_branch, branch_subpath=None, + name=name, version=str(version) if version else None, + metadata={'X-Cargo-Crate': data['crate']['name']}, + buildsystem='cargo') + + +def apt_to_cargo_requirement(m, rels): + name = m.group(1) + api_version = m.group(2) + if m.group(3): + features = set(m.group(3)[1:].split('-')) + else: + features = set() + if not rels: + minimum_version = None + elif len(rels) == 1 and rels[0][0] == '>=': + minimum_version = Version(rels[0][1]).upstream_version + else: + logging.warning('Unable to parse Debian version %r', rels) + minimum_version = None + + return CargoCrateRequirement( + name, api_version=api_version, + features=features, minimum_version=minimum_version) + + +def apt_to_python_requirement(m, rels): + name = m.group(2) + python_version = m.group(1) + if not rels: + minimum_version = None + elif len(rels) == 1 and rels[0][0] == '>=': + minimum_version = Version(rels[0][1]).upstream_version + else: + logging.warning('Unable to parse Debian version %r', rels) + minimum_version = None + return PythonPackageRequirement( + name, python_version=(python_version or None), + minimum_version=minimum_version) + + +def apt_to_go_requirement(m, rels): + parts = m.group(1).split('-') + if parts[0] == 'github': + parts[0] = 'github.com' + if parts[0] == 'gopkg': + parts[0] = 'gopkg.in' + if not rels: + version = None + elif len(rels) == 1 and rels[0][0] == '=': + version = Version(rels[0][1]).upstream_version + else: + logging.warning('Unable to parse Debian version %r', rels) + version = None + return GoPackageRequirement('/'.join(parts), version=version) + + +BINARY_PACKAGE_UPSTREAM_MATCHERS = [ + (r'librust-(.*)-([^-+]+)(\+.*?)-dev', apt_to_cargo_requirement), + (r'python([0-9.]*)-(.*)', apt_to_python_requirement), + (r'golang-(.*)-dev', apt_to_go_requirement), +] + + +_BINARY_PACKAGE_UPSTREAM_MATCHERS = [ + (re.compile(r), fn) for (r, fn) in BINARY_PACKAGE_UPSTREAM_MATCHERS] + + +def find_apt_upstream(requirement: AptRequirement) -> Optional[UpstreamInfo]: + for option in requirement.relations: + for rel in option: + for matcher, fn in _BINARY_PACKAGE_UPSTREAM_MATCHERS: + m = matcher.fullmatch(rel['name']) + if m: + upstream_requirement = fn( + m, [rel['version']] if rel['version'] else []) + return find_upstream(upstream_requirement) + + logging.warning( + 'Unable to map binary package name %s to upstream', + rel['name']) + return None + + +def find_or_upstream(requirement: OneOfRequirement) -> Optional[UpstreamInfo]: + for req in requirement.elements: + info = find_upstream(req) + if info is not None: + return info + return None + + +UPSTREAM_FINDER = { + 'python-package': find_python_package_upstream, + 'go-package': find_go_package_upstream, + 'cargo-crate': find_cargo_crate_upstream, + 'apt': find_apt_upstream, + 'or': find_or_upstream, + } + + +def find_upstream(requirement: Requirement) -> Optional[UpstreamInfo]: + try: + return UPSTREAM_FINDER[requirement.family](requirement) + except KeyError: + return None diff --git a/ognibuild/vcs.py b/ognibuild/vcs.py index 45b32c6..9115e28 100644 --- a/ognibuild/vcs.py +++ b/ognibuild/vcs.py @@ -43,7 +43,8 @@ def dupe_vcs_tree(tree, directory): tree = tree.basis_tree() try: result = tree._repository.controldir.sprout( - directory, create_tree_if_local=True, revision_id=tree.get_revision_id() + directory, create_tree_if_local=True, + revision_id=tree.get_revision_id() ) except OSError as e: if e.errno == errno.ENOSPC: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fed528d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/releaser.conf b/releaser.conf deleted file mode 100644 index c4c19a4..0000000 --- a/releaser.conf +++ /dev/null @@ -1,14 +0,0 @@ -name: "ognibuild" -timeout_days: 5 -tag_name: "v$VERSION" -verify_command: "python3 setup.py test" -update_version { - path: "setup.py" - match: "^ version=\"(.*)\",$" - new_line: " version=\"$VERSION\"," -} -update_version { - path: "ognibuild/__init__.py" - match: "^__version__ = \\((.*)\\)$" - new_line: "__version__ = $TUPLED_VERSION" -} diff --git a/scripts/report-apt-deps-status b/scripts/report-apt-deps-status new file mode 100755 index 0000000..b0681d3 --- /dev/null +++ b/scripts/report-apt-deps-status @@ -0,0 +1,89 @@ +#!/usr/bin/python3 + +import argparse +from contextlib import ExitStack +import logging +import sys +from typing import Dict, List + +from ognibuild.buildsystem import NoBuildToolsFound, detect_buildsystems +from ognibuild.requirements import Requirement +from ognibuild.resolver.apt import AptResolver +from ognibuild.session.plain import PlainSession + +parser = argparse.ArgumentParser('report-apt-deps-status') +parser.add_argument('directory', type=str, default='.', nargs='?') +parser.add_argument( + '--detailed', action='store_true', help='Show detailed analysis') +args = parser.parse_args() + +logging.basicConfig(format='%(message)s', level=logging.INFO) + +session = PlainSession() +with ExitStack() as es: + es.enter_context(session) + session.chdir(args.directory) + resolver = AptResolver.from_session(session) + + try: + bss = list(detect_buildsystems(args.directory)) + except NoBuildToolsFound: + logging.fatal('No build tools found') + sys.exit(1) + logging.debug("Detected buildsystems: %s", ", ".join(map(str, bss))) + deps: Dict[str, List[Requirement]] = {} + for buildsystem in bss: + try: + declared_reqs = buildsystem.get_declared_dependencies(session, []) + for stage, req in declared_reqs: + deps.setdefault(stage, []).append(req) + except NotImplementedError: + logging.warning( + 'Unable to get dependencies from buildsystem %r, skipping', + buildsystem) + continue + + if args.detailed: + for stage, reqs in deps.items(): + logging.info("Stage: %s", stage) + for req in reqs: + apt_req = resolver.resolve(req) + logging.info("%s: %s", req, apt_req.pkg_relation_str()) + logging.info('') + else: + build_depends = [] + test_depends = [] + run_depends = [] + unresolved = [] + for stage, reqs in deps.items(): + for req in reqs: + apt_req = resolver.resolve(req) + if apt_req is None: + unresolved.append(req) + elif stage == 'core': + build_depends.append(apt_req) + run_depends.append(apt_req) + elif stage == 'build': + build_depends.append(apt_req) + elif stage == 'test': + test_depends.append(apt_req) + else: + raise NotImplementedError('stage %s not supported' % stage) + if build_depends: + logging.info( + 'Build-Depends: %s', + ', '.join([d.pkg_relation_str() for d in build_depends])) + if test_depends: + logging.info( + 'Test-Depends: %s', + ', '.join([d.pkg_relation_str() for d in test_depends])) + if run_depends: + logging.info( + 'Depends: %s', + ', '.join([d.pkg_relation_str() for d in run_depends])) + if unresolved: + sys.stdout.write('\n') + logging.warning( + 'Unable to find apt packages for the following dependencies:') + for req in unresolved: + logging.warning('* %s', req) diff --git a/setup.cfg b/setup.cfg index 07ea45f..e4e8584 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,13 +1,65 @@ +[metadata] +name = ognibuild +description = Detect and run any build system +version = attr:ognibuild.__version__ +maintainer = Jelmer Vernooij +maintainer_email = jelmer@jelmer.uk +license = GNU GPLv2 or later +url = https://jelmer.uk/code/ognibuild +classifiers = + Development Status :: 4 - Beta + License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: Implementation :: CPython + Operating System :: POSIX + +[options] +packages = + ognibuild + ognibuild.debian + ognibuild.resolver + ognibuild.session +scripts = scripts/report-apt-deps-status +install_requires = + breezy>=3.2 + buildlog-consultant>=0.0.21 + requirements-parser + toml + setuptools + ruamel.yaml +tests_require = + testtools + types-toml + +[options.entry_points] +console_scripts = + ogni=ognibuild.__main__:main + deb-fix-build=ognibuild.debian.fix_build:main + +[options.extras_require] +dev = + testtools +debian = + debmutate + python_debian + python_apt + brz-debian + lz4 +remote = + breezy + dulwich +dep_server = + aiohttp + aiohttp-openmetrics +gcp = google-cloud-logging + [flake8] banned-modules = silver-platter = Should not use silver-platter +exclude = build,.eggs/ [mypy] ignore_missing_imports = True [bdist_wheel] universal = 1 - -[egg_info] -tag_build = -tag_date = 0 - diff --git a/setup.py b/setup.py index 8167aae..eb46b40 100755 --- a/setup.py +++ b/setup.py @@ -1,40 +1,3 @@ -#!/usr/bin/env python3 -# encoding: utf-8 - +#!/usr/bin/python3 from setuptools import setup - - -setup(name="ognibuild", - description="Detect and run any build system", - version="0.0.7", - maintainer="Jelmer Vernooij", - maintainer_email="jelmer@jelmer.uk", - license="GNU GPLv2 or later", - url="https://jelmer.uk/code/ognibuild", - packages=['ognibuild', 'ognibuild.tests', 'ognibuild.debian', 'ognibuild.resolver', 'ognibuild.session'], - classifiers=[ - 'Development Status :: 4 - Beta', - 'License :: OSI Approved :: ' - 'GNU General Public License v2 or later (GPLv2+)', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Operating System :: POSIX', - ], - entry_points={ - "console_scripts": [ - "ogni=ognibuild.__main__:main", - "deb-fix-build=ognibuild.debian.fix_build:main", - ] - }, - install_requires=[ - 'breezy', - 'buildlog-consultant>=0.0.10', - 'requirements-parser', - ], - extras_require={ - 'debian': ['debmutate', 'python_debian', 'python_apt'], - }, - tests_require=['python_debian', 'buildlog-consultant', 'breezy', 'testtools'], - test_suite='ognibuild.tests.test_suite', - ) +setup() diff --git a/ognibuild/tests/__init__.py b/tests/__init__.py similarity index 85% rename from ognibuild/tests/__init__.py rename to tests/__init__.py index 075535d..71727f9 100644 --- a/ognibuild/tests/__init__.py +++ b/tests/__init__.py @@ -23,10 +23,13 @@ import unittest def test_suite(): names = [ - "debian_build", + 'buildlog', + 'logs', ] if os.path.exists("/usr/bin/dpkg-architecture"): + names.append("debian_build") names.append("debian_fix_build") - module_names = ["ognibuild.tests.test_" + name for name in names] + names.append("resolver_apt") + module_names = ["tests.test_" + name for name in names] loader = unittest.TestLoader() return loader.loadTestsFromNames(module_names) diff --git a/tests/test_buildlog.py b/tests/test_buildlog.py new file mode 100644 index 0000000..e1d19d9 --- /dev/null +++ b/tests/test_buildlog.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# Copyright (C) 2022 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from ognibuild.buildlog import PROBLEM_CONVERTERS + +from buildlog_consultant import ( + problem_clses, + __version__ as buildlog_consultant_version, +) + +from unittest import TestCase + + +class TestProblemsExists(TestCase): + + def test_exist(self): + for entry in PROBLEM_CONVERTERS: + if len(entry) == 2: + problem_kind, fn = entry + min_version = None + elif len(entry) == 3: + problem_kind, fn, min_version = entry + else: + raise TypeError(entry) + if min_version is not None: + min_version_tuple = tuple( + [int(x) for x in min_version.split('.')]) + if buildlog_consultant_version < min_version_tuple: + continue + self.assertTrue( + problem_kind in problem_clses, + f"{problem_kind} does not exist in known " + "buildlog-consultant problem kinds") diff --git a/ognibuild/tests/test_debian_build.py b/tests/test_debian_build.py similarity index 72% rename from ognibuild/tests/test_debian_build.py rename to tests/test_debian_build.py index 0b06869..fe656cd 100644 --- a/ognibuild/tests/test_debian_build.py +++ b/tests/test_debian_build.py @@ -17,8 +17,17 @@ import datetime import os +import sys -from ..debian.build import add_dummy_changelog_entry, get_build_architecture +from debian.changelog import Version + +from ognibuild.debian.build import ( + add_dummy_changelog_entry, + get_build_architecture, + version_add_suffix, + _builddeb_command, + DEFAULT_BUILDER, +) from breezy.tests import TestCaseWithTransport, TestCase @@ -150,3 +159,43 @@ class BuildArchitectureTests(TestCase): def test_is_str(self): self.assertIsInstance(get_build_architecture(), str) + + +class VersionAddSuffixTests(TestCase): + + def test_native(self): + self.assertEqual( + Version('1.0~jan+lint4'), + version_add_suffix(Version('1.0~jan+lint3'), '~jan+lint')) + self.assertEqual( + Version('1.0~jan+lint1'), + version_add_suffix(Version('1.0'), '~jan+lint')) + + def test_normal(self): + self.assertEqual( + Version('1.0-1~jan+lint4'), + version_add_suffix(Version('1.0-1~jan+lint3'), '~jan+lint')) + self.assertEqual( + Version('1.0-1~jan+lint1'), + version_add_suffix(Version('1.0-1'), '~jan+lint')) + self.assertEqual( + Version('0.0.12-1~jan+lint1'), + version_add_suffix(Version('0.0.12-1'), '~jan+lint')) + self.assertEqual( + Version('0.0.12-1~jan+unchanged1~jan+lint1'), + version_add_suffix( + Version('0.0.12-1~jan+unchanged1'), '~jan+lint')) + + +class BuilddebCommandTests(TestCase): + + def test_simple(self): + self.assertEqual( + [sys.executable, "-m", "breezy", "builddeb", + "--guess-upstream-branch-url", "--builder=" + DEFAULT_BUILDER], + _builddeb_command()) + self.assertEqual( + [sys.executable, "-m", "breezy", "builddeb", + "--guess-upstream-branch-url", "--builder=" + DEFAULT_BUILDER, + "--result-dir=/tmp/blah"], + _builddeb_command(result_dir="/tmp/blah")) diff --git a/ognibuild/tests/test_debian_fix_build.py b/tests/test_debian_fix_build.py similarity index 72% rename from ognibuild/tests/test_debian_fix_build.py rename to tests/test_debian_fix_build.py index c0a5456..ba28ffc 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/tests/test_debian_fix_build.py @@ -29,13 +29,15 @@ from buildlog_consultant.common import ( MissingRubyGem, MissingValaPackage, ) -from ..debian.apt import AptManager, FileSearcher -from ..debian.fix_build import ( +from ognibuild.debian.apt import AptManager, FileSearcher +from ognibuild.debian.fix_build import ( resolve_error, versioned_package_fixers, apt_fixers, DebianPackagingContext, + add_build_dependency, ) +from ognibuild.resolver.apt import AptRequirement from breezy.commit import NullCommitReporter from breezy.tests import TestCaseWithTransport @@ -44,7 +46,7 @@ class DummyAptSearcher(FileSearcher): def __init__(self, files): self._apt_files = files - def search_files(self, path, regex=False, case_insensitive=False): + async def search_files(self, path, regex=False, case_insensitive=False): for p, pkg in sorted(self._apt_files.items()): if case_insensitive: flags = re.I @@ -97,7 +99,7 @@ blah (0.1) UNRELEASED; urgency=medium self._apt_files = {} def resolve(self, error, context=("build",)): - from ..session.plain import PlainSession + from ognibuild.session.plain import PlainSession session = PlainSession() apt = AptManager(session) @@ -109,7 +111,8 @@ blah (0.1) UNRELEASED; urgency=medium update_changelog=True, commit_reporter=NullCommitReporter(), ) - fixers = versioned_package_fixers(session, context, apt) + apt_fixers(apt, context) + fixers = versioned_package_fixers( + session, context, apt) + apt_fixers(apt, context) return resolve_error(error, ("build",), fixers) def get_build_deps(self): @@ -118,7 +121,8 @@ blah (0.1) UNRELEASED; urgency=medium def test_missing_command_unknown(self): self._apt_files = {} - self.assertFalse(self.resolve(MissingCommand("acommandthatdoesnotexist"))) + self.assertFalse(self.resolve( + MissingCommand("acommandthatdoesnotexist"))) def test_missing_command_brz(self): self._apt_files = { @@ -130,7 +134,8 @@ blah (0.1) UNRELEASED; urgency=medium self.overrideEnv("DEBFULLNAME", "Jelmer Vernooij") self.assertTrue(self.resolve(MissingCommand("brz"))) self.assertEqual("libc6, brz", self.get_build_deps()) - rev = self.tree.branch.repository.get_revision(self.tree.branch.last_revision()) + rev = self.tree.branch.repository.get_revision( + self.tree.branch.last_revision()) self.assertEqual("Add missing build dependency on brz.\n", rev.message) self.assertFalse(self.resolve(MissingCommand("brz"))) self.assertEqual("libc6, brz", self.get_build_deps()) @@ -153,10 +158,12 @@ blah (0.1) UNRELEASED; urgency=medium def test_missing_ruby_file_from_gem(self): self._apt_files = { "/usr/share/rubygems-integration/all/gems/activesupport-" - "5.2.3/lib/active_support/core_ext/string/strip.rb": "ruby-activesupport" + "5.2.3/lib/active_support/core_ext/string/strip.rb": + "ruby-activesupport" } self.assertTrue( - self.resolve(MissingRubyFile("active_support/core_ext/string/strip")) + self.resolve(MissingRubyFile( + "active_support/core_ext/string/strip")) ) self.assertEqual("libc6, ruby-activesupport", self.get_build_deps()) @@ -173,7 +180,8 @@ blah (0.1) UNRELEASED; urgency=medium self.assertEqual("libc6, ruby-bio (>= 2.0.3)", self.get_build_deps()) def test_missing_perl_module(self): - self._apt_files = {"/usr/share/perl5/App/cpanminus/fatscript.pm": "cpanminus"} + self._apt_files = { + "/usr/share/perl5/App/cpanminus/fatscript.pm": "cpanminus"} self.assertTrue( self.resolve( MissingPerlModule( @@ -200,28 +208,34 @@ blah (0.1) UNRELEASED; urgency=medium def test_missing_pkg_config(self): self._apt_files = { - "/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": "libxcb-xfixes0-dev" + "/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": + "libxcb-xfixes0-dev" } self.assertTrue(self.resolve(MissingPkgConfig("xcb-xfixes"))) self.assertEqual("libc6, libxcb-xfixes0-dev", self.get_build_deps()) def test_missing_pkg_config_versioned(self): self._apt_files = { - "/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": "libxcb-xfixes0-dev" + "/usr/lib/x86_64-linux-gnu/pkgconfig/xcb-xfixes.pc": + "libxcb-xfixes0-dev" } self.assertTrue(self.resolve(MissingPkgConfig("xcb-xfixes", "1.0"))) - self.assertEqual("libc6, libxcb-xfixes0-dev (>= 1.0)", self.get_build_deps()) + self.assertEqual( + "libc6, libxcb-xfixes0-dev (>= 1.0)", self.get_build_deps()) def test_missing_python_module(self): - self._apt_files = {"/usr/lib/python3/dist-packages/m2r.py": "python3-m2r"} + self._apt_files = { + "/usr/lib/python3/dist-packages/m2r.py": "python3-m2r"} self.assertTrue(self.resolve(MissingPythonModule("m2r"))) self.assertEqual("libc6, python3-m2r", self.get_build_deps()) def test_missing_go_package(self): self._apt_files = { - "/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go": "golang-github-chzyer-readline-dev", + "/usr/share/gocode/src/github.com/chzyer/readline/utils_test.go": + "golang-github-chzyer-readline-dev", } - self.assertTrue(self.resolve(MissingGoPackage("github.com/chzyer/readline"))) + self.assertTrue(self.resolve( + MissingGoPackage("github.com/chzyer/readline"))) self.assertEqual( "libc6, golang-github-chzyer-readline-dev", self.get_build_deps() ) @@ -232,3 +246,63 @@ blah (0.1) UNRELEASED; urgency=medium } self.assertTrue(self.resolve(MissingValaPackage("posix"))) self.assertEqual("libc6, valac-0.48-vapi", self.get_build_deps()) + + +class AddBuildDependencyTests(TestCaseWithTransport): + + def setUp(self): + super(AddBuildDependencyTests, self).setUp() + self.tree = self.make_branch_and_tree(".") + self.build_tree_contents( + [ + ("debian/",), + ( + "debian/control", + """\ +Source: blah +Build-Depends: libc6 + +Package: python-blah +Depends: ${python3:Depends} +Description: A python package + Foo +""", + ), + ( + "debian/changelog", + """\ +blah (0.1) UNRELEASED; urgency=medium + + * Initial release. (Closes: #XXXXXX) + + -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 +""", + ), + ] + ) + self.tree.add(["debian", "debian/control", "debian/changelog"]) + self.tree.commit("Initial commit") + self.context = DebianPackagingContext( + self.tree, + subpath="", + committer="ognibuild ", + update_changelog=True, + commit_reporter=NullCommitReporter(), + ) + + def test_already_present(self): + requirement = AptRequirement.simple('libc6') + self.assertFalse(add_build_dependency(self.context, requirement)) + + def test_basic(self): + requirement = AptRequirement.simple('foo') + self.assertTrue(add_build_dependency(self.context, requirement)) + self.assertFileEqual("""\ +Source: blah +Build-Depends: libc6, foo + +Package: python-blah +Depends: ${python3:Depends} +Description: A python package + Foo +""", 'debian/control') diff --git a/tests/test_logs.py b/tests/test_logs.py new file mode 100644 index 0000000..6fc4e65 --- /dev/null +++ b/tests/test_logs.py @@ -0,0 +1,95 @@ +#!/usr/bin/python +# Copyright (C) 2022 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import sys +import tempfile +from unittest import TestCase + +from ognibuild.logs import ( + copy_output, + redirect_output, + rotate_logfile, + DirectoryLogManager, +) + + +class TestCopyOutput(TestCase): + + def test_no_tee(self): + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, 'foo.log') + with copy_output(p, tee=False): + sys.stdout.write('lala\n') + sys.stdout.flush() + with open(p, 'r') as f: + self.assertEqual('lala\n', f.read()) + + def test_tee(self): + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, 'foo.log') + with copy_output(p, tee=True): + sys.stdout.write('lala\n') + sys.stdout.flush() + with open(p, 'r') as f: + self.assertEqual('lala\n', f.read()) + + +class TestRedirectOutput(TestCase): + + def test_simple(self): + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, 'foo.log') + with open(p, 'w') as f: + with redirect_output(f): + sys.stdout.write('lala\n') + sys.stdout.flush() + with open(p, 'r') as f: + self.assertEqual('lala\n', f.read()) + + +class TestRotateLogfile(TestCase): + + def test_does_not_exist(self): + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, 'foo.log') + rotate_logfile(p) + self.assertEqual([], os.listdir(td)) + + def test_simple(self): + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, 'foo.log') + with open(p, 'w') as f: + f.write('contents\n') + rotate_logfile(p) + self.assertEqual(['foo.log.1'], os.listdir(td)) + + +class TestLogManager(TestCase): + + def test_simple(self): + with tempfile.TemporaryDirectory() as td: + p = os.path.join(td, 'foo.log') + lm = DirectoryLogManager(p, mode='redirect') + + def writesomething(): + sys.stdout.write('foo\n') + sys.stdout.flush() + fn = lm.wrap(writesomething) + fn() + with open(p, 'r') as f: + self.assertEqual('foo\n', f.read()) diff --git a/tests/test_resolver_apt.py b/tests/test_resolver_apt.py new file mode 100644 index 0000000..abcad3f --- /dev/null +++ b/tests/test_resolver_apt.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# Copyright (C) 2022 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from unittest import TestCase + + +from ognibuild.resolver.apt import get_possible_python3_paths_for_python_object + + +class TestPython3Paths(TestCase): + + def test_paths(self): + self.assertEqual([ + '/usr/lib/python3/dist\\-packages/dulwich/__init__\\.py', + '/usr/lib/python3/dist\\-packages/dulwich\\.py', + '/usr/lib/python3\\.[0-9]+/' + 'lib\\-dynload/dulwich.cpython\\-.*\\.so', + '/usr/lib/python3\\.[0-9]+/dulwich\\.py', + '/usr/lib/python3\\.[0-9]+/dulwich/__init__\\.py'], + get_possible_python3_paths_for_python_object('dulwich')) + self.assertEqual([ + '/usr/lib/python3/dist\\-packages/cleo/foo/__init__\\.py', + '/usr/lib/python3/dist\\-packages/cleo/foo\\.py', + '/usr/lib/python3\\.[0-9]+/' + 'lib\\-dynload/cleo/foo.cpython\\-.*\\.so', + '/usr/lib/python3\\.[0-9]+/cleo/foo\\.py', + '/usr/lib/python3\\.[0-9]+/cleo/foo/__init__\\.py', + '/usr/lib/python3/dist\\-packages/cleo/__init__\\.py', + '/usr/lib/python3/dist\\-packages/cleo\\.py', + '/usr/lib/python3\\.[0-9]+/lib\\-dynload/cleo.cpython\\-.*\\.so', + '/usr/lib/python3\\.[0-9]+/cleo\\.py', + '/usr/lib/python3\\.[0-9]+/cleo/__init__\\.py'], + get_possible_python3_paths_for_python_object('cleo.foo'))