diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 123fa45..9c0a80c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,28 +13,34 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip flake8 cython - python -m pip install git+https://github.com/jelmer/buildlog-consultant - python setup.py develop - mkdir -p ~/.config/breezy/plugins - brz branch lp:brz-debian ~/.config/breezy/plugins/debian - - name: Style checks - run: | - python -m flake8 - - name: Typing checks - run: | - pip install -U mypy - python -m mypy ognibuild - if: "matrix.python-version != 'pypy3'" - - name: Test suite run - run: | - python -m unittest ognibuild.tests.test_suite - env: - PYTHONHASHSEED: random + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip flake8 cython + python setup.py develop + - name: Install Debian-specific dependencies + run: | + sudo apt install libapt-pkg-dev + python -m pip install wheel + python -m pip install git+https://salsa.debian.org/apt-team/python-apt + python -m pip install -e ".[debian]" + mkdir -p ~/.config/breezy/plugins + brz branch lp:brz-debian ~/.config/breezy/plugins/debian + if: "matrix.python-version != 'pypy3' && matrix.os == 'ubuntu-latest'" + - name: Style checks + run: | + python -m flake8 + - name: Typing checks + run: | + pip install -U mypy + python -m mypy ognibuild + if: "matrix.python-version != 'pypy3'" + - name: Test suite run + run: | + python -m unittest ognibuild.tests.test_suite + env: + PYTHONHASHSEED: random diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..f7c7bb4 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,17 @@ +Metadata-Version: 2.1 +Name: ognibuild +Version: 0.0.3 +Summary: Detect and run any build system +Home-page: https://jelmer.uk/code/ognibuild +Maintainer: Jelmer Vernooij +Maintainer-email: jelmer@jelmer.uk +License: GNU GPLv2 or later +Description: UNKNOWN +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Operating System :: POSIX +Provides-Extra: debian diff --git a/README.md b/README.md index 30d9861..2eb483d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -ognibuild -========= +# ognibuild Ognibuild is a simple wrapper with a common interface for invoking any kind of build tool. @@ -10,8 +9,7 @@ parameters. It can also detect and install missing dependencies. -Goals ------ +## Goals The goal of ognibuild is to provide a consistent CLI that can be used for any software package. It is mostly useful for automated building of @@ -20,8 +18,7 @@ large sets of diverse packages (e.g. different programming languages). It is not meant to expose all functionality that is present in the underlying build systems. To use that, invoke those build systems directly. -Usage ------ +## Usage Ognibuild has a number of subcommands: @@ -34,7 +31,53 @@ Ognibuild has a number of subcommands: It also includes a subcommand that can fix up the build dependencies for Debian packages, called deb-fix-build. -License -------- +## Status + +Ognibuild is functional, but sometimes rough around the edges. If you run into +issues (or lack of support for a particular ecosystem), please file a bug. + +### Supported Build Systems + +- Cabal +- Cargo +- Golang +- Gradle +- Make, including various makefile generators: + - autoconf/automake + - CMake + - Makefile.PL + - qmake +- Maven +- ninja, including ninja file generators: + - meson +- Node +- Octave +- Perl + - Module::Build::Tiny +- PHP Pear +- Python - setup.py/setup.cfg/pyproject.toml +- R +- Ruby gems +- Waf + +### Supported package repositories + +Package repositories are used to install missing dependencies. + +The following "native" repositories are supported: + +- pypi +- cpan +- hackage +- npm +- cargo +- cran +- golang\* + +As well one distribution repository: + +- apt + +## License Ognibuild is licensed under the GNU GPL, v2 or later. diff --git a/ognibuild.egg-info/PKG-INFO b/ognibuild.egg-info/PKG-INFO new file mode 100644 index 0000000..f7c7bb4 --- /dev/null +++ b/ognibuild.egg-info/PKG-INFO @@ -0,0 +1,17 @@ +Metadata-Version: 2.1 +Name: ognibuild +Version: 0.0.3 +Summary: Detect and run any build system +Home-page: https://jelmer.uk/code/ognibuild +Maintainer: Jelmer Vernooij +Maintainer-email: jelmer@jelmer.uk +License: GNU GPLv2 or later +Description: UNKNOWN +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+) +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Operating System :: POSIX +Provides-Extra: debian diff --git a/ognibuild.egg-info/SOURCES.txt b/ognibuild.egg-info/SOURCES.txt new file mode 100644 index 0000000..2067e75 --- /dev/null +++ b/ognibuild.egg-info/SOURCES.txt @@ -0,0 +1,52 @@ +.flake8 +.gitignore +AUTHORS +CODE_OF_CONDUCT.md +LICENSE +README.md +SECURITY.md +TODO +releaser.conf +setup.cfg +setup.py +.github/workflows/pythonpackage.yml +notes/architecture.md +notes/concepts.md +notes/roadmap.md +ognibuild/__init__.py +ognibuild/__main__.py +ognibuild/build.py +ognibuild/buildlog.py +ognibuild/buildsystem.py +ognibuild/clean.py +ognibuild/dist.py +ognibuild/dist_catcher.py +ognibuild/fix_build.py +ognibuild/fixers.py +ognibuild/info.py +ognibuild/install.py +ognibuild/outputs.py +ognibuild/requirements.py +ognibuild/test.py +ognibuild/vcs.py +ognibuild.egg-info/PKG-INFO +ognibuild.egg-info/SOURCES.txt +ognibuild.egg-info/dependency_links.txt +ognibuild.egg-info/entry_points.txt +ognibuild.egg-info/requires.txt +ognibuild.egg-info/top_level.txt +ognibuild/debian/__init__.py +ognibuild/debian/apt.py +ognibuild/debian/build.py +ognibuild/debian/build_deps.py +ognibuild/debian/file_search.py +ognibuild/debian/fix_build.py +ognibuild/debian/udd.py +ognibuild/resolver/__init__.py +ognibuild/resolver/apt.py +ognibuild/session/__init__.py +ognibuild/session/plain.py +ognibuild/session/schroot.py +ognibuild/tests/__init__.py +ognibuild/tests/test_debian_build.py +ognibuild/tests/test_debian_fix_build.py \ No newline at end of file diff --git a/ognibuild.egg-info/dependency_links.txt b/ognibuild.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ognibuild.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/ognibuild.egg-info/entry_points.txt b/ognibuild.egg-info/entry_points.txt new file mode 100644 index 0000000..dc61857 --- /dev/null +++ b/ognibuild.egg-info/entry_points.txt @@ -0,0 +1,4 @@ +[console_scripts] +deb-fix-build = ognibuild.debian.fix_build:main +ogni = ognibuild.__main__:main + diff --git a/ognibuild.egg-info/requires.txt b/ognibuild.egg-info/requires.txt new file mode 100644 index 0000000..6bafe7d --- /dev/null +++ b/ognibuild.egg-info/requires.txt @@ -0,0 +1,8 @@ +breezy +buildlog-consultant>=0.0.4 +requirements-parser + +[debian] +debmutate +python_apt +python_debian diff --git a/ognibuild.egg-info/top_level.txt b/ognibuild.egg-info/top_level.txt new file mode 100644 index 0000000..7e9084f --- /dev/null +++ b/ognibuild.egg-info/top_level.txt @@ -0,0 +1 @@ +ognibuild diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 6e210c2..ab42dbb 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -20,6 +20,12 @@ import os import stat +__version__ = (0, 0, 3) + + +USER_AGENT = "Ognibuild" + + class DetailedFailure(Exception): def __init__(self, retcode, argv, error): self.retcode = retcode @@ -28,12 +34,22 @@ class DetailedFailure(Exception): class UnidentifiedError(Exception): + """An unidentified error.""" + def __init__(self, retcode, argv, lines, secondary=None): self.retcode = retcode self.argv = argv self.lines = lines self.secondary = secondary + def __repr__(self): + return "<%s(%r, %r, ..., secondary=%r)>" % ( + type(self).__name__, + self.retcode, + self.argv, + self.secondary, + ) + def shebang_binary(p): if not (os.stat(p).st_mode & stat.S_IEXEC): @@ -42,7 +58,7 @@ def shebang_binary(p): firstline = f.readline() if not firstline.startswith(b"#!"): return None - args = firstline[2:].split(b" ") + args = firstline[2:].strip().split(b" ") if args[0] in (b"/usr/bin/env", b"env"): return os.path.basename(args[1].decode()).strip() return os.path.basename(args[0].decode()).strip() diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index a889962..8f82ca2 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -20,12 +20,16 @@ import os import shlex import sys from . import UnidentifiedError, DetailedFailure -from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall +from .buildlog import ( + InstallFixer, + ExplainInstallFixer, + ExplainInstall, + install_missing_reqs, +) from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, native_resolvers, - UnsatisfiedRequirements, ) from .resolver.apt import AptResolver @@ -35,8 +39,7 @@ def display_explain_commands(commands): for command, reqs in commands: if isinstance(command, list): command = shlex.join(command) - logging.info( - ' %s (to install %s)', command, ', '.join(map(str, reqs))) + logging.info(" %s (to install %s)", command, ", ".join(map(str, reqs))) def get_necessary_declared_requirements(resolver, requirements, stages): @@ -47,12 +50,14 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(session, resolver, buildsystems, stages, explain=False): +def install_necessary_declared_requirements( + session, resolver, fixers, buildsystems, stages, explain=False +): relevant = [] declared_reqs = [] for buildsystem in buildsystems: try: - declared_reqs.extend(buildsystem.get_declared_dependencies()) + declared_reqs.extend(buildsystem.get_declared_dependencies(session, fixers)) except NotImplementedError: logging.warning( "Unable to determine declared dependencies from %r", buildsystem @@ -60,21 +65,8 @@ def install_necessary_declared_requirements(session, resolver, buildsystems, sta relevant.extend( get_necessary_declared_requirements(resolver, declared_reqs, stages) ) - missing = [] - for req in relevant: - try: - if not req.met(session): - missing.append(req) - except NotImplementedError: - missing.append(req) - if missing: - if explain: - commands = resolver.explain(missing) - if not commands: - raise UnsatisfiedRequirements(missing) - raise ExplainInstall(commands) - else: - resolver.install(missing) + + install_missing_reqs(session, resolver, relevant, explain=explain) # Types of dependencies: @@ -154,35 +146,46 @@ def main(): # noqa: C901 session = PlainSession() with session: + logging.info("Preparing directory %s", args.directory) + external_dir, internal_dir = session.setup_from_directory(args.directory) + session.chdir(internal_dir) + os.chdir(external_dir) if args.resolve == "apt": resolver = AptResolver.from_session(session) elif args.resolve == "native": - resolver = native_resolvers(session) + resolver = native_resolvers(session, user_local=args.user) elif args.resolve == "auto": - resolver = auto_resolver(session) + resolver = auto_resolver(session, explain=args.explain) logging.info("Using requirement resolver: %s", resolver) - os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) - logging.info( - "Detected buildsystems: %s", ', '.join(map(str, bss))) + logging.info("Detected buildsystems: %s", ", ".join(map(str, bss))) + fixers = determine_fixers(session, resolver, explain=args.explain) if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: logging.info("Checking that declared requirements are present") try: install_necessary_declared_requirements( - session, resolver, bss, stages, explain=args.explain) + session, resolver, fixers, bss, stages, explain=args.explain + ) except ExplainInstall as e: display_explain_commands(e.commands) return 1 - fixers = determine_fixers(session, resolver, explain=args.explain) if args.subcommand == "dist": - from .dist import run_dist + from .dist import run_dist, DistNoTarball - run_dist( - session=session, buildsystems=bss, resolver=resolver, fixers=fixers - ) + try: + run_dist( + session=session, + buildsystems=bss, + resolver=resolver, + fixers=fixers, + target_directory=".", + ) + except DistNoTarball: + logging.fatal('No tarball created.') + return 1 if args.subcommand == "build": from .build import run_build diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index bb596b8..fabb279 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -34,10 +34,14 @@ from buildlog_consultant.common import ( MissingPerlModule, MissingXmlEntity, MissingJDKFile, + MissingJDK, + MissingJRE, MissingNodeModule, + MissingNodePackage, MissingPhpClass, MissingRubyGem, MissingLibrary, + MissingSetupPyCommand, MissingJavaClass, MissingCSharpCompiler, MissingRPackage, @@ -46,10 +50,16 @@ from buildlog_consultant.common import ( MissingValaPackage, MissingXfceDependency, MissingHaskellDependencies, + MissingVagueDependency, DhAddonLoadFailure, MissingMavenArtifacts, GnomeCommonMissing, MissingGnomeCommonDependency, + UnknownCertificateAuthority, + CMakeFilesMissing, + MissingLibtool, + MissingQt, + MissingX11, ) from .fix_build import BuildFixer @@ -71,15 +81,24 @@ from .requirements import ( XmlEntityRequirement, SprocketsFileRequirement, JavaClassRequirement, + CMakefileRequirement, HaskellPackageRequirement, MavenArtifactRequirement, GnomeCommonRequirement, JDKFileRequirement, + JDKRequirement, + JRERequirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, + CertificateAuthorityRequirement, + NodeModuleRequirement, + QTRequirement, + X11Requirement, + LibtoolRequirement, + VagueDependencyRequirement, ) from .resolver import UnsatisfiedRequirements @@ -108,7 +127,11 @@ def problem_to_upstream_requirement(problem): # noqa: C901 elif isinstance(problem, MissingRPackage): return RPackageRequirement(problem.package, problem.minimum_version) elif isinstance(problem, MissingNodeModule): - return NodePackageRequirement(problem.module) + return NodeModuleRequirement(problem.module) + elif isinstance(problem, MissingNodePackage): + return NodePackageRequirement(problem.package) + elif isinstance(problem, MissingVagueDependency): + return VagueDependencyRequirement(problem.name, minimum_version=problem.minimum_version) elif isinstance(problem, MissingLibrary): return LibraryRequirement(problem.library) elif isinstance(problem, MissingRubyFile): @@ -119,16 +142,37 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return SprocketsFileRequirement(problem.content_type, problem.name) elif isinstance(problem, MissingJavaClass): return JavaClassRequirement(problem.classname) + elif isinstance(problem, CMakeFilesMissing): + return [CMakefileRequirement(filename) for filename in problem.filenames] elif isinstance(problem, MissingHaskellDependencies): return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps] elif isinstance(problem, MissingMavenArtifacts): - return [MavenArtifactRequirement(artifact) for artifact in problem.artifacts] + return [ + MavenArtifactRequirement.from_str(artifact) + for artifact in problem.artifacts + ] elif isinstance(problem, MissingCSharpCompiler): return BinaryRequirement("msc") elif isinstance(problem, GnomeCommonMissing): return GnomeCommonRequirement() elif isinstance(problem, MissingJDKFile): return JDKFileRequirement(problem.jdk_path, problem.filename) + elif isinstance(problem, MissingJDK): + return JDKRequirement() + elif isinstance(problem, MissingJRE): + return JRERequirement() + elif isinstance(problem, MissingQt): + return QTRequirement() + elif isinstance(problem, MissingX11): + return X11Requirement() + elif isinstance(problem, MissingLibtool): + return LibtoolRequirement() + elif isinstance(problem, UnknownCertificateAuthority): + return CertificateAuthorityRequirement(problem.url) + elif isinstance(problem, MissingSetupPyCommand): + if problem.command == "test": + return PythonPackageRequirement("setuptools") + return None elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": return BinaryRequirement("glib-gettextize") @@ -159,7 +203,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 ) elif isinstance(problem, MissingPythonDistribution): return PythonPackageRequirement( - problem.module, + problem.distribution, python_version=problem.python_version, minimum_version=problem.minimum_version, ) @@ -181,7 +225,7 @@ class InstallFixer(BuildFixer): req = problem_to_upstream_requirement(error) return req is not None - def fix(self, error, context): + def fix(self, error, phase): reqs = problem_to_upstream_requirement(error) if reqs is None: return False @@ -197,7 +241,6 @@ class InstallFixer(BuildFixer): class ExplainInstall(Exception): - def __init__(self, commands): self.commands = commands @@ -216,7 +259,7 @@ class ExplainInstallFixer(BuildFixer): req = problem_to_upstream_requirement(error) return req is not None - def fix(self, error, context): + def fix(self, error, phase): reqs = problem_to_upstream_requirement(error) if reqs is None: return False @@ -228,3 +271,23 @@ class ExplainInstallFixer(BuildFixer): if not explanations: return False raise ExplainInstall(explanations) + + +def install_missing_reqs(session, resolver, reqs, explain=False): + if not reqs: + return + missing = [] + for req in reqs: + try: + if not req.met(session): + missing.append(req) + except NotImplementedError: + missing.append(req) + if missing: + if explain: + commands = resolver.explain(missing) + if not commands: + raise UnsatisfiedRequirements(missing) + raise ExplainInstall(commands) + else: + resolver.install(missing) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 2b47cc0..0776e57 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,13 +20,16 @@ import logging import os import re -from typing import Optional +import shlex +from typing import Optional, Tuple import warnings from . import shebang_binary, UnidentifiedError +from .dist_catcher import DistCatcher from .outputs import ( BinaryOutput, PythonPackageOutput, + RPackageOutput, ) from .requirements import ( BinaryRequirement, @@ -34,8 +37,22 @@ from .requirements import ( PerlModuleRequirement, NodePackageRequirement, CargoCrateRequirement, + RPackageRequirement, + OctavePackageRequirement, + PhpPackageRequirement, + MavenArtifactRequirement, + GoRequirement, + GoPackageRequirement, ) from .fix_build import run_with_build_fixers +from .session import which + + +def guaranteed_which(session, resolver, name): + path = which(session, name) + if not path: + resolver.install([BinaryRequirement(name)]) + return which(session, name) class NoBuildToolsFound(Exception): @@ -58,7 +75,9 @@ class BuildSystem(object): def __str__(self): return self.name - def dist(self, session, resolver, fixers, quiet=False): + def dist( + self, session, resolver, fixers, target_directory: str, quiet=False + ) -> str: raise NotImplementedError(self.dist) def test(self, session, resolver, fixers): @@ -73,12 +92,27 @@ class BuildSystem(object): def install(self, session, resolver, fixers, install_target): raise NotImplementedError(self.install) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): raise NotImplementedError(self.get_declared_dependencies) - def get_declared_outputs(self): + def get_declared_outputs(self, session, fixers=None): raise NotImplementedError(self.get_declared_outputs) + @classmethod + def probe(cls, path): + return None + + +def xmlparse_simplify_namespaces(path, namespaces): + import xml.etree.ElementTree as ET + + namespaces = ["{%s}" % ns for ns in namespaces] + tree = ET.iterparse(path) + for _, el in tree: + for namespace in namespaces: + el.tag = el.tag.replace(namespace, "") + return tree.root + class Pear(BuildSystem): @@ -87,28 +121,61 @@ class Pear(BuildSystem): def __init__(self, path): self.path = path - def setup(self, resolver): - resolver.install([BinaryRequirement("pear")]) - - def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "package"], fixers) + def dist(self, session, resolver, fixers, target_directory: str, quiet=False): + with DistCatcher([session.external_path(".")]) as dc: + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "package"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "run-tests"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "run-tests"], fixers) def build(self, session, resolver, fixers): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "build", self.path], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "build", self.path], fixers) def clean(self, session, resolver, fixers): self.setup(resolver) # TODO def install(self, session, resolver, fixers, install_target): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "install", self.path], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "install", self.path], fixers) + + def get_declared_dependencies(self, session, fixers=None): + path = os.path.join(self.path, "package.xml") + import xml.etree.ElementTree as ET + + try: + root = xmlparse_simplify_namespaces( + path, + [ + "http://pear.php.net/dtd/package-2.0", + "http://pear.php.net/dtd/package-2.1", + ], + ) + except ET.ParseError as e: + logging.warning("Unable to parse package.xml: %s", e) + return + assert root.tag == "package", "root tag is %r" % root.tag + dependencies_tag = root.find("dependencies") + if dependencies_tag is not None: + required_tag = root.find("dependencies") + if required_tag is not None: + for package_tag in root.findall("package"): + name = package_tag.find("name").text + min_tag = package_tag.find("min") + max_tag = package_tag.find("max") + channel_tag = package_tag.find("channel") + yield "core", PhpPackageRequirement( + name, + channel=(channel_tag.text if channel_tag else None), + min_version=(min_tag.text if min_tag else None), + max_version=(max_tag.text if max_tag else None), + ) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "package.xml")): + logging.debug("Found package.xml, assuming pear package.") + return cls(os.path.join(path, "package.xml")) # run_setup, but setting __name__ @@ -127,6 +194,8 @@ def run_setup(script_name, script_args=None, stop_after="run"): save_argv = sys.argv.copy() g = {"__file__": script_name, "__name__": "__main__"} try: + old_cwd = os.getcwd() + os.chdir(os.path.dirname(script_name)) try: sys.argv[0] = script_name if script_args is not None: @@ -134,6 +203,7 @@ def run_setup(script_name, script_args=None, stop_after="run"): with open(script_name, "rb") as f: exec(f.read(), g) finally: + os.chdir(old_cwd) sys.argv = save_argv core._setup_stop_after = None except SystemExit: @@ -141,141 +211,513 @@ def run_setup(script_name, script_args=None, stop_after="run"): # (ie. error)? pass - if core._setup_distribution is None: - raise RuntimeError( - ( - "'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?" - ) - % script_name - ) - return core._setup_distribution +_setup_wrapper = """\ +import distutils +from distutils import core +import sys + +script_name = %(script_name)s + +save_argv = sys.argv.copy() +g = {"__file__": script_name, "__name__": "__main__"} +try: + core._setup_stop_after = "init" + sys.argv[0] = script_name + with open(script_name, "rb") as f: + exec(f.read(), g) +except SystemExit: + # Hmm, should we do something if exiting with a non-zero code + # (ie. error)? + pass + +if core._setup_distribution is None: + raise RuntimeError( + ( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" + ) + % script_name + ) + +d = core._setup_distribution +r = { + 'name': d.name, + 'setup_requires': getattr(d, "setup_requires", []), + 'install_requires': getattr(d, "install_requires", []), + 'tests_require': getattr(d, "tests_require", []) or [], + 'scripts': getattr(d, "scripts", []) or [], + 'entry_points': getattr(d, "entry_points", None) or {}, + 'packages': getattr(d, "packages", []) or [], + 'requires': d.get_requires() or [], + } +import os +import json +with open(%(output_path)s, 'w') as f: + json.dump(r, f) +""" + + class SetupPy(BuildSystem): name = "setup.py" + DEFAULT_PYTHON = "python3" def __init__(self, path): self.path = path - # TODO(jelmer): Perhaps run this in session, so we can install - # missing dependencies? + if os.path.exists(os.path.join(self.path, "setup.py")): + self.has_setup_py = True + else: + self.has_setup_py = False + try: - self.result = run_setup(os.path.abspath(path), stop_after="init") + self.config = self.load_setup_cfg() + except FileNotFoundError: + self.config = None + + try: + self.pyproject = self.load_toml() + except FileNotFoundError: + self.pyproject = None + self.build_backend = None + else: + self.build_backend = self.pyproject.get("build-system", {}).get( + "build-backend" + ) + + def load_toml(self): + import toml + + with open(os.path.join(self.path, "pyproject.toml"), "r") as pf: + return toml.load(pf) + + def load_setup_cfg(self): + from setuptools.config import read_configuration + + p = os.path.join(self.path, "setup.cfg") + if os.path.exists(p): + return read_configuration(p) + raise FileNotFoundError(p) + + def _extract_setup(self, session=None, fixers=None): + if not self.has_setup_py: + return None + if session is None: + return self._extract_setup_direct() + else: + return self._extract_setup_in_session(session, fixers) + + def _extract_setup_direct(self): + p = os.path.join(self.path, "setup.py") + try: + d = run_setup(os.path.abspath(p), stop_after="init") except RuntimeError as e: logging.warning("Unable to load setup.py metadata: %s", e) - self.result = None + return None + if d is None: + logging.warning( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p) + ) + return None + + return { + "name": d.name, + "setup_requires": getattr(d, "setup_requires", []), + "install_requires": getattr(d, "install_requires", []), + "tests_require": getattr(d, "tests_require", []) or [], + "scripts": getattr(d, "scripts", []), + "entry_points": getattr(d, "entry_points", None) or {}, + "packages": getattr(d, "packages", []), + "requires": d.get_requires() or [], + } + + def _extract_setup_in_session(self, session, fixers=None): + import tempfile + import json + + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) + if interpreter is None: + interpreter = self.DEFAULT_PYTHON + output_f = tempfile.NamedTemporaryFile( + dir=os.path.join(session.location, "tmp"), mode="w+t" + ) + with output_f: + # TODO(jelmer): Perhaps run this in session, so we can install + # missing dependencies? + argv = [ + interpreter, + "-c", + _setup_wrapper.replace("%(script_name)s", '"setup.py"').replace( + "%(output_path)s", + '"/' + os.path.relpath(output_f.name, session.location) + '"', + ), + ] + try: + if fixers is not None: + run_with_build_fixers(session, argv, fixers) + else: + session.check_call(argv, close_fds=False) + except RuntimeError as e: + logging.warning("Unable to load setup.py metadata: %s", e) + return None + output_f.seek(0) + return json.load(output_f) def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def setup(self, resolver): - with open(self.path, "r") as f: - setup_py_contents = f.read() - try: - with open("setup.cfg", "r") as f: - setup_cfg_contents = f.read() - except FileNotFoundError: - setup_cfg_contents = "" - if "setuptools" in setup_py_contents: - logging.debug("Reference to setuptools found, installing.") - resolver.install([PythonPackageRequirement("setuptools")]) - if ( - "setuptools_scm" in setup_py_contents - or "setuptools_scm" in setup_cfg_contents - ): - logging.debug("Reference to setuptools-scm found, installing.") - resolver.install( - [ - PythonPackageRequirement("setuptools_scm"), - BinaryRequirement("git"), - BinaryRequirement("hg"), - ] - ) - - # TODO(jelmer): Install setup_requires - def test(self, session, resolver, fixers): - self.setup(resolver) - self._run_setup(session, resolver, ["test"], fixers) + if os.path.exists(os.path.join(self.path, "tox.ini")): + run_with_build_fixers(session, ["tox"], fixers) + elif self.pyproject: + run_with_build_fixers( + session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers + ) + elif self.has_setup_py: + # Pre-emptively insall setuptools, since distutils doesn't provide + # a 'test' subcommand and some packages fall back to distutils + # if setuptools is not available. + setuptools_req = PythonPackageRequirement("setuptools") + if not setuptools_req.met(session): + resolver.install([setuptools_req]) + self._run_setup(session, resolver, ["test"], fixers) + else: + raise NotImplementedError def build(self, session, resolver, fixers): - self.setup(resolver) - self._run_setup(session, resolver, ["build"], fixers) + if self.has_setup_py: + self._run_setup(session, resolver, ["build"], fixers) + else: + raise NotImplementedError - def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) - preargs = [] - if quiet: - preargs.append("--quiet") - self._run_setup(session, resolver, preargs + ["sdist"], fixers) + def dist(self, session, resolver, fixers, target_directory, quiet=False): + # TODO(jelmer): Look at self.build_backend + if self.has_setup_py: + preargs = [] + if quiet: + preargs.append("--quiet") + # Preemptively install setuptools since some packages fail in + # some way without it. + setuptools_req = PythonPackageRequirement("setuptools") + if not setuptools_req.met(session): + resolver.install([setuptools_req]) + with DistCatcher([session.external_path("dist")]) as dc: + self._run_setup(session, resolver, preargs + ["sdist"], fixers) + return dc.copy_single(target_directory) + elif self.pyproject: + with DistCatcher([session.external_path("dist")]) as dc: + run_with_build_fixers( + session, + [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], + fixers, + ) + return dc.copy_single(target_directory) + raise AssertionError("no setup.py or pyproject.toml") def clean(self, session, resolver, fixers): - self.setup(resolver) - self._run_setup(session, resolver, ["clean"], fixers) + if self.has_setup_py: + self._run_setup(session, resolver, ["clean"], fixers) + else: + raise NotImplementedError def install(self, session, resolver, fixers, install_target): - self.setup(resolver) - extra_args = [] - if install_target.user: - extra_args.append("--user") - self._run_setup(session, resolver, ["install"] + extra_args, fixers) + if self.has_setup_py: + extra_args = [] + if install_target.user: + extra_args.append("--user") + self._run_setup(session, resolver, ["install"] + extra_args, fixers) + else: + raise NotImplementedError def _run_setup(self, session, resolver, args, fixers): - interpreter = shebang_binary("setup.py") - if interpreter is not None: - resolver.install([BinaryRequirement(interpreter)]) - run_with_build_fixers(session, ["./setup.py"] + args, fixers) - else: - # Just assume it's Python 3 - resolver.install([BinaryRequirement("python3")]) - run_with_build_fixers(session, ["python3", "./setup.py"] + args, fixers) + from .buildlog import install_missing_reqs - def get_declared_dependencies(self): - if self.result is None: - raise NotImplementedError - for require in self.result.get_requires(): - yield "core", PythonPackageRequirement.from_requirement_str(require) - # Not present for distutils-only packages - if getattr(self.result, "install_requires", []): - for require in self.result.install_requires: + # Install the setup_requires beforehand, since otherwise + # setuptools might fetch eggs instead of our preferred resolver. + install_missing_reqs(session, resolver, list(self._setup_requires())) + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) + if interpreter is None: + interpreter = self.DEFAULT_PYTHON + argv = [interpreter, "./setup.py"] + args + env = {} + # Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment + if "SETUPTOOLS_SCM_PRETEND_VERSION" in os.environ: + env["SETUPTOOLS_SCM_PRETEND_VERSION"] = os.environ[ + "SETUPTOOLS_SCM_PRETEND_VERSION" + ] + run_with_build_fixers(session, argv, fixers, env=env) + + def _setup_requires(self): + if self.pyproject: + if "build-system" in self.pyproject: + for require in self.pyproject["build-system"].get("requires", []): + yield PythonPackageRequirement.from_requirement_str(require) + if self.config: + options = self.config.get("options", {}) + for require in options.get("setup_requires", []): + yield PythonPackageRequirement.from_requirement_str(require) + + def get_declared_dependencies(self, session, fixers=None): + distribution = self._extract_setup(session, fixers) + if distribution is not None: + for require in distribution["requires"]: yield "core", PythonPackageRequirement.from_requirement_str(require) - # Not present for distutils-only packages - if getattr(self.result, "tests_require", []): - for require in self.result.tests_require: + # Not present for distutils-only packages + for require in distribution["setup_requires"]: + yield "build", PythonPackageRequirement.from_requirement_str(require) + # Not present for distutils-only packages + for require in distribution["install_requires"]: + yield "core", PythonPackageRequirement.from_requirement_str(require) + # Not present for distutils-only packages + for require in distribution["tests_require"]: yield "test", PythonPackageRequirement.from_requirement_str(require) + if self.pyproject: + if "build-system" in self.pyproject: + for require in self.pyproject["build-system"].get("requires", []): + yield "build", PythonPackageRequirement.from_requirement_str( + require + ) + if self.config: + options = self.config.get("options", {}) + for require in options.get("setup_requires", []): + yield "build", PythonPackageRequirement.from_requirement_str(require) + for require in options.get("install_requires", []): + yield "core", PythonPackageRequirement.from_requirement_str(require) - def get_declared_outputs(self): - if self.result is None: - raise NotImplementedError - for script in self.result.scripts or []: - yield BinaryOutput(os.path.basename(script)) - entry_points = getattr(self.result, "entry_points", None) or {} - for script in entry_points.get("console_scripts", []): - yield BinaryOutput(script.split("=")[0]) - for package in self.result.packages or []: + def get_declared_outputs(self, session, fixers=None): + distribution = self._extract_setup(session, fixers) + all_packages = set() + if distribution is not None: + for script in distribution["scripts"]: + yield BinaryOutput(os.path.basename(script)) + for script in distribution["entry_points"].get("console_scripts", []): + yield BinaryOutput(script.split("=")[0]) + all_packages.update(distribution["packages"]) + if self.config: + options = self.config.get("options", {}) + all_packages.update(options.get("packages", [])) + for script in options.get("scripts", []): + yield BinaryOutput(os.path.basename(script)) + for script in options.get("entry_points", {}).get("console_scripts", []): + yield BinaryOutput(script.split("=")[0]) + + packages = set() + for package in sorted(all_packages): + pts = package.split(".") + b = [] + for e in pts: + b.append(e) + if ".".join(b) in packages: + break + else: + packages.add(package) + for package in packages: yield PythonPackageOutput(package, python_version="cpython3") + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "setup.py")): + logging.debug("Found setup.py, assuming python project.") + return cls(path) + if os.path.exists(os.path.join(path, "setup.cfg")): + logging.debug("Found setup.py, assuming python project.") + return cls(path) + if os.path.exists(os.path.join(path, "pyproject.toml")): + logging.debug("Found pyproject.toml, assuming python project.") + return cls(path) + + +class Octave(BuildSystem): + + name = "octave" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def exists(cls, path): + if not os.path.exists(os.path.join(path, "DESCRIPTION")): + return False + # Urgh, isn't there a better way to see if this is an octave package? + for entry in os.scandir(path): + if entry.name.endswith(".m"): + return True + if not entry.is_dir(): + continue + for subentry in os.scandir(entry.path): + if subentry.name.endswith(".m"): + return True + return False + + @classmethod + def probe(cls, path): + if cls.exists(path): + logging.debug("Found DESCRIPTION, assuming octave package.") + return cls(path) + + def _read_description(self): + path = os.path.join(self.path, "DESCRIPTION") + from email.parser import BytesParser + + with open(path, "rb") as f: + return BytesParser().parse(f) + + def get_declared_dependencies(self, session, fixers=None): + def parse_list(t): + return [s.strip() for s in t.split(",") if s.strip()] + + description = self._read_description() + if "Depends" in description: + for s in parse_list(description["Depends"]): + yield "build", OctavePackageRequirement.from_str(s) + class Gradle(BuildSystem): name = "gradle" + def __init__(self, path, executable="gradle"): + self.path = path + self.executable = executable + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def exists(cls, path): + return os.path.exists(os.path.join(path, "build.gradle")) or os.path.exists( + os.path.join(path, "build.gradle.kts") + ) + + @classmethod + def from_path(cls, path): + if os.path.exists(os.path.join(path, "gradlew")): + return cls(path, "./gradlew") + return cls(path) + + @classmethod + def probe(cls, path): + if cls.exists(path): + logging.debug("Found build.gradle, assuming gradle package.") + return cls.from_path(path) + + def setup(self, session, resolver): + if not self.executable.startswith("./"): + binary_req = BinaryRequirement(self.executable) + if not binary_req.met(session): + resolver.install([binary_req]) + + def _run(self, session, resolver, task, args, fixers): + self.setup(session, resolver) + argv = [] + if self.executable.startswith("./") and ( + not os.access(os.path.join(self.path, self.executable), os.X_OK) + ): + argv.append("sh") + argv.extend([self.executable, task]) + argv.extend(args) + try: + run_with_build_fixers(session, argv, fixers) + except UnidentifiedError as e: + if any( + [ + re.match( + r"Task '" + task + r"' not found in root project '.*'\.", line + ) + for line in e.lines + ] + ): + raise NotImplementedError + raise + + def clean(self, session, resolver, fixers): + self._run(session, resolver, "clean", [], fixers) + + def build(self, session, resolver, fixers): + self._run(session, resolver, "build", [], fixers) + + def test(self, session, resolver, fixers): + self._run(session, resolver, "test", [], fixers) + + def dist(self, session, resolver, fixers, target_directory, quiet=False): + with DistCatcher([session.external_path(".")]) as dc: + self._run(session, resolver, "distTar", [], fixers) + return dc.copy_single(target_directory) + + def install(self, session, resolver, fixers, install_target): + raise NotImplementedError + # TODO(jelmer): installDist just creates files under build/install/... + self._run(session, resolver, "installDist", [], fixers) + + +class R(BuildSystem): + + # https://r-pkgs.org/description.html + + name = "R" + def __init__(self, path): self.path = path def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def clean(self, session, resolver, fixers): - run_with_build_fixers(session, ["gradle", "clean"], fixers) - def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["gradle", "build"], fixers) + pass + + def dist(self, session, resolver, fixers, target_directory, quiet=False): + r_path = guaranteed_which(session, resolver, "R") + with DistCatcher([session.external_path(".")]) as dc: + run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) + return dc.copy_single(target_directory) + + def install(self, session, resolver, fixers, install_target): + r_path = guaranteed_which(session, resolver, "R") + run_with_build_fixers(session, [r_path, "CMD", "INSTALL", "."], fixers) def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["gradle", "test"], fixers) + r_path = guaranteed_which(session, resolver, "R") + run_with_build_fixers(session, [r_path, "CMD", "check", "."], fixers) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "DESCRIPTION")) and os.path.exists( + os.path.join(path, "NAMESPACE") + ): + return cls(path) + + def _read_description(self): + path = os.path.join(self.path, "DESCRIPTION") + from email.parser import BytesParser + + with open(path, "rb") as f: + return BytesParser().parse(f) + + def get_declared_dependencies(self, session, fixers=None): + def parse_list(t): + return [s.strip() for s in t.split(",") if s.strip()] + + description = self._read_description() + if "Suggests" in description: + for s in parse_list(description["Suggests"]): + yield "build", RPackageRequirement.from_str(s) + if "Depends" in description: + for s in parse_list(description["Depends"]): + yield "build", RPackageRequirement.from_str(s) + if "Imports" in description: + for s in parse_list(description["Imports"]): + yield "build", RPackageRequirement.from_str(s) + + def get_declared_outputs(self, session, fixers=None): + description = self._read_description() + if "Package" in description: + yield RPackageOutput(description["Package"]) class Meson(BuildSystem): @@ -289,9 +731,8 @@ class Meson(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def _setup(self, session, fixers): - if session.exists("build"): - return - session.mkdir("build") + if not session.exists("build"): + session.check_call(["mkdir", "build"]) run_with_build_fixers(session, ["meson", "setup", "build"], fixers) def clean(self, session, resolver, fixers): @@ -302,6 +743,12 @@ class Meson(BuildSystem): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build"], fixers) + def dist(self, session, resolver, fixers, target_directory, quiet=False): + self._setup(session, fixers) + with DistCatcher([session.external_path("build/meson-dist")]) as dc: + run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) + return dc.copy_single(target_directory) + def test(self, session, resolver, fixers): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build", "test"], fixers) @@ -310,57 +757,11 @@ class Meson(BuildSystem): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build", "install"], fixers) - -class PyProject(BuildSystem): - - name = "pyproject" - - def __init__(self, path): - self.path = path - self.pyproject = self.load_toml() - - def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.path) - - def load_toml(self): - import toml - - with open(self.path, "r") as pf: - return toml.load(pf) - - def dist(self, session, resolver, fixers, quiet=False): - if "poetry" in self.pyproject.get("tool", []): - logging.debug( - "Found pyproject.toml with poetry section, " "assuming poetry project." - ) - resolver.install( - [ - PythonPackageRequirement("venv"), - PythonPackageRequirement("poetry"), - ] - ) - session.check_call(["poetry", "build", "-f", "sdist"]) - return - raise AssertionError("no supported section in pyproject.toml") - - -class SetupCfg(BuildSystem): - - name = "setup.cfg" - - def __init__(self, path): - self.path = path - - def setup(self, resolver): - resolver.install( - [ - PythonPackageRequirement("pep517"), - ] - ) - - def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) - session.check_call(["python3", "-m", "pep517.build", "-s", "."]) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "meson.build")): + logging.debug("Found meson.build, assuming meson package.") + return Meson(os.path.join(path, "meson.build")) class Npm(BuildSystem): @@ -370,21 +771,64 @@ class Npm(BuildSystem): def __init__(self, path): import json + self.path = path + with open(path, "r") as f: self.package = json.load(f) - def get_declared_dependencies(self): + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def get_declared_dependencies(self, session, fixers=None): + if "dependencies" in self.package: + for name, unused_version in self.package["dependencies"].items(): + # TODO(jelmer): Look at version + yield "core", NodePackageRequirement(name) if "devDependencies" in self.package: for name, unused_version in self.package["devDependencies"].items(): # TODO(jelmer): Look at version - yield "dev", NodePackageRequirement(name) + yield "build", NodePackageRequirement(name) - def setup(self, resolver): - resolver.install([BinaryRequirement("npm")]) + def setup(self, session, resolver): + binary_req = BinaryRequirement("npm") + if not binary_req.met(session): + resolver.install([binary_req]) - def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) - run_with_build_fixers(session, ["npm", "pack"], fixers) + def dist(self, session, resolver, fixers, target_directory, quiet=False): + self.setup(session, resolver) + with DistCatcher([session.external_path(".")]) as dc: + run_with_build_fixers(session, ["npm", "pack"], fixers) + return dc.copy_single(target_directory) + + def test(self, session, resolver, fixers): + self.setup(session, resolver) + test_script = self.package["scripts"].get("test") + if test_script: + run_with_build_fixers(session, shlex.split(test_script), fixers) + else: + raise NotImplementedError + + def build(self, session, resolver, fixers): + self.setup(session, resolver) + build_script = self.package["scripts"].get("build") + if build_script: + run_with_build_fixers(session, shlex.split(build_script), fixers) + else: + raise NotImplementedError + + def clean(self, session, resolver, fixers): + self.setup(session, resolver) + clean_script = self.package["scripts"].get("clean") + if clean_script: + run_with_build_fixers(session, shlex.split(clean_script), fixers) + else: + raise NotImplementedError + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "package.json")): + logging.debug("Found package.json, assuming node package.") + return cls(os.path.join(path, "package.json")) class Waf(BuildSystem): @@ -395,16 +839,26 @@ class Waf(BuildSystem): self.path = path def setup(self, session, resolver, fixers): - resolver.install([BinaryRequirement("python3")]) + binary_req = BinaryRequirement("python3") + if not binary_req.met(session): + resolver.install([binary_req]) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) - run_with_build_fixers(session, ["./waf", "dist"], fixers) + with DistCatcher.default(session.external_path(".")) as dc: + run_with_build_fixers(session, ["./waf", "dist"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["./waf", "test"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "waf")): + logging.debug("Found waf, assuming waf package.") + return cls(os.path.join(path, "waf")) + class Gem(BuildSystem): @@ -413,30 +867,40 @@ class Gem(BuildSystem): def __init__(self, path): self.path = path - def setup(self, resolver): - resolver.install([BinaryRequirement("gem2deb")]) - - def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) + def dist(self, session, resolver, fixers, target_directory, quiet=False): gemfiles = [ entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") ] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") - run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) + with DistCatcher.default(session.external_path(".")) as dc: + run_with_build_fixers( + session, + [guaranteed_which(session, resolver, "gem2tgz"), gemfiles[0]], fixers) + return dc.copy_single(target_directory) + + @classmethod + def probe(cls, path): + gemfiles = [ + entry.path for entry in os.scandir(path) if entry.name.endswith(".gem") + ] + if gemfiles: + return cls(gemfiles[0]) -class DistInkt(BuildSystem): +class DistZilla(BuildSystem): + + name = "dist-zilla" + def __init__(self, path): self.path = path - self.name = "dist-zilla" self.dist_inkt_class = None - with open("dist.ini", "rb") as f: + with open(self.path, "rb") as f: for line in f: if not line.startswith(b";;"): continue try: - (key, value) = line[2:].split(b"=", 1) + (key, value) = line[2:].strip().split(b"=", 1) except ValueError: continue if key.strip() == b"class" and value.strip().startswith(b"'Dist::Inkt"): @@ -455,46 +919,102 @@ class DistInkt(BuildSystem): ] ) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(resolver) if self.name == "dist-inkt": - resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) - run_with_build_fixers(session, ["distinkt-dist"], fixers) + with DistCatcher.default(session.external_path(".")) as dc: + run_with_build_fixers(session, [guaranteed_which(session, resolver, "distinkt-dist")], fixers) + return dc.copy_single(target_directory) else: # Default to invoking Dist::Zilla - resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers(session, ["dzil", "build", "--in", ".."], fixers) + with DistCatcher.default(session.external_path(".")) as dc: + run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "build", "--tgz"], fixers) + return dc.copy_single(target_directory) + + def test(self, session, resolver, fixers): + self.setup(resolver) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "test"], fixers) + + def build(self, session, resolver, fixers): + self.setup(resolver) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "build"], fixers) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "dist.ini")) and not os.path.exists( + os.path.join(path, "Makefile.PL") + ): + return cls(os.path.join(path, "dist.ini")) + + def get_declared_dependencies(self, session, fixers=None): + lines = run_with_build_fixers(session, ["dzil", "authordeps"], fixers) + for entry in lines: + yield "build", PerlModuleRequirement(entry.strip()) + if os.path.exists(os.path.join(os.path.dirname(self.path), "cpanfile")): + yield from _declared_deps_from_cpanfile(session, fixers) + + +class RunTests(BuildSystem): + + name = "runtests" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "runtests.sh")): + return cls(path) + + def test(self, session, resolver, fixers): + if shebang_binary(os.path.join(self.path, "runtests.sh")) is not None: + run_with_build_fixers(session, ["./runtests.sh"], fixers) + else: + run_with_build_fixers(session, ["/bin/bash", "./runtests.sh"], fixers) + + +def _read_cpanfile(session, args, kind, fixers): + for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers): + yield kind, PerlModuleRequirement(line) + + +def _declared_deps_from_cpanfile(session, fixers): + yield from _read_cpanfile(session, ["--configure", "--build"], "build", fixers) + yield from _read_cpanfile(session, ["--test"], "test", fixers) class Make(BuildSystem): name = "make" + def __init__(self, path): + self.path = path + def __repr__(self): - return "%s()" % type(self).__name__ + return "%s(%r)" % (type(self).__name__, self.path) def setup(self, session, resolver, fixers): - resolver.install([BinaryRequirement("make")]) - def makefile_exists(): return any( [session.exists(p) for p in ["Makefile", "GNUmakefile", "makefile"]] ) if session.exists("Makefile.PL") and not makefile_exists(): - resolver.install([BinaryRequirement("perl")]) run_with_build_fixers(session, ["perl", "Makefile.PL"], fixers) if not makefile_exists() and not session.exists("configure"): if session.exists("autogen.sh"): - if shebang_binary("autogen.sh") is None: + if shebang_binary(os.path.join(self.path, "autogen.sh")) is None: run_with_build_fixers(session, ["/bin/sh", "./autogen.sh"], fixers) try: run_with_build_fixers(session, ["./autogen.sh"], fixers) except UnidentifiedError as e: if ( "Gnulib not yet bootstrapped; " - "run ./bootstrap instead.\n" in e.lines + "run ./bootstrap instead." in e.lines ): run_with_build_fixers(session, ["./bootstrap"], fixers) run_with_build_fixers(session, ["./autogen.sh"], fixers) @@ -502,19 +1022,16 @@ class Make(BuildSystem): raise elif session.exists("configure.ac") or session.exists("configure.in"): - resolver.install( - [ - BinaryRequirement("autoconf"), - BinaryRequirement("automake"), - BinaryRequirement("gettextize"), - BinaryRequirement("libtoolize"), - ] - ) run_with_build_fixers(session, ["autoreconf", "-i"], fixers) if not makefile_exists() and session.exists("configure"): run_with_build_fixers(session, ["./configure"], fixers) + if not makefile_exists() and any( + [n.name.endswith(".pro") for n in session.scandir(".")] + ): + run_with_build_fixers(session, ["qmake"], fixers) + def build(self, session, resolver, fixers): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "all"], fixers) @@ -531,69 +1048,70 @@ class Make(BuildSystem): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "install"], fixers) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) - try: - run_with_build_fixers(session, ["make", "dist"], fixers) - except UnidentifiedError as e: - if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: - pass - elif "make[1]: *** No rule to make target 'dist'. Stop.\n" in e.lines: - pass - elif ( - "Reconfigure the source tree " - "(via './config' or 'perl Configure'), please.\n" - ) in e.lines: - run_with_build_fixers(session, ["./config"], fixers) + with DistCatcher.default(session.external_path(".")) as dc: + try: run_with_build_fixers(session, ["make", "dist"], fixers) - elif ( - "Please try running 'make manifest' and then run " - "'make dist' again.\n" in e.lines - ): - run_with_build_fixers(session, ["make", "manifest"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - elif "Please run ./configure first\n" in e.lines: - run_with_build_fixers(session, ["./configure"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - elif any( - [ - re.match( - r"(Makefile|GNUmakefile|makefile):[0-9]+: " - r"\*\*\* Missing \'Make.inc\' " - r"Run \'./configure \[options\]\' and retry. Stop.\n", - line, - ) - for line in e.lines - ] - ): - run_with_build_fixers(session, ["./configure"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - elif any( - [ - re.match( - r"Problem opening MANIFEST: No such file or directory " - r"at .* line [0-9]+\.", - line, - ) - for line in e.lines - ] - ): - run_with_build_fixers(session, ["make", "manifest"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - else: - raise - else: - return + except UnidentifiedError as e: + if "make: *** No rule to make target 'dist'. Stop." in e.lines: + raise NotImplementedError + elif "make[1]: *** No rule to make target 'dist'. Stop." in e.lines: + raise NotImplementedError + elif ( + "Reconfigure the source tree " + "(via './config' or 'perl Configure'), please." + ) in e.lines: + run_with_build_fixers(session, ["./config"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif ( + "Please try running 'make manifest' and then run " + "'make dist' again." in e.lines + ): + run_with_build_fixers(session, ["make", "manifest"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif "Please run ./configure first" in e.lines: + run_with_build_fixers(session, ["./configure"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif any( + [ + re.match( + r"(Makefile|GNUmakefile|makefile):[0-9]+: " + r"\*\*\* Missing \'Make.inc\' " + r"Run \'./configure \[options\]\' and retry. Stop.", + line, + ) + for line in e.lines + ] + ): + run_with_build_fixers(session, ["./configure"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif any( + [ + re.match( + r"Problem opening MANIFEST: No such file or directory " + r"at .* line [0-9]+\.", + line, + ) + for line in e.lines + ] + ): + run_with_build_fixers(session, ["make", "manifest"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + else: + raise + return dc.copy_single(target_directory) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): + something = False # TODO(jelmer): Split out the perl-specific stuff? - if os.path.exists("META.yml"): + if os.path.exists(os.path.join(self.path, "META.yml")): # See http://module-build.sourceforge.net/META-spec-v1.4.html for # the specification of the format. import ruamel.yaml import ruamel.yaml.reader - with open("META.yml", "rb") as f: + with open(os.path.join(self.path, "META.yml"), "rb") as f: try: data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) except ruamel.yaml.reader.ReaderError as e: @@ -601,9 +1119,36 @@ class Make(BuildSystem): return for require in data.get("requires", []): yield "build", PerlModuleRequirement(require) - else: + something = True + if os.path.exists(os.path.join(self.path, "cpanfile")): + yield from _declared_deps_from_cpanfile(session, fixers) + something = True + if not something: raise NotImplementedError + @classmethod + def probe(cls, path): + if any( + [ + os.path.exists(os.path.join(path, p)) + for p in [ + "Makefile", + "GNUmakefile", + "makefile", + "Makefile.PL", + "CMakeLists.txt", + "autogen.sh", + "configure.ac", + "configure.in", + ] + ] + ): + return cls(path) + for n in os.scandir(path): + # qmake + if n.name.endswith(".pro"): + return cls(path) + class Cargo(BuildSystem): @@ -620,11 +1165,17 @@ class Cargo(BuildSystem): with open(path, "r") as f: self.cargo = load(f) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): if "dependencies" in self.cargo: for name, details in self.cargo["dependencies"].items(): - # TODO(jelmer): Look at details['features'], details['version'] - yield "build", CargoCrateRequirement(name) + if isinstance(details, str): + details = {"version": details} + # TODO(jelmer): Look at details['version'] + yield "build", CargoCrateRequirement( + name, + features=details.get("features", []), + version=details.get("version"), + ) def test(self, session, resolver, fixers): run_with_build_fixers(session, ["cargo", "test"], fixers) @@ -635,17 +1186,50 @@ class Cargo(BuildSystem): def build(self, session, resolver, fixers): run_with_build_fixers(session, ["cargo", "build"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "Cargo.toml")): + logging.debug("Found Cargo.toml, assuming rust cargo package.") + return Cargo(os.path.join(path, "Cargo.toml")) + + +def _parse_go_mod(f): + def readline(): + line = f.readline() + if not line: + return line + return line.split("//")[0] + "\n" + + line = readline() + while line: + parts = line.strip().split(" ") + if not parts or parts == [""]: + continue + if len(parts) == 2 and parts[1] == "(": + line = readline() + while line.strip() != ")": + yield [parts[0]] + list(line.strip().split(" ")) + line = readline() + if not line: + raise AssertionError("list of %s interrupted?" % parts[0]) + else: + yield parts + line = readline() + class Golang(BuildSystem): """Go builds.""" name = "golang" + def __init__(self, path): + self.path = path + def __repr__(self): return "%s()" % (type(self).__name__) def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["go", "test"], fixers) + run_with_build_fixers(session, ["go", "test", "./..."], fixers) def build(self, session, resolver, fixers): run_with_build_fixers(session, ["go", "build"], fixers) @@ -656,6 +1240,40 @@ class Golang(BuildSystem): def clean(self, session, resolver, fixers): session.check_call(["go", "clean"]) + def get_declared_dependencies(self, session, fixers=None): + go_mod_path = os.path.join(self.path, "go.mod") + if os.path.exists(go_mod_path): + with open(go_mod_path, "r") as f: + for parts in _parse_go_mod(f): + if parts[0] == "go": + yield "build", GoRequirement(parts[1]) + elif parts[0] == "require": + yield "build", GoPackageRequirement( + parts[1], parts[2].lstrip("v") if len(parts) > 2 else None + ) + elif parts[0] == "exclude": + pass # TODO(jelmer): Create conflicts? + elif parts[0] == "replace": + pass # TODO(jelmer): do.. something? + elif parts[0] == "module": + pass + else: + logging.warning("Unknown directive %s in go.mod", parts[0]) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "go.mod")): + return Golang(path) + if os.path.exists(os.path.join(path, "go.sum")): + return Golang(path) + for entry in os.scandir(path): + if entry.name.endswith(".go"): + return Golang(path) + if entry.is_dir(): + for entry in os.scandir(entry.path): + if entry.name.endswith(".go"): + return Golang(path) + class Maven(BuildSystem): @@ -664,6 +1282,49 @@ class Maven(BuildSystem): def __init__(self, path): self.path = path + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "pom.xml")): + logging.debug("Found pom.xml, assuming maven package.") + return cls(os.path.join(path, "pom.xml")) + + def test(self, session, resolver, fixers): + run_with_build_fixers(session, ["mvn", "test"], fixers) + + def clean(self, session, resolver, fixers): + run_with_build_fixers(session, ["mvn", "clean"], fixers) + + def install(self, session, resolver, fixers, install_target): + run_with_build_fixers(session, ["mvn", "install"], fixers) + + def build(self, session, resolver, fixers): + run_with_build_fixers(session, ["mvn", "compile"], fixers) + + def dist(self, session, resolver, fixers, target_directory, quiet=False): + # TODO(jelmer): 'mvn generate-sources' creates a jar in target/. + # is that what we need? + raise NotImplementedError + + def get_declared_dependencies(self, session, fixers=None): + import xml.etree.ElementTree as ET + + try: + root = xmlparse_simplify_namespaces( + self.path, ["http://maven.apache.org/POM/4.0.0"] + ) + except ET.ParseError as e: + logging.warning("Unable to parse package.xml: %s", e) + return + assert root.tag == "project", "root tag is %r" % root.tag + deps_tag = root.find("dependencies") + if deps_tag: + for dep in deps_tag.findall("dependency"): + yield "core", MavenArtifactRequirement( + dep.find("groupId").text, + dep.find("artifactId").text, + dep.find("version").text, + ) + class Cabal(BuildSystem): @@ -679,7 +1340,7 @@ class Cabal(BuildSystem): try: run_with_build_fixers(session, ["runhaskell", "Setup.hs"] + args, fixers) except UnidentifiedError as e: - if "Run the 'configure' command first.\n" in e.lines: + if "Run the 'configure' command first." in e.lines: run_with_build_fixers( session, ["runhaskell", "Setup.hs", "configure"], fixers ) @@ -692,104 +1353,122 @@ class Cabal(BuildSystem): def test(self, session, resolver, fixers): self._run(session, ["test"], fixers) - def dist(self, session, resolver, fixers, quiet=False): - self._run(session, ["sdist"], fixers) - - -def detect_buildsystems(path, trust_package=False): # noqa: C901 - """Detect build systems.""" - if os.path.exists(os.path.join(path, "package.xml")): - logging.debug("Found package.xml, assuming pear package.") - yield Pear("package.xml") - - if os.path.exists(os.path.join(path, "setup.py")): - logging.debug("Found setup.py, assuming python project.") - yield SetupPy("setup.py") - elif os.path.exists(os.path.join(path, "pyproject.toml")): - logging.debug("Found pyproject.toml, assuming python project.") - yield PyProject("pyproject.toml") - elif os.path.exists(os.path.join(path, "setup.cfg")): - logging.debug("Found setup.cfg, assuming python project.") - yield SetupCfg("setup.cfg") - - if os.path.exists(os.path.join(path, "package.json")): - logging.debug("Found package.json, assuming node package.") - yield Npm("package.json") - - if os.path.exists(os.path.join(path, "waf")): - logging.debug("Found waf, assuming waf package.") - yield Waf("waf") - - if os.path.exists(os.path.join(path, "Cargo.toml")): - logging.debug("Found Cargo.toml, assuming rust cargo package.") - yield Cargo("Cargo.toml") - - if os.path.exists(os.path.join(path, "build.gradle")): - logging.debug("Found build.gradle, assuming gradle package.") - yield Gradle("build.gradle") - - if os.path.exists(os.path.join(path, "meson.build")): - logging.debug("Found meson.build, assuming meson package.") - yield Meson("meson.build") - - if os.path.exists(os.path.join(path, "Setup.hs")): - logging.debug("Found Setup.hs, assuming haskell package.") - yield Cabal("Setup.hs") - - if os.path.exists(os.path.join(path, "pom.xml")): - logging.debug("Found pom.xml, assuming maven package.") - yield Maven("pom.xml") - - if os.path.exists(os.path.join(path, "dist.ini")) and not os.path.exists( - os.path.join(path, "Makefile.PL") - ): - yield DistInkt("dist.ini") - - gemfiles = [entry.name for entry in os.scandir(path) if entry.name.endswith(".gem")] - if gemfiles: - yield Gem(gemfiles[0]) - - if any( - [ - os.path.exists(os.path.join(path, p)) - for p in [ - "Makefile", - "GNUmakefile", - "makefile", - "Makefile.PL", - "CMakeLists.txt", - "autogen.sh", - "configure.ac", - "configure.in", + def dist(self, session, resolver, fixers, target_directory, quiet=False): + with DistCatcher( + [ + session.external_path("dist-newstyle/sdist"), + session.external_path("dist"), ] - ] - ): - yield Make() + ) as dc: + self._run(session, ["sdist"], fixers) + return dc.copy_single(target_directory) - seen_golang = False - if os.path.exists(os.path.join(path, ".travis.yml")): - import ruamel.yaml.reader + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "Setup.hs")): + logging.debug("Found Setup.hs, assuming haskell package.") + return cls(os.path.join(path, "Setup.hs")) - with open(".travis.yml", "rb") as f: - try: - data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) - except ruamel.yaml.reader.ReaderError as e: - warnings.warn("Unable to parse .travis.yml: %s" % (e,)) - else: - language = data.get("language") - if language == "go": - yield Golang() - seen_golang = True - if not seen_golang: +class Composer(BuildSystem): + + name = "composer" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "composer.json")): + logging.debug("Found composer.json, assuming composer package.") + return cls(path) + + +class PerlBuildTiny(BuildSystem): + + name = "perl-build-tiny" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def setup(self, session, fixers): + run_with_build_fixers(session, ["perl", "Build.PL"], fixers) + + def test(self, session, resolver, fixers): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "test"], fixers) + + def build(self, session, resolver, fixers): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "build"], fixers) + + def clean(self, session, resolver, fixers): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "clean"], fixers) + + def install(self, session, resolver, fixers, install_target): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "install"], fixers) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "Build.PL")): + logging.debug("Found Build.PL, assuming Module::Build::Tiny package.") + return cls(path) + + +BUILDSYSTEM_CLSES = [ + Pear, + SetupPy, + Npm, + Waf, + Cargo, + Meson, + Cabal, + Gradle, + Maven, + DistZilla, + Gem, + PerlBuildTiny, + Golang, + R, + Octave, + # Make is intentionally at the end of the list. + Make, + Composer, + RunTests, +] + + +def scan_buildsystems(path): + """Detect build systems.""" + ret = [] + ret.extend([(".", bs) for bs in detect_buildsystems(path)]) + + if not ret: + # Nothing found. Try the next level? for entry in os.scandir(path): - if entry.name.endswith(".go"): - yield Golang() - break + if entry.is_dir(): + ret.extend([(entry.name, bs) for bs in detect_buildsystems(entry.path)]) + + return ret -def get_buildsystem(path, trust_package=False): - for buildsystem in detect_buildsystems(path, trust_package=trust_package): - return buildsystem +def detect_buildsystems(path): + for bs_cls in BUILDSYSTEM_CLSES: + bs = bs_cls.probe(path) + if bs is not None: + yield bs + + +def get_buildsystem(path: str) -> Tuple[str, BuildSystem]: + for subpath, buildsystem in scan_buildsystems(path): + return subpath, buildsystem raise NoBuildToolsFound() diff --git a/ognibuild/debian/__init__.py b/ognibuild/debian/__init__.py index 8879a4c..23a56a1 100644 --- a/ognibuild/debian/__init__.py +++ b/ognibuild/debian/__init__.py @@ -15,14 +15,14 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import os from debian.deb822 import Deb822 from ..session import Session -# TODO(jelmer): move this to debian/ -def satisfy_build_deps(session: Session, tree): - source = Deb822(tree.get_file("debian/control")) +def satisfy_build_deps(session: Session, tree, debian_path): + source = Deb822(tree.get_file(os.path.join(debian_path, "control"))) deps = [] for name in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"]: try: diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 6dc89f7..7c168aa 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -17,8 +17,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging -import re -from typing import List, Iterator, Optional, Set +from typing import List, Optional import os from buildlog_consultant.apt import ( @@ -26,12 +25,22 @@ from buildlog_consultant.apt import ( ) from .. import DetailedFailure, UnidentifiedError -from ..session import Session, run_with_tee +from ..session import Session, run_with_tee, get_user +from .file_search import ( + FileSearcher, + AptCachedContentsFileSearcher, + GENERATED_FILE_SEARCHER, + get_packages_for_paths, +) -def run_apt(session: Session, args: List[str]) -> None: +def run_apt( + session: Session, args: List[str], prefix: Optional[List[str]] = None +) -> None: """Run apt.""" - args = ["apt", "-y"] + args + if prefix is None: + prefix = [] + args = prefix = ["apt", "-y"] + args retcode, lines = run_with_tee(session, args, cwd="/", user="root") if retcode == 0: return @@ -43,25 +52,31 @@ def run_apt(session: Session, args: List[str]) -> None: raise UnidentifiedError(retcode, args, lines, secondary=match) -class FileSearcher(object): - def search_files(self, path: str, regex: bool = False) -> Iterator[str]: - raise NotImplementedError(self.search_files) - - class AptManager(object): session: Session _searchers: Optional[List[FileSearcher]] - def __init__(self, session): + def __init__(self, session, prefix=None): self.session = session self._apt_cache = None self._searchers = None + if prefix is None: + prefix = [] + self.prefix = prefix + + @classmethod + def from_session(cls, session): + if get_user(session) != "root": + prefix = ["sudo"] + else: + prefix = [] + return cls(session, prefix=prefix) def searchers(self): if self._searchers is None: self._searchers = [ - AptContentsFileSearcher.from_session(self.session), + AptCachedContentsFileSearcher.from_session(self.session), GENERATED_FILE_SEARCHER, ] return self._searchers @@ -73,10 +88,12 @@ class AptManager(object): self._apt_cache = apt.Cache(rootdir=self.session.location) return package in self._apt_cache - def get_package_for_paths(self, paths, regex=False): + def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): logging.debug("Searching for packages containing %r", paths) # TODO(jelmer): Make sure we use whatever is configured in self.session - return get_package_for_paths(paths, self.searchers(), regex=regex) + return get_packages_for_paths( + paths, self.searchers(), regex=regex, case_insensitive=case_insensitive + ) def missing(self, packages): root = getattr(self.session, "location", "/") @@ -98,220 +115,10 @@ class AptManager(object): logging.info("Installing using apt: %r", packages) packages = self.missing(packages) if packages: - run_apt(self.session, ["install"] + packages) + run_apt(self.session, ["install"] + packages, prefix=self.prefix) def satisfy(self, deps: List[str]) -> None: - run_apt(self.session, ["satisfy"] + deps) + run_apt(self.session, ["satisfy"] + deps, prefix=self.prefix) - -class ContentsFileNotFound(Exception): - """The contents file was not found.""" - - -class AptContentsFileSearcher(FileSearcher): - def __init__(self): - self._db = {} - - @classmethod - def from_session(cls, session): - logging.info("Loading apt contents information") - # TODO(jelmer): what about sources.list.d? - from aptsources.sourceslist import SourcesList - - sl = SourcesList() - sl.load(os.path.join(session.location, "etc/apt/sources.list")) - return cls.from_sources_list( - sl, - cache_dirs=[ - os.path.join(session.location, "var/lib/apt/lists"), - "/var/lib/apt/lists", - ], - ) - - def __setitem__(self, path, package): - self._db[path] = package - - def search_files(self, path, regex=False): - c = re.compile(path) - for p, pkg in sorted(self._db.items()): - if regex: - if c.match(p): - yield pkg - else: - if path == p: - yield pkg - - def load_file(self, f): - for line in f: - (path, rest) = line.rsplit(maxsplit=1) - package = rest.split(b"/")[-1] - decoded_path = "/" + path.decode("utf-8", "surrogateescape") - self[decoded_path] = package.decode("utf-8") - - @classmethod - def _load_cache_file(cls, url, cache_dir): - from urllib.parse import urlparse - - parsed = urlparse(url) - p = os.path.join( - cache_dir, parsed.hostname + parsed.path.replace("/", "_") + ".lz4" - ) - if not os.path.exists(p): - return None - logging.debug("Loading cached contents file %s", p) - import lz4.frame - - return lz4.frame.open(p, mode="rb") - - @classmethod - def from_urls(cls, urls, cache_dirs=None): - self = cls() - for url, mandatory in urls: - for cache_dir in cache_dirs or []: - f = cls._load_cache_file(url, cache_dir) - if f is not None: - self.load_file(f) - break - else: - if not mandatory and self._db: - logging.debug( - "Not attempting to fetch optional contents " "file %s", url - ) - else: - logging.debug("Fetching contents file %s", url) - try: - self.load_url(url) - except ContentsFileNotFound: - if mandatory: - logging.warning("Unable to fetch contents file %s", url) - else: - logging.debug( - "Unable to fetch optional contents file %s", url - ) - return self - - @classmethod - def from_sources_list(cls, sl, cache_dirs=None): - # TODO(jelmer): Use aptsources.sourceslist.SourcesList - from .build import get_build_architecture - - # TODO(jelmer): Verify signatures, etc. - urls = [] - arches = [(get_build_architecture(), True), ("all", False)] - for source in sl.list: - if source.invalid or source.disabled: - continue - if source.type == "deb-src": - continue - if source.type != "deb": - logging.warning("Invalid line in sources: %r", source) - continue - base_url = source.uri.rstrip("/") - name = source.dist.rstrip("/") - components = source.comps - if components: - dists_url = base_url + "/dists" - else: - dists_url = base_url - if components: - for component in components: - for arch, mandatory in arches: - urls.append( - ( - "%s/%s/%s/Contents-%s" - % (dists_url, name, component, arch), - mandatory, - ) - ) - else: - for arch, mandatory in arches: - urls.append( - ( - "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), - mandatory, - ) - ) - return cls.from_urls(urls, cache_dirs=cache_dirs) - - @staticmethod - def _get(url): - from urllib.request import urlopen, Request - - request = Request(url, headers={"User-Agent": "Debian Janitor"}) - return urlopen(request) - - def load_url(self, url, allow_cache=True): - from urllib.error import HTTPError - - for ext in [".xz", ".gz", ""]: - try: - response = self._get(url + ext) - except HTTPError as e: - if e.status == 404: - continue - raise - break - else: - raise ContentsFileNotFound(url) - if ext == ".gz": - import gzip - - f = gzip.GzipFile(fileobj=response) - elif ext == ".xz": - import lzma - from io import BytesIO - - f = BytesIO(lzma.decompress(response.read())) - elif response.headers.get_content_type() == "text/plain": - f = response - else: - raise Exception( - "Unknown content type %r" % response.headers.get_content_type() - ) - self.load_file(f) - - -class GeneratedFileSearcher(FileSearcher): - def __init__(self, db): - self._db = db - - def search_files(self, path: str, regex: bool = False) -> Iterator[str]: - for p, pkg in sorted(self._db.items()): - if regex: - if re.match(path, p): - yield pkg - else: - if path == p: - yield pkg - - -# TODO(jelmer): read from a file -GENERATED_FILE_SEARCHER = GeneratedFileSearcher( - { - "/etc/locale.gen": "locales", - # Alternative - "/usr/bin/rst2html": "/usr/share/docutils/scripts/python3/rst2html", - } -) - - -def get_package_for_paths( - paths: List[str], searchers: List[FileSearcher], regex: bool = False -) -> Optional[str]: - candidates: Set[str] = set() - for path in paths: - for searcher in searchers: - candidates.update(searcher.search_files(path, regex=regex)) - if candidates: - break - if len(candidates) == 0: - logging.warning("No packages found that contain %r", paths) - return None - if len(candidates) > 1: - logging.warning( - "More than 1 packages found that contain %r: %r", path, candidates - ) - # Euhr. Pick the one with the shortest name? - return sorted(candidates, key=len)[0] - else: - return candidates.pop() + def satisfy_command(self, deps: List[str]) -> List[str]: + return self.prefix + ["apt", "satisfy"] + deps diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 227a38b..cca50fb 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -33,9 +33,9 @@ import sys from debian.changelog import Changelog from debmutate.changelog import get_maintainer, format_datetime -from breezy import osutils from breezy.mutabletree import MutableTree from breezy.plugins.debian.builder import BuildFailedError +from breezy.tree import Tree from buildlog_consultant.sbuild import ( worker_failure_from_sbuild_log, @@ -71,6 +71,18 @@ def get_build_architecture(): raise Exception("Could not find the build architecture: %s" % e) +def control_files_in_root(tree: Tree, subpath: str) -> bool: + debian_path = os.path.join(subpath, "debian") + if tree.has_filename(debian_path): + return False + control_path = os.path.join(subpath, "control") + if tree.has_filename(control_path): + return True + if tree.has_filename(control_path + ".in"): + return True + return False + + def add_dummy_changelog_entry( tree: MutableTree, subpath: str, @@ -99,7 +111,10 @@ def add_dummy_changelog_entry( else: return v + suffix + "1" - path = os.path.join(subpath, "debian", "changelog") + if control_files_in_root(tree, subpath): + path = os.path.join(subpath, "changelog") + else: + path = os.path.join(subpath, "debian", "changelog") if maintainer is None: maintainer = get_maintainer() if timestamp is None: @@ -126,7 +141,10 @@ def add_dummy_changelog_entry( def get_latest_changelog_version(local_tree, subpath=""): - path = osutils.pathjoin(subpath, "debian/changelog") + if control_files_in_root(local_tree, subpath): + path = os.path.join(subpath, "changelog") + else: + path = os.path.join(subpath, "debian", "changelog") with local_tree.get_file(path) as f: cl = Changelog(f, max_blocks=1) return cl.package, cl.version diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py new file mode 100644 index 0000000..888325f --- /dev/null +++ b/ognibuild/debian/build_deps.py @@ -0,0 +1,84 @@ +#!/usr/bin/python3 +# Copyright (C) 2021 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +"""Tie breaking by build deps.""" + + +import logging + + +class BuildDependencyTieBreaker(object): + def __init__(self, rootdir): + self.rootdir = rootdir + self._counts = None + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.rootdir) + + @classmethod + def from_session(cls, session): + return cls(session.location) + + def _count(self): + counts = {} + import apt_pkg + + apt_pkg.init() + apt_pkg.config.set("Dir", self.rootdir) + apt_cache = apt_pkg.SourceRecords() + apt_cache.restart() + while apt_cache.step(): + try: + for d in apt_cache.build_depends.values(): + for o in d: + for p in o: + counts.setdefault(p[0], 0) + counts[p[0]] += 1 + except AttributeError: + pass + return counts + + def __call__(self, reqs): + if self._counts is None: + self._counts = self._count() + by_count = {} + for req in reqs: + try: + by_count[req] = self._counts[list(req.package_names())[0]] + except KeyError: + pass + if not by_count: + return None + top = max(by_count.items(), key=lambda k: k[1]) + logging.info( + "Breaking tie between %r to %r based on build-depends count", + [repr(r) for r in reqs], + top[0], + ) + return top[0] + + +if __name__ == "__main__": + import argparse + from ..resolver.apt import AptRequirement + + parser = argparse.ArgumentParser() + parser.add_argument("req", nargs="+") + args = parser.parse_args() + reqs = [AptRequirement.from_str(req) for req in args.req] + tie_breaker = BuildDependencyTieBreaker("/") + print(tie_breaker(reqs)) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py new file mode 100644 index 0000000..d0d9ff4 --- /dev/null +++ b/ognibuild/debian/file_search.py @@ -0,0 +1,349 @@ +#!/usr/bin/python +# Copyright (C) 2019-2020 Jelmer Vernooij +# encoding: utf-8 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import apt_pkg +from datetime import datetime +from debian.deb822 import Release +import os +import re +from typing import Iterator, List +import logging + + +from .. import USER_AGENT + + +class FileSearcher(object): + def search_files( + self, path: str, regex: bool = False, case_insensitive: bool = False + ) -> Iterator[str]: + raise NotImplementedError(self.search_files) + + +class ContentsFileNotFound(Exception): + """The contents file was not found.""" + + +def read_contents_file(f): + for line in f: + (path, rest) = line.rsplit(maxsplit=1) + yield path, rest + + +def contents_urls_from_sources_entry(source, arches, load_url): + if source.invalid or source.disabled: + return + if source.type == "deb-src": + return + if source.type != "deb": + logging.warning("Invalid line in sources: %r", source) + return + base_url = source.uri.rstrip("/") + name = source.dist.rstrip("/") + components = source.comps + if components: + dists_url = base_url + "/dists" + else: + dists_url = base_url + inrelease_url = "%s/%s/InRelease" % (dists_url, name) + try: + response = load_url(inrelease_url) + except FileNotFoundError: + release_url = "%s/%s/Release" % (dists_url, name) + try: + response = load_url(release_url) + except FileNotFoundError as e: + logging.warning( + "Unable to download %s or %s: %s", inrelease_url, release_url, e + ) + return + + existing_names = {} + release = Release(response.read()) + for hn in ["MD5Sum", "SHA1Sum", "SHA256Sum"]: + for entry in release.get(hn, []): + existing_names[os.path.splitext(entry["name"])[0]] = entry["name"] + + contents_files = set() + if components: + for component in components: + for arch in arches: + contents_files.add("%s/Contents-%s" % (component, arch)) + else: + for arch in arches: + contents_files.add("Contents-%s" % (arch,)) + + for fn in contents_files: + if fn in existing_names: + url = "%s/%s/%s" % (dists_url, name, fn) + yield url + + +def contents_urls_from_sourceslist(sl, arch, load_url): + # TODO(jelmer): Verify signatures, etc. + arches = [arch, "all"] + for source in sl.list: + yield from contents_urls_from_sources_entry(source, arches, load_url) + + +def _unwrap(f, ext): + if ext == ".gz": + import gzip + + return gzip.GzipFile(fileobj=f) + elif ext == ".xz": + import lzma + from io import BytesIO + + f = BytesIO(lzma.decompress(f.read())) + else: + return f + + +def load_direct_url(url): + from urllib.error import HTTPError + from urllib.request import urlopen, Request + + for ext in [".xz", ".gz", ""]: + try: + request = Request(url + ext, headers={"User-Agent": USER_AGENT}) + response = urlopen(request) + except HTTPError as e: + if e.status == 404: + continue + raise + break + else: + raise FileNotFoundError(url) + return _unwrap(response, ext) + + +def load_url_with_cache(url, cache_dirs): + for cache_dir in cache_dirs: + try: + return load_apt_cache_file(url, cache_dir) + except FileNotFoundError: + pass + return load_direct_url(url) + + +def load_apt_cache_file(url, cache_dir): + fn = apt_pkg.uri_to_filename(url) + for ext in [".xz", ".gz", ".lz4", ""]: + p = os.path.join(cache_dir, fn + ext) + if not os.path.exists(p): + continue + # return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) + logging.debug("Loading cached contents file %s", p) + if ext == ".lz4": + import lz4.frame + + return lz4.frame.open(p, mode="rb") + return _unwrap(open(p, "rb"), ext) + raise FileNotFoundError(url) + + +class AptCachedContentsFileSearcher(FileSearcher): + def __init__(self): + self._db = {} + + @classmethod + def from_session(cls, session): + logging.info("Loading apt contents information") + + self = cls() + self.load_from_session(session) + return self + + def load_local(self): + # TODO(jelmer): what about sources.list.d? + from aptsources.sourceslist import SourcesList + + sl = SourcesList() + sl.load("/etc/apt/sources.list") + + from .build import get_build_architecture + + cache_dirs = set(["/var/lib/apt/lists"]) + + def load_url(url): + return load_url_with_cache(url, cache_dirs) + + urls = list( + contents_urls_from_sourceslist(sl, get_build_architecture(), load_url) + ) + self._load_urls(urls, cache_dirs, load_url) + + def load_from_session(self, session): + # TODO(jelmer): what about sources.list.d? + from aptsources.sourceslist import SourcesList + + sl = SourcesList() + sl.load(os.path.join(session.location, "etc/apt/sources.list")) + + from .build import get_build_architecture + + cache_dirs = set( + [ + os.path.join(session.location, "var/lib/apt/lists"), + "/var/lib/apt/lists", + ] + ) + + def load_url(url): + return load_url_with_cache(url, cache_dirs) + + urls = list( + contents_urls_from_sourceslist(sl, get_build_architecture(), load_url) + ) + self._load_urls(urls, cache_dirs, load_url) + + def _load_urls(self, urls, cache_dirs, load_url): + for url in urls: + try: + f = load_url(url) + self.load_file(f, url) + except ContentsFileNotFound: + logging.warning("Unable to fetch contents file %s", url) + + def __setitem__(self, path, package): + self._db[path] = package + + def search_files(self, path, regex=False, case_insensitive=False): + path = path.lstrip("/").encode("utf-8", "surrogateescape") + if case_insensitive and not regex: + regex = True + path = re.escape(path) + if regex: + flags = 0 + if case_insensitive: + flags |= re.I + c = re.compile(path, flags=flags) + ret = [] + for p, rest in self._db.items(): + if c.match(p): + pkg = rest.split(b"/")[-1] + ret.append((p, pkg.decode("utf-8"))) + for p, pkg in sorted(ret): + yield pkg + else: + try: + yield self._db[path].split(b"/")[-1].decode("utf-8") + except KeyError: + pass + + def load_file(self, f, url): + start_time = datetime.now() + for path, rest in read_contents_file(f.readlines()): + self[path] = rest + logging.debug("Read %s in %s", url, datetime.now() - start_time) + + +class GeneratedFileSearcher(FileSearcher): + def __init__(self, db): + self._db = db + + @classmethod + def from_path(cls, path): + self = cls({}) + self.load_from_path(path) + return self + + def load_from_path(self, path): + with open(path, "r") as f: + for line in f: + (path, pkg) = line.strip().split(None, 1) + self._db[path] = pkg + + def search_files( + self, path: str, regex: bool = False, case_insensitive: bool = False + ) -> Iterator[str]: + for p, pkg in sorted(self._db.items()): + if regex: + flags = 0 + if case_insensitive: + flags |= re.I + if re.match(path, p, flags=flags): + yield pkg + elif case_insensitive: + if path.lower() == p.lower(): + yield pkg + else: + if path == p: + yield pkg + + +# TODO(jelmer): read from a file +GENERATED_FILE_SEARCHER = GeneratedFileSearcher( + { + "/etc/locale.gen": "locales", + # Alternative + "/usr/bin/rst2html": "python3-docutils", + # aclocal is a symlink to aclocal-1.XY + "/usr/bin/aclocal": "automake", + "/usr/bin/automake": "automake", + # maven lives in /usr/share + "/usr/bin/mvn": "maven", + } +) + + +def get_packages_for_paths( + paths: List[str], + searchers: List[FileSearcher], + regex: bool = False, + case_insensitive: bool = False, +) -> List[str]: + candidates: List[str] = list() + for path in paths: + for searcher in searchers: + for pkg in searcher.search_files( + path, regex=regex, case_insensitive=case_insensitive + ): + if pkg not in candidates: + candidates.append(pkg) + return candidates + + +def main(argv): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("path", help="Path to search for.", type=str, nargs="*") + parser.add_argument("--regex", "-x", help="Search for regex.", action="store_true") + parser.add_argument("--debug", action="store_true") + args = parser.parse_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + main_searcher = AptCachedContentsFileSearcher() + main_searcher.load_local() + searchers = [main_searcher, GENERATED_FILE_SEARCHER] + + packages = get_packages_for_paths(args.path, searchers=searchers, regex=args.regex) + for package in packages: + print(package) + + +if __name__ == "__main__": + import sys + + sys.exit(main(sys.argv)) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 753a454..5e12726 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -19,8 +19,10 @@ __all__ = [ "build_incrementally", ] +from functools import partial import logging import os +import shutil import sys from typing import List, Set, Optional, Type @@ -30,8 +32,8 @@ from debian.deb822 import ( ) from breezy.commit import PointlessCommit -from breezy.mutabletree import MutableTree from breezy.tree import Tree +from debmutate.changelog import ChangelogEditor from debmutate.control import ( ensure_relation, ControlEditor, @@ -49,12 +51,47 @@ from debmutate.reformatting import ( try: from breezy.workspace import reset_tree -except ImportError: - from lintian_brush import reset_tree +except ImportError: # breezy < 3.2 + + def delete_items(deletables, dry_run=False): + """Delete files in the deletables iterable""" + import errno + import shutil + + def onerror(function, path, excinfo): + """Show warning for errors seen by rmtree.""" + # Handle only permission error while removing files. + # Other errors are re-raised. + if function is not os.remove or excinfo[1].errno != errno.EACCES: + raise + logging.warning("unable to remove %s" % path) + + for path, subp in deletables: + if os.path.isdir(path): + shutil.rmtree(path, onerror=onerror) + else: + try: + os.unlink(path) + except OSError as e: + # We handle only permission error here + if e.errno != errno.EACCES: + raise e + logging.warning('unable to remove "%s": %s.', path, e.strerror) + + def reset_tree(local_tree, subpath=""): + from breezy.transform import revert + from breezy.clean_tree import iter_deletables + + revert( + local_tree, + local_tree.branch.basis_tree(), + [subpath] if subpath not in (".", "") else None, + ) + deletables = list( + iter_deletables(local_tree, unknown=True, ignored=False, detritus=False) + ) + delete_items(deletables) -from lintian_brush.changelog import ( - add_changelog_entry, -) from debmutate._rules import ( dh_invoke_add_with, @@ -71,8 +108,6 @@ from buildlog_consultant.common import ( MissingAutomakeInput, MissingConfigure, NeedPgBuildExtUpdateControl, - MissingPythonModule, - MissingPythonDistribution, MissingPerlFile, ) from buildlog_consultant.sbuild import ( @@ -80,10 +115,9 @@ from buildlog_consultant.sbuild import ( ) from ..buildlog import problem_to_upstream_requirement -from ..fix_build import BuildFixer, resolve_error, DependencyContext +from ..fix_build import BuildFixer, resolve_error from ..resolver.apt import ( AptRequirement, - get_package_for_python_module, ) from .build import attempt_build, DEFAULT_BUILDER @@ -98,10 +132,46 @@ class CircularDependency(Exception): self.package = package -class PackageDependencyFixer(BuildFixer): +class DebianPackagingContext(object): + def __init__( + self, tree, subpath, committer, update_changelog, commit_reporter=None + ): + self.tree = tree + self.subpath = subpath + self.committer = committer + self.update_changelog = update_changelog + self.commit_reporter = commit_reporter - def __init__(self, apt_resolver): + def abspath(self, *parts): + return self.tree.abspath(os.path.join(self.subpath, *parts)) + + def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool: + if update_changelog is None: + update_changelog = self.update_changelog + with self.tree.lock_write(): + try: + if update_changelog: + cl_path = self.abspath("debian/changelog") + with ChangelogEditor(cl_path) as editor: + editor.add_entry([summary]) + debcommit(self.tree, committer=self.committer, subpath=self.subpath) + else: + self.tree.commit( + message=summary, + committer=self.committer, + specific_files=[self.subpath], + reporter=self.commit_reporter, + ) + except PointlessCommit: + return False + else: + return True + + +class PackageDependencyFixer(BuildFixer): + def __init__(self, context, apt_resolver): self.apt_resolver = apt_resolver + self.context = context def __repr__(self): return "%s(%r)" % (type(self).__name__, self.apt_resolver) @@ -113,7 +183,7 @@ class PackageDependencyFixer(BuildFixer): req = problem_to_upstream_requirement(error) return req is not None - def fix(self, error, context): + def fix(self, error, phase): reqs = problem_to_upstream_requirement(error) if reqs is None: return False @@ -123,82 +193,29 @@ class PackageDependencyFixer(BuildFixer): changed = False for req in reqs: - package = self.apt_resolver.resolve(req) - if package is None: - return False - if context.phase[0] == "autopkgtest": - return add_test_dependency( - context.tree, - context.phase[1], - package, - committer=context.committer, - subpath=context.subpath, - update_changelog=context.update_changelog, - ) - elif context.phase[0] == "build": - return add_build_dependency( - context.tree, - package, - committer=context.committer, - subpath=context.subpath, - update_changelog=context.update_changelog, - ) - else: - logging.warning('Unknown phase %r', context.phase) + apt_req = self.apt_resolver.resolve(req) + if apt_req is None: return False + if add_dependency(self.context, phase, apt_req): + changed = True return changed -class BuildDependencyContext(DependencyContext): - def __init__( - self, phase, tree, apt, subpath="", committer=None, update_changelog=True - ): - self.phase = phase - super(BuildDependencyContext, self).__init__( - tree, apt, subpath, committer, update_changelog - ) - - def add_dependency(self, requirement: AptRequirement): - return add_build_dependency( - self.tree, - requirement, - committer=self.committer, - subpath=self.subpath, - update_changelog=self.update_changelog, - ) +def add_dependency(context, phase, requirement: AptRequirement): + if phase[0] == "autopkgtest": + return add_test_dependency(context, phase[1], requirement) + elif phase[0] == "build": + return add_build_dependency(context, requirement) + else: + logging.warning("Unknown phase %r", phase) + return False -class AutopkgtestDependencyContext(DependencyContext): - def __init__( - self, phase, tree, apt, subpath="", committer=None, update_changelog=True - ): - self.phase = phase - super(AutopkgtestDependencyContext, self).__init__( - tree, apt, subpath, committer, update_changelog - ) - - def add_dependency(self, requirement): - return add_test_dependency( - self.tree, - self.testname, - requirement, - committer=self.committer, - subpath=self.subpath, - update_changelog=self.update_changelog, - ) - - -def add_build_dependency( - tree: Tree, - requirement: AptRequirement, - committer: Optional[str] = None, - subpath: str = "", - update_changelog: bool = True, -): +def add_build_dependency(context, requirement: AptRequirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - control_path = os.path.join(tree.abspath(subpath), "debian/control") + control_path = context.abspath("debian/control") try: with ControlEditor(path=control_path) as updater: for binary in updater.binaries: @@ -219,27 +236,17 @@ def add_build_dependency( return False logging.info("Adding build dependency: %s", desc) - return commit_debian_changes( - tree, - subpath, - "Add missing build dependency on %s." % desc, - committer=committer, - update_changelog=update_changelog, - ) + return context.commit("Add missing build dependency on %s." % desc) -def add_test_dependency( - tree, - testname, - requirement, - committer=None, - subpath="", - update_changelog=True, -): +def add_test_dependency(context, testname, requirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - tests_control_path = os.path.join(tree.abspath(subpath), "debian/tests/control") + tests_control_path = context.abspath("debian/tests/control") + + # TODO(jelmer): If requirement is for one of our binary packages + # but "@" is already present then don't do anything. try: with Deb822Editor(path=tests_control_path) as updater: @@ -265,176 +272,59 @@ def add_test_dependency( desc = requirement.pkg_relation_str() logging.info("Adding dependency to test %s: %s", testname, desc) - return commit_debian_changes( - tree, - subpath, + return context.commit( "Add missing dependency for test %s on %s." % (testname, desc), - update_changelog=update_changelog, ) -def commit_debian_changes( - tree: MutableTree, - subpath: str, - summary: str, - committer: Optional[str] = None, - update_changelog: bool = True, -) -> bool: - with tree.lock_write(): - try: - if update_changelog: - add_changelog_entry( - tree, os.path.join(subpath, "debian/changelog"), [summary] - ) - debcommit(tree, committer=committer, subpath=subpath) - else: - tree.commit( - message=summary, committer=committer, specific_files=[subpath] - ) - except PointlessCommit: - return False - else: - return True - - -def targeted_python_versions(tree: Tree) -> Set[str]: - with tree.get_file("debian/control") as f: +def targeted_python_versions(tree: Tree, subpath: str) -> List[str]: + with tree.get_file(os.path.join(subpath, "debian/control")) as f: control = Deb822(f) build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) all_build_deps: Set[str] = set() for or_deps in build_depends: all_build_deps.update(or_dep["name"] for or_dep in or_deps) - targeted = set() - if any(x.startswith("pypy") for x in all_build_deps): - targeted.add("pypy") - if any(x.startswith("python-") for x in all_build_deps): - targeted.add("cpython2") + targeted = [] if any(x.startswith("python3-") for x in all_build_deps): - targeted.add("cpython3") + targeted.append("python3") + if any(x.startswith("pypy") for x in all_build_deps): + targeted.append("pypy") + if any(x.startswith("python-") for x in all_build_deps): + targeted.append("python") return targeted -def fix_missing_python_distribution(error, context): # noqa: C901 - targeted = targeted_python_versions(context.tree) - default = not targeted +def python_tie_breaker(tree, subpath, reqs): + targeted = targeted_python_versions(tree, subpath) + if not targeted: + return None - pypy_pkg = context.apt.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True - ) - if pypy_pkg is None: - pypy_pkg = "pypy-%s" % error.distribution - if not context.apt.package_exists(pypy_pkg): - pypy_pkg = None - - py2_pkg = context.apt.get_package_for_paths( - ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution], - regex=True, - ) - if py2_pkg is None: - py2_pkg = "python-%s" % error.distribution - if not context.apt.package_exists(py2_pkg): - py2_pkg = None - - py3_pkg = context.apt.get_package_for_paths( - ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution], - regex=True, - ) - if py3_pkg is None: - py3_pkg = "python3-%s" % error.distribution - if not context.apt.package_exists(py3_pkg): - py3_pkg = None - - extra_build_deps = [] - if error.python_version == 2: - if "pypy" in targeted: - if not pypy_pkg: - logging.warning("no pypy package found for %s", error.module) - else: - extra_build_deps.append(pypy_pkg) - if "cpython2" in targeted or default: - if not py2_pkg: - logging.warning("no python 2 package found for %s", error.module) - return False - extra_build_deps.append(py2_pkg) - elif error.python_version == 3: - if not py3_pkg: - logging.warning("no python 3 package found for %s", error.module) - return False - extra_build_deps.append(py3_pkg) - else: - if py3_pkg and ("cpython3" in targeted or default): - extra_build_deps.append(py3_pkg) - if py2_pkg and ("cpython2" in targeted or default): - extra_build_deps.append(py2_pkg) - if pypy_pkg and "pypy" in targeted: - extra_build_deps.append(pypy_pkg) - - if not extra_build_deps: + def same(pkg, python_version): + if pkg.startswith(python_version + "-"): + return True + if pkg.startswith("lib%s-" % python_version): + return True return False - for dep_pkg in extra_build_deps: - assert dep_pkg is not None - if not context.add_dependency(dep_pkg): - return False + for python_version in targeted: + for req in reqs: + if any(same(name, python_version) for name in req.package_names()): + logging.info( + "Breaking tie between %r to %r, since package already " + "has %r build-dependencies", + [str(req) for req in reqs], + str(req), + python_version, + ) + return req + return None + + +def retry_apt_failure(error, phase, apt, context): return True -def fix_missing_python_module(error, context): - if getattr(context, "tree", None) is not None: - targeted = targeted_python_versions(context.tree) - else: - targeted = set() - default = not targeted - - if error.minimum_version: - specs = [(">=", error.minimum_version)] - else: - specs = [] - - pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", specs) - py2_pkg = get_package_for_python_module(context.apt, error.module, "python2", specs) - py3_pkg = get_package_for_python_module(context.apt, error.module, "python3", specs) - - extra_build_deps = [] - if error.python_version == 2: - if "pypy" in targeted: - if not pypy_pkg: - logging.warning("no pypy package found for %s", error.module) - else: - extra_build_deps.append(pypy_pkg) - if "cpython2" in targeted or default: - if not py2_pkg: - logging.warning("no python 2 package found for %s", error.module) - return False - extra_build_deps.append(py2_pkg) - elif error.python_version == 3: - if not py3_pkg: - logging.warning("no python 3 package found for %s", error.module) - return False - extra_build_deps.append(py3_pkg) - else: - if py3_pkg and ("cpython3" in targeted or default): - extra_build_deps.append(py3_pkg) - if py2_pkg and ("cpython2" in targeted or default): - extra_build_deps.append(py2_pkg) - if pypy_pkg and "pypy" in targeted: - extra_build_deps.append(pypy_pkg) - - if not extra_build_deps: - return False - - for dep_pkg in extra_build_deps: - assert dep_pkg is not None - if not context.add_dependency(dep_pkg): - return False - return True - - -def retry_apt_failure(error, context): - return True - - -def enable_dh_autoreconf(context): +def enable_dh_autoreconf(context, phase): # Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by # default. debhelper_compat_version = get_debhelper_compat_level(context.tree.abspath(".")) @@ -448,28 +338,30 @@ def enable_dh_autoreconf(context): return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return context.add_dependency(AptRequirement.simple("dh-autoreconf")) + return add_dependency( + context, phase, AptRequirement.simple("dh-autoreconf") + ) return False -def fix_missing_configure(error, context): +def fix_missing_configure(error, phase, context): if not context.tree.has_filename("configure.ac") and not context.tree.has_filename( "configure.in" ): return False - return enable_dh_autoreconf(context) + return enable_dh_autoreconf(context, phase) -def fix_missing_automake_input(error, context): +def fix_missing_automake_input(error, phase, context): # TODO(jelmer): If it's ./NEWS, ./AUTHORS or ./README that's missing, then # try to set 'export AUTOMAKE = automake --foreign' in debian/rules. # https://salsa.debian.org/jelmer/debian-janitor/issues/88 - return enable_dh_autoreconf(context) + return enable_dh_autoreconf(context, phase) -def fix_missing_config_status_input(error, context): +def fix_missing_config_status_input(error, phase, context): autogen_path = "autogen.sh" rules_path = "debian/rules" if context.subpath not in (".", ""): @@ -488,38 +380,39 @@ def fix_missing_config_status_input(error, context): if not update_rules(makefile_cb=add_autogen, path=rules_path): return False - if context.update_changelog: - commit_debian_changes( - context.tree, - context.subpath, - "Run autogen.sh during build.", - committer=context.committer, - update_changelog=context.update_changelog, - ) - - return True + return context.commit("Run autogen.sh during build.") class PgBuildExtOutOfDateControlFixer(BuildFixer): - def __init__(self, session): + def __init__(self, packaging_context, session, apt): self.session = session + self.context = packaging_context + self.apt = apt def can_fix(self, problem): return isinstance(problem, NeedPgBuildExtUpdateControl) - def _fix(self, error, context): + def __repr__(self): + return "%s()" % (type(self).__name__,) + + def _fix(self, error, phase): logging.info("Running 'pg_buildext updatecontrol'") + self.apt.install(['postgresql-common']) + external_dir, internal_dir = self.session.setup_from_vcs( + self.context.tree, include_controldir=None, + subdir=self.context.subpath) + self.session.chdir(internal_dir) self.session.check_call(["pg_buildext", "updatecontrol"]) - return commit_debian_changes( - context.tree, - context.subpath, - "Run 'pgbuildext updatecontrol'.", - committer=context.committer, - update_changelog=False, + shutil.copy( + os.path.join(external_dir, error.generated_path), + self.context.abspath(error.generated_path) + ) + return self.context.commit( + "Run 'pgbuildext updatecontrol'.", update_changelog=False ) -def fix_missing_makefile_pl(error, context): +def fix_missing_makefile_pl(error, phase, context): if ( error.filename == "Makefile.PL" and not context.tree.has_filename("Makefile.PL") @@ -531,38 +424,76 @@ def fix_missing_makefile_pl(error, context): class SimpleBuildFixer(BuildFixer): - def __init__(self, problem_cls: Type[Problem], fn): + def __init__(self, packaging_context, problem_cls: Type[Problem], fn): + self.context = packaging_context self._problem_cls = problem_cls self._fn = fn def __repr__(self): - return "%s(%r, %r)" % (type(self).__name__, self._problem_cls, self._fn) + return "%s(%s, %s)" % ( + type(self).__name__, + self._problem_cls.__name__, + self._fn.__name__, + ) def can_fix(self, problem: Problem): return isinstance(problem, self._problem_cls) - def _fix(self, problem: Problem, context): - return self._fn(problem, context) + def _fix(self, problem: Problem, phase): + return self._fn(problem, phase, self.context) -def versioned_package_fixers(session): +class DependencyBuildFixer(BuildFixer): + def __init__(self, packaging_context, apt_resolver, problem_cls: Type[Problem], fn): + self.context = packaging_context + self.apt_resolver = apt_resolver + self._problem_cls = problem_cls + self._fn = fn + + def __repr__(self): + return "%s(%s, %s)" % ( + type(self).__name__, + self._problem_cls.__name__, + self._fn.__name__, + ) + + def can_fix(self, problem: Problem): + return isinstance(problem, self._problem_cls) + + def _fix(self, problem: Problem, phase): + return self._fn(problem, phase, self.apt_resolver, self.context) + + +def versioned_package_fixers(session, packaging_context, apt): return [ - PgBuildExtOutOfDateControlFixer(session), - SimpleBuildFixer(MissingConfigure, fix_missing_configure), - SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input), - SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input), - SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl), + PgBuildExtOutOfDateControlFixer(packaging_context, session, apt), + SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure), + SimpleBuildFixer( + packaging_context, MissingAutomakeInput, fix_missing_automake_input + ), + SimpleBuildFixer( + packaging_context, MissingConfigStatusInput, fix_missing_config_status_input + ), + SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl), ] -def apt_fixers(apt) -> List[BuildFixer]: +def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver - resolver = AptResolver(apt) + from .udd import popcon_tie_breaker + from .build_deps import BuildDependencyTieBreaker + + apt_tie_breakers = [ + partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), + BuildDependencyTieBreaker.from_session(apt.session), + popcon_tie_breaker, + ] + resolver = AptResolver(apt, apt_tie_breakers) return [ - SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), - SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution), - SimpleBuildFixer(AptFetchFailure, retry_apt_failure), - PackageDependencyFixer(resolver), + DependencyBuildFixer( + packaging_context, apt, AptFetchFailure, retry_apt_failure + ), + PackageDependencyFixer(packaging_context, resolver), ] @@ -581,7 +512,12 @@ def build_incrementally( update_changelog=True, ): fixed_errors = [] - fixers = versioned_package_fixers(apt.session) + apt_fixers(apt) + packaging_context = DebianPackagingContext( + local_tree, subpath, committer, update_changelog + ) + fixers = versioned_package_fixers(apt.session, packaging_context, apt) + apt_fixers( + apt, packaging_context + ) logging.info("Using fixers: %r", fixers) while True: try: @@ -608,30 +544,9 @@ def build_incrementally( if max_iterations is not None and len(fixed_errors) > max_iterations: logging.warning("Last fix did not address the issue. Giving up.") raise - reset_tree(local_tree, local_tree.basis_tree(), subpath=subpath) - if e.phase[0] == "build": - context = BuildDependencyContext( - e.phase, - local_tree, - apt, - subpath=subpath, - committer=committer, - update_changelog=update_changelog, - ) - elif e.phase[0] == "autopkgtest": - context = AutopkgtestDependencyContext( - e.phase, - local_tree, - apt, - subpath=subpath, - committer=committer, - update_changelog=update_changelog, - ) - else: - logging.warning("unable to install for context %r", e.phase) - raise + reset_tree(local_tree, subpath=subpath) try: - if not resolve_error(e.error, context, fixers): + if not resolve_error(e.error, e.phase, fixers): logging.warning("Failed to resolve error %r. Giving up.", e.error) raise except GeneratedFile: @@ -654,10 +569,9 @@ def build_incrementally( os.path.join(output_directory, "build.log.%d" % i) ): i += 1 - os.rename( - os.path.join(output_directory, "build.log"), - os.path.join(output_directory, "build.log.%d" % i), - ) + target_path = os.path.join(output_directory, "build.log.%d" % i) + os.rename(os.path.join(output_directory, "build.log"), target_path) + logging.debug("Storing build log at %s", target_path) def main(argv=None): @@ -696,17 +610,23 @@ def main(argv=None): help="force updating of the changelog", default=None, ) + parser.add_argument("--schroot", type=str, help="chroot to use.") + parser.add_argument("--verbose", action="store_true", help="Be verbose") args = parser.parse_args() from breezy.workingtree import WorkingTree + import breezy.git # noqa: F401 + import breezy.bzr # noqa: F401 from .apt import AptManager from ..session.plain import PlainSession + from ..session.schroot import SchrootSession import tempfile import contextlib - apt = AptManager(PlainSession()) - - logging.basicConfig(level=logging.INFO, format="%(message)s") + if args.verbose: + logging.basicConfig(level=logging.DEBUG, format="%(message)s") + else: + logging.basicConfig(level=logging.INFO, format="%(message)s") with contextlib.ExitStack() as es: if args.output_directory is None: @@ -716,17 +636,43 @@ def main(argv=None): output_directory = args.output_directory tree = WorkingTree.open(".") - build_incrementally( - tree, - apt, - args.suffix, - args.suite, - output_directory, - args.build_command, - None, - committer=args.committer, - update_changelog=args.update_changelog, - ) + if args.schroot: + session = SchrootSession(args.schroot) + else: + session = PlainSession() + + es.enter_context(session) + + apt = AptManager(session) + + try: + (changes_filename, cl_version) = build_incrementally( + tree, + apt, + args.suffix, + args.suite, + output_directory, + args.build_command, + None, + committer=args.committer, + update_changelog=args.update_changelog, + ) + except SbuildFailure as e: + if e.phase is None: + phase = "unknown phase" + elif len(e.phase) == 1: + phase = e.phase[0] + else: + phase = "%s (%s)" % (e.phase[0], e.phase[1]) + if e.error: + logging.fatal("Error during %s: %s", phase, e.error) + else: + logging.fatal("Error during %s: %s", phase, e.description) + return 1 + + logging.info( + 'Built %s - changes file at %s.', + os.path.join(output_directory, changes_filename)) if __name__ == "__main__": diff --git a/ognibuild/debian/udd.py b/ognibuild/debian/udd.py new file mode 100644 index 0000000..0c73818 --- /dev/null +++ b/ognibuild/debian/udd.py @@ -0,0 +1,60 @@ +#!/usr/bin/python3 +# Copyright (C) 2021 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +"""Support for accessing UDD.""" + +import logging + + +class UDD(object): + def connect(self): + import psycopg2 + + self._conn = psycopg2.connect( + database="udd", + user="udd-mirror", + password="udd-mirror", + port=5432, + host="udd-mirror.debian.net", + ) + + def get_most_popular(self, packages): + cursor = self._conn.cursor() + cursor.execute( + "SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1", + (tuple(packages),), + ) + return cursor.fetchone()[0] + + +def popcon_tie_breaker(candidates): + # TODO(jelmer): Pick package based on what appears most commonly in + # build-depends{-indep,-arch} + try: + from .udd import UDD + except ModuleNotFoundError: + logging.warning("Unable to import UDD, not ranking by popcon") + return sorted(candidates, key=len)[0] + udd = UDD() + udd.connect() + names = {list(c.package_names())[0]: c for c in candidates} + winner = udd.get_most_popular(list(names.keys())) + if winner is None: + logging.warning("No relevant popcon information found, not ranking by popcon") + return None + logging.info("Picked winner using popcon") + return names[winner] diff --git a/ognibuild/dist.py b/ognibuild/dist.py index cfdb5db..cdf1265 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -15,14 +15,18 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +__all__ = [ + "UnidentifiedError", + "DetailedFailure", + "create_dist", + "create_dist_schroot", +] + import errno import logging import os -import shutil import sys -import tempfile -import time -from typing import Optional +from typing import Optional, List from debian.deb822 import Deb822 @@ -34,93 +38,69 @@ from buildlog_consultant.common import ( ) -from . import DetailedFailure +from . import DetailedFailure, UnidentifiedError +from .dist_catcher import DistNoTarball from .buildsystem import NoBuildToolsFound +from .resolver import auto_resolver +from .session import Session from .session.schroot import SchrootSession -from .vcs import dupe_vcs_tree, export_vcs_tree -SUPPORTED_DIST_EXTENSIONS = [ - ".tar.gz", - ".tgz", - ".tar.bz2", - ".tar.xz", - ".tar.lzma", - ".tbz2", - ".tar", - ".zip", -] - - -def is_dist_file(fn): - for ext in SUPPORTED_DIST_EXTENSIONS: - if fn.endswith(ext): - return True - return False - - -class DistNoTarball(Exception): - """Dist operation did not create a tarball.""" - - -def run_dist(session, buildsystems, resolver, fixers, quiet=False): +def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=False): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.dist(session, resolver, fixers, quiet=quiet) - return + filename = buildsystem.dist( + session, resolver, fixers, target_directory, quiet=quiet + ) + return filename raise NoBuildToolsFound() -class DistCatcher(object): - def __init__(self, directory): - self.export_directory = directory - self.files = [] - self.existing_files = None - self.start_time = time.time() +def create_dist( + session: Session, + tree: Tree, + target_dir: str, + include_controldir: bool = True, + subdir: Optional[str] = None, + cleanup: bool = False, +) -> Optional[str]: + from .buildsystem import detect_buildsystems + from .buildlog import InstallFixer + from .fix_build import BuildFixer + from .fixers import ( + GitIdentityFixer, + SecretGpgKeyFixer, + UnexpandedAutoconfMacroFixer, + ) - def __enter__(self): - self.existing_files = os.listdir(self.export_directory) - return self + if subdir is None: + subdir = "package" + try: + export_directory, reldir = session.setup_from_vcs( + tree, include_controldir=include_controldir, subdir=subdir + ) + except OSError as e: + if e.errno == errno.ENOSPC: + raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) + raise - def find_files(self): - new_files = os.listdir(self.export_directory) - diff_files = set(new_files) - set(self.existing_files) - diff = set([n for n in diff_files if is_dist_file(n)]) - if len(diff) == 1: - fn = diff.pop() - logging.info("Found tarball %s in package directory.", fn) - self.files.append(os.path.join(self.export_directory, fn)) - return fn - if "dist" in diff_files: - for entry in os.scandir(os.path.join(self.export_directory, "dist")): - if is_dist_file(entry.name): - logging.info("Found tarball %s in dist directory.", entry.name) - self.files.append(entry.path) - return entry.name - logging.info("No tarballs found in dist directory.") + # TODO(jelmer): use scan_buildsystems to also look in subdirectories + buildsystems = list(detect_buildsystems(export_directory)) + resolver = auto_resolver(session) + fixers: List[BuildFixer] = [UnexpandedAutoconfMacroFixer(session, resolver)] - parent_directory = os.path.dirname(self.export_directory) - diff = set(os.listdir(parent_directory)) - set([self.export_directory]) - if len(diff) == 1: - fn = diff.pop() - logging.info("Found tarball %s in parent directory.", fn) - self.files.append(os.path.join(parent_directory, fn)) - return fn + fixers.append(InstallFixer(resolver)) - if "dist" in new_files: - for entry in os.scandir(os.path.join(self.export_directory, "dist")): - if is_dist_file(entry.name) and entry.stat().st_mtime > self.start_time: - logging.info("Found tarball %s in dist directory.", entry.name) - self.files.append(entry.path) - return entry.name + if session.is_temporary: + # Only muck about with temporary sessions + fixers.extend([GitIdentityFixer(session), SecretGpgKeyFixer(session)]) - def __exit__(self, exc_type, exc_val, exc_tb): - self.find_files() - return False + session.chdir(reldir) + return run_dist(session, buildsystems, resolver, fixers, target_dir) def create_dist_schroot( @@ -128,54 +108,24 @@ def create_dist_schroot( target_dir: str, chroot: str, packaging_tree: Optional[Tree] = None, + packaging_subpath: Optional[str] = None, include_controldir: bool = True, subdir: Optional[str] = None, -) -> str: - from .buildsystem import detect_buildsystems - from .resolver.apt import AptResolver - from .buildlog import InstallFixer - - if subdir is None: - subdir = "package" + cleanup: bool = False, +) -> Optional[str]: with SchrootSession(chroot) as session: if packaging_tree is not None: from .debian import satisfy_build_deps - satisfy_build_deps(session, packaging_tree) - build_dir = os.path.join(session.location, "build") - - try: - directory = tempfile.mkdtemp(dir=build_dir) - except OSError as e: - if e.errno == errno.ENOSPC: - raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) - reldir = "/" + os.path.relpath(directory, session.location) - - export_directory = os.path.join(directory, subdir) - if not include_controldir: - export_vcs_tree(tree, export_directory) - else: - dupe_vcs_tree(tree, export_directory) - - buildsystems = list(detect_buildsystems(export_directory)) - resolver = AptResolver.from_session(session) - fixers = [InstallFixer(resolver)] - - with DistCatcher(export_directory) as dc: - oldcwd = os.getcwd() - os.chdir(export_directory) - try: - session.chdir(os.path.join(reldir, subdir)) - run_dist(session, buildsystems, resolver, fixers) - finally: - os.chdir(oldcwd) - - for path in dc.files: - shutil.copy(path, target_dir) - return os.path.join(target_dir, os.path.basename(path)) - - logging.info("No tarball created :(") - raise DistNoTarball() + satisfy_build_deps(session, packaging_tree, packaging_subpath) + return create_dist( + session, + tree, + target_dir, + include_controldir=include_controldir, + subdir=subdir, + cleanup=cleanup, + ) if __name__ == "__main__": @@ -205,6 +155,9 @@ if __name__ == "__main__": "--target-directory", type=str, default="..", help="Target directory" ) parser.add_argument("--verbose", action="store_true", help="Be verbose") + parser.add_argument( + "--include-controldir", action="store_true", help="Clone rather than export." + ) args = parser.parse_args() @@ -231,10 +184,23 @@ if __name__ == "__main__": target_dir=os.path.abspath(args.target_directory), packaging_tree=packaging_tree, chroot=args.chroot, + include_controldir=args.include_controldir, ) - except NoBuildToolsFound: + except (NoBuildToolsFound, NotImplementedError): logging.info("No build tools found, falling back to simple export.") export(tree, "dist.tar.gz", "tgz", None) + except NotImplementedError: + logging.info( + "Build system does not support dist tarball creation, " + "falling back to simple export." + ) + export(tree, "dist.tar.gz", "tgz", None) + except UnidentifiedError as e: + logging.fatal("Unidentified error: %r", e.lines) + except DetailedFailure as e: + logging.fatal("Identified error during dist creation: %s", e.error) + except DistNoTarball: + logging.fatal("dist operation did not create a tarball") else: - print("Created %s" % ret) + logging.info("Created %s", ret) sys.exit(0) diff --git a/ognibuild/dist_catcher.py b/ognibuild/dist_catcher.py new file mode 100644 index 0000000..2739773 --- /dev/null +++ b/ognibuild/dist_catcher.py @@ -0,0 +1,117 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import logging +import shutil +import time + + +class DistNoTarball(Exception): + """Dist operation did not create a tarball.""" + + +SUPPORTED_DIST_EXTENSIONS = [ + ".tar.gz", + ".tgz", + ".tar.bz2", + ".tar.xz", + ".tar.lzma", + ".tbz2", + ".tar", + ".zip", +] + + +def is_dist_file(fn): + for ext in SUPPORTED_DIST_EXTENSIONS: + if fn.endswith(ext): + return True + return False + + +class DistCatcher(object): + def __init__(self, directories): + self.directories = [os.path.abspath(d) for d in directories] + self.files = [] + self.existing_files = None + self.start_time = time.time() + + @classmethod + def default(cls, directory): + return cls( + [os.path.join(directory, "dist"), directory, os.path.join(directory, "..")] + ) + + def __enter__(self): + self.existing_files = {} + for directory in self.directories: + try: + self.existing_files[directory] = { + entry.name: entry for entry in os.scandir(directory) + } + except FileNotFoundError: + self.existing_files[directory] = {} + return self + + def find_files(self): + for directory in self.directories: + old_files = self.existing_files[directory] + possible_new = [] + possible_updated = [] + if not os.path.isdir(directory): + continue + for entry in os.scandir(directory): + if not entry.is_file() or not is_dist_file(entry.name): + continue + old_entry = old_files.get(entry.name) + if not old_entry: + possible_new.append(entry) + continue + if entry.stat().st_mtime > self.start_time: + possible_updated.append(entry) + continue + if len(possible_new) == 1: + entry = possible_new[0] + logging.info("Found new tarball %s in %s.", entry.name, directory) + self.files.append(entry.path) + return entry.name + elif len(possible_new) > 1: + logging.warning( + "Found multiple tarballs %r in %s.", possible_new, directory + ) + return + + if len(possible_updated) == 1: + entry = possible_updated[0] + logging.info("Found updated tarball %s in %s.", entry.name, directory) + self.files.append(entry.path) + return entry.name + + def __exit__(self, exc_type, exc_val, exc_tb): + self.find_files() + return False + + def copy_single(self, target_dir): + for path in self.files: + try: + shutil.copy(path, target_dir) + except shutil.SameFileError: + pass + return os.path.basename(path) + logging.info("No tarball created :(") + raise DistNoTarball() diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index d7dca7f..e83b84b 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -15,60 +15,45 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from functools import partial import logging -from typing import List, Optional +from typing import List, Tuple, Callable, Any +from buildlog_consultant import Problem from buildlog_consultant.common import ( find_build_failure_description, + MissingCommand, ) -from breezy.mutabletree import MutableTree from . import DetailedFailure, UnidentifiedError -from .debian.apt import AptManager from .session import Session, run_with_tee class BuildFixer(object): """Build fixer.""" - def can_fix(self, problem): + def can_fix(self, problem: Problem): raise NotImplementedError(self.can_fix) - def _fix(self, problem, context): + def _fix(self, problem: Problem, phase: Tuple[str, ...]): raise NotImplementedError(self._fix) - def fix(self, problem, context): + def fix(self, problem: Problem, phase: Tuple[str, ...]): if not self.can_fix(problem): return None - return self._fix(problem, context) + return self._fix(problem, phase) -class DependencyContext(object): - def __init__( - self, - tree: MutableTree, - apt: AptManager, - subpath: str = "", - committer: Optional[str] = None, - update_changelog: bool = True, - ): - self.tree = tree - self.apt = apt - self.subpath = subpath - self.committer = committer - self.update_changelog = update_changelog - - def add_dependency(self, package) -> bool: - raise NotImplementedError(self.add_dependency) - - -def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer]): - logging.info("Running %r", args) - fixed_errors = [] - while True: - retcode, lines = run_with_tee(session, args) +def run_detecting_problems(session: Session, args: List[str], **kwargs): + try: + retcode, contents = run_with_tee(session, args, **kwargs) + except FileNotFoundError: + error = MissingCommand(args[0]) + retcode = 1 + else: if retcode == 0: - return + return contents + lines = "".join(contents).splitlines(False) match, error = find_build_failure_description(lines) if error is None: if match: @@ -77,24 +62,56 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF else: logging.warning("Build failed and unable to find cause. Giving up.") raise UnidentifiedError(retcode, args, lines, secondary=match) - - logging.info("Identified error: %r", error) - if error in fixed_errors: - logging.warning( - "Failed to resolve error %r, it persisted. Giving up.", error - ) - raise DetailedFailure(retcode, args, error) - if not resolve_error( - error, - None, - fixers=fixers, - ): - logging.warning("Failed to find resolution for error %r. Giving up.", error) - raise DetailedFailure(retcode, args, error) - fixed_errors.append(error) + raise DetailedFailure(retcode, args, error) -def resolve_error(error, context, fixers): +def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): + """Call cb() until there are no more DetailedFailures we can fix. + + Args: + fixers: List of fixers to use to resolve issues + """ + fixed_errors = [] + while True: + to_resolve = [] + try: + return cb() + except DetailedFailure as e: + to_resolve.append(e) + while to_resolve: + f = to_resolve.pop(-1) + logging.info("Identified error: %r", f.error) + if f.error in fixed_errors: + logging.warning( + "Failed to resolve error %r, it persisted. Giving up.", f.error + ) + raise f + try: + resolved = resolve_error(f.error, None, fixers=fixers) + except DetailedFailure as n: + logging.info("New error %r while resolving %r", n, f) + if n in to_resolve: + raise + to_resolve.append(f) + to_resolve.append(n) + else: + if not resolved: + logging.warning( + "Failed to find resolution for error %r. Giving up.", f.error + ) + raise f + fixed_errors.append(f.error) + + +def run_with_build_fixers( + session: Session, args: List[str], fixers: List[BuildFixer], **kwargs +): + return iterate_with_build_fixers( + fixers, partial(run_detecting_problems, session, args, **kwargs) + ) + + +def resolve_error(error, phase, fixers): relevant_fixers = [] for fixer in fixers: if fixer.can_fix(error): @@ -104,7 +121,7 @@ def resolve_error(error, context, fixers): return False for fixer in relevant_fixers: logging.info("Attempting to use fixer %s to address %r", fixer, error) - made_changes = fixer.fix(error, context) + made_changes = fixer.fix(error, phase) if made_changes: return True return False diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py new file mode 100644 index 0000000..c413f20 --- /dev/null +++ b/ognibuild/fixers.py @@ -0,0 +1,103 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import subprocess +from typing import Tuple + +from buildlog_consultant import Problem +from buildlog_consultant.common import ( + MissingGitIdentity, + MissingSecretGpgKey, + MissingAutoconfMacro, +) +from ognibuild.requirements import AutoconfMacroRequirement +from ognibuild.resolver import UnsatisfiedRequirements + +from .fix_build import BuildFixer + + +class GitIdentityFixer(BuildFixer): + def __init__(self, session): + self.session = session + + def can_fix(self, problem: Problem): + return isinstance(problem, MissingGitIdentity) + + def _fix(self, problem: Problem, phase: Tuple[str, ...]): + for name in ["user.email", "user.name"]: + value = ( + subprocess.check_output(["git", "config", "--global", name]) + .decode() + .strip() + ) + self.session.check_call(["git", "config", "--global", name, value]) + return True + + +class SecretGpgKeyFixer(BuildFixer): + def __init__(self, session): + self.session = session + + def can_fix(self, problem: Problem): + return isinstance(problem, MissingSecretGpgKey) + + def _fix(self, problem: Problem, phase: Tuple[str, ...]): + SCRIPT = b"""\ +Key-Type: 1 +Key-Length: 4096 +Subkey-Type: 1 +Subkey-Length: 4096 +Name-Real: Dummy Key for ognibuild +Name-Email: dummy@example.com +Expire-Date: 0 +Passphrase: "" +""" + p = self.session.Popen( + ["gpg", "--gen-key", "--batch", "/dev/stdin"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + p.communicate(SCRIPT) + if p.returncode == 0: + return True + return False + + +class UnexpandedAutoconfMacroFixer(BuildFixer): + def __init__(self, session, resolver): + self.session = session + self.resolver = resolver + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.resolver) + + def __str__(self): + return "unexpanded m4 macro fixer (%s)" % self.resolver + + def can_fix(self, error): + return isinstance(error, MissingAutoconfMacro) + + def _fix(self, error, phase): + try: + self.resolver.install([AutoconfMacroRequirement(error.macro)]) + except UnsatisfiedRequirements: + return False + from .fix_build import run_detecting_problems + + run_detecting_problems(self.session, ["autoconf", "-f"]) + + return True diff --git a/ognibuild/info.py b/ognibuild/info.py index 553b620..0ee1cfd 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -21,7 +21,7 @@ def run_info(session, buildsystems): print("%r:" % buildsystem) deps = {} try: - for kind, dep in buildsystem.get_declared_dependencies(): + for kind, dep in buildsystem.get_declared_dependencies(session): deps.setdefault(kind, []).append(dep) except NotImplementedError: print( @@ -35,7 +35,7 @@ def run_info(session, buildsystems): print("\t\t\t%s" % dep) print("") try: - outputs = list(buildsystem.get_declared_outputs()) + outputs = list(buildsystem.get_declared_outputs(session)) except NotImplementedError: print("\tUnable to detect declared outputs for this type of build system") outputs = [] diff --git a/ognibuild/outputs.py b/ognibuild/outputs.py index ba1bf85..a39caa9 100644 --- a/ognibuild/outputs.py +++ b/ognibuild/outputs.py @@ -46,3 +46,15 @@ class PythonPackageOutput(UpstreamOutput): self.name, self.python_version, ) + + +class RPackageOutput(UpstreamOutput): + def __init__(self, name): + super(RPackageOutput, self).__init__("r-package") + self.name = name + + def __str__(self): + return "R package: %s" % self.name + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.name) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index e2e5ff6..014bee9 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -17,8 +17,9 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import posixpath +import re import subprocess -from typing import Optional, List, Tuple +from typing import Optional, List, Set from . import Requirement @@ -71,14 +72,39 @@ class PythonPackageRequirement(Requirement): cmd = "python3" else: raise NotImplementedError - text = self.package + ','.join([''.join(spec) for spec in self.specs]) + text = self.package + ",".join(["".join(spec) for spec in self.specs]) p = session.Popen( [cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) p.communicate() return p.returncode == 0 +class PhpPackageRequirement(Requirement): + def __init__( + self, + package: str, + channel: Optional[str] = None, + min_version: Optional[str] = None, + max_version: Optional[str] = None, + ): + self.package = package + self.channel = channel + self.min_version = min_version + self.max_version = max_version + + def __repr__(self): + return "%s(%r, %r, %r, %r)" % ( + type(self).__name__, + self.package, + self.channel, + self.min_version, + self.max_version, + ) + + class BinaryRequirement(Requirement): binary_name: str @@ -87,10 +113,15 @@ class BinaryRequirement(Requirement): super(BinaryRequirement, self).__init__("binary") self.binary_name = binary_name + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.binary_name) + def met(self, session): p = session.Popen( - ["which", self.binary_name], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) + ["which", self.binary_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) p.communicate() return p.returncode == 0 @@ -107,9 +138,47 @@ class PerlModuleRequirement(Requirement): self.filename = filename self.inc = inc + @property def relfilename(self): return self.module.replace("::", "/") + ".pm" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.module) + + +class VagueDependencyRequirement(Requirement): + + name: str + minimum_version: Optional[str] = None + + def __init__(self, name, minimum_version=None): + super(VagueDependencyRequirement, self).__init__("vague") + self.name = name + self.minimum_version = minimum_version + + def expand(self): + if " " not in self.name: + yield BinaryRequirement(self.name) + yield LibraryRequirement(self.name) + yield PkgConfigRequirement(self.name, minimum_version=self.minimum_version) + if self.name.lower() != self.name: + yield BinaryRequirement(self.name.lower()) + yield LibraryRequirement(self.name.lower()) + yield PkgConfigRequirement(self.name.lower(), minimum_version=self.minimum_version) + from .resolver.apt import AptRequirement + + yield AptRequirement.simple(self.name.lower(), minimum_version=self.minimum_version) + yield AptRequirement.simple('lib%s-dev' % self.name.lower(), minimum_version=self.minimum_version) + + def met(self, session): + for x in self.expand(): + if x.met(session): + return True + return False + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.name) + class NodePackageRequirement(Requirement): @@ -119,23 +188,53 @@ class NodePackageRequirement(Requirement): super(NodePackageRequirement, self).__init__("npm-package") self.package = package + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.package) + + +class NodeModuleRequirement(Requirement): + + module: str + + def __init__(self, module): + super(NodeModuleRequirement, self).__init__("npm-module") + self.module = module + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.module) + class CargoCrateRequirement(Requirement): crate: str + features: Set[str] + version: Optional[str] - def __init__(self, crate): + def __init__(self, crate, features=None, version=None): super(CargoCrateRequirement, self).__init__("cargo-crate") self.crate = crate + if features is None: + features = set() + self.features = features + self.version = version def __repr__(self): - return "%s(%r)" % ( + return "%s(%r, features=%r, version=%r)" % ( type(self).__name__, self.crate, + self.features, + self.version, ) def __str__(self): - return "cargo crate: %s" % self.crate + if self.features: + return "cargo crate: %s %s (%s)" % ( + self.crate, + self.version or "", + ", ".join(sorted(self.features)), + ) + else: + return "cargo crate: %s %s" % (self.crate, self.version or "") class PkgConfigRequirement(Requirement): @@ -194,10 +293,29 @@ class RubyGemRequirement(Requirement): class GoPackageRequirement(Requirement): package: str + version: Optional[str] - def __init__(self, package: str): - super(GoPackageRequirement, self).__init__("go") + def __init__(self, package: str, version: Optional[str] = None): + super(GoPackageRequirement, self).__init__("go-package") self.package = package + self.version = version + + def __str__(self): + if self.version: + return "go package: %s (= %s)" % (self.package, self.version) + return "go package: %s" % self.package + + +class GoRequirement(Requirement): + + version: Optional[str] + + def __init__(self, version: Optional[str] = None): + super(GoRequirement, self).__init__("go") + self.version = version + + def __str__(self): + return "go %s" % self.version class DhAddonRequirement(Requirement): @@ -228,6 +346,65 @@ class RPackageRequirement(Requirement): self.package = package self.minimum_version = minimum_version + def __repr__(self): + return "%s(%r, minimum_version=%r)" % ( + type(self).__name__, + self.package, + self.minimum_version, + ) + + def __str__(self): + if self.minimum_version: + return "R package: %s (>= %s)" % (self.package, self.minimum_version) + else: + return "R package: %s" % (self.package,) + + @classmethod + def from_str(cls, text): + # TODO(jelmer): More complex parser + m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text) + if m: + return cls(m.group(1), m.group(2)) + m = re.fullmatch(r"([^ ]+)", text) + if m: + return cls(m.group(1)) + raise ValueError(text) + + +class OctavePackageRequirement(Requirement): + + package: str + minimum_version: Optional[str] + + def __init__(self, package: str, minimum_version: Optional[str] = None): + super(OctavePackageRequirement, self).__init__("octave-package") + self.package = package + self.minimum_version = minimum_version + + def __repr__(self): + return "%s(%r, minimum_version=%r)" % ( + type(self).__name__, + self.package, + self.minimum_version, + ) + + def __str__(self): + if self.minimum_version: + return "Octave package: %s (>= %s)" % (self.package, self.minimum_version) + else: + return "Octave package: %s" % (self.package,) + + @classmethod + def from_str(cls, text): + # TODO(jelmer): More complex parser + m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text) + if m: + return cls(m.group(1), m.group(2)) + m = re.fullmatch(r"([^ ]+)", text) + if m: + return cls(m.group(1)) + raise ValueError(text) + class LibraryRequirement(Requirement): @@ -276,6 +453,15 @@ class JavaClassRequirement(Requirement): self.classname = classname +class CMakefileRequirement(Requirement): + + filename: str + + def __init__(self, filename: str): + super(CMakefileRequirement, self).__init__("cmake-file") + self.filename = filename + + class HaskellPackageRequirement(Requirement): package: str @@ -293,11 +479,43 @@ class HaskellPackageRequirement(Requirement): class MavenArtifactRequirement(Requirement): - artifacts: List[Tuple[str, str, str]] + group_id: str + artifact_id: str + version: Optional[str] + kind: Optional[str] - def __init__(self, artifacts): + def __init__(self, group_id, artifact_id, version=None, kind=None): super(MavenArtifactRequirement, self).__init__("maven-artifact") - self.artifacts = artifacts + self.group_id = group_id + self.artifact_id = artifact_id + self.version = version + self.kind = kind + + def __str__(self): + return "maven requirement: %s:%s:%s" % ( + self.group_id, + self.artifact_id, + self.version, + ) + + @classmethod + def from_str(cls, text): + return cls.from_tuple(text.split(":")) + + @classmethod + def from_tuple(cls, parts): + if len(parts) == 4: + (group_id, artifact_id, kind, version) = parts + elif len(parts) == 3: + (group_id, artifact_id, version) = parts + kind = "jar" + elif len(parts) == 2: + version = None + (group_id, artifact_id) = parts + kind = "jar" + else: + raise ValueError("invalid number of parts to artifact %r" % parts) + return cls(group_id, artifact_id, version, kind) class GnomeCommonRequirement(Requirement): @@ -320,6 +538,32 @@ class JDKFileRequirement(Requirement): return posixpath.join(self.jdk_path, self.filename) +class JDKRequirement(Requirement): + def __init__(self): + super(JDKRequirement, self).__init__("jdk") + + +class JRERequirement(Requirement): + def __init__(self): + super(JRERequirement, self).__init__("jre") + + +class QTRequirement(Requirement): + def __init__(self): + super(QTRequirement, self).__init__("qt") + + +class X11Requirement(Requirement): + def __init__(self): + super(X11Requirement, self).__init__("x11") + + +class CertificateAuthorityRequirement(Requirement): + def __init__(self, url): + super(CertificateAuthorityRequirement, self).__init__("ca-cert") + self.url = url + + class PerlFileRequirement(Requirement): filename: str @@ -338,6 +582,11 @@ class AutoconfMacroRequirement(Requirement): self.macro = macro +class LibtoolRequirement(Requirement): + def __init__(self): + super(LibtoolRequirement, self).__init__("libtool") + + class PythonModuleRequirement(Requirement): module: str @@ -346,6 +595,7 @@ class PythonModuleRequirement(Requirement): def __init__(self, module, python_version=None, minimum_version=None): super(PythonModuleRequirement, self).__init__("python-module") + self.module = module self.python_version = python_version self.minimum_version = minimum_version @@ -364,6 +614,8 @@ class PythonModuleRequirement(Requirement): raise NotImplementedError p = session.Popen( [cmd, "-c", "import %s" % self.module], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) p.communicate() return p.returncode == 0 diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 51e0467..7451481 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -17,6 +17,7 @@ import subprocess +from ..fix_build import run_detecting_problems class UnsatisfiedRequirements(Exception): @@ -34,13 +35,15 @@ class Resolver(object): def explain(self, requirements): raise NotImplementedError(self.explain) - def met(self, requirement): - raise NotImplementedError(self.met) + def env(self): + return {} class CPANResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False, skip_tests=True): self.session = session + self.user_local = user_local + self.skip_tests = skip_tests def __str__(self): return "cpan" @@ -48,6 +51,13 @@ class CPANResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def _cmd(self, reqs): + ret = ["cpan", "-i"] + if self.skip_tests: + ret.append("-T") + ret.extend([req.module for req in reqs]) + return ret + def explain(self, requirements): from ..requirements import PerlModuleRequirement @@ -57,28 +67,148 @@ class CPANResolver(Resolver): continue perlreqs.append(requirement) if perlreqs: - yield (["cpan", "-i"] + [req.module for req in perlreqs], [perlreqs]) + yield (self._cmd(perlreqs), [perlreqs]) def install(self, requirements): from ..requirements import PerlModuleRequirement + env = { + "PERL_MM_USE_DEFAULT": "1", + "PERL_MM_OPT": "", + "PERL_MB_OPT": "", + } + + if not self.user_local: + user = "root" + else: + user = None + missing = [] for requirement in requirements: if not isinstance(requirement, PerlModuleRequirement): missing.append(requirement) continue - # TODO(jelmer): Specify -T to skip tests? - self.session.check_call( - ["cpan", "-i", requirement.module], - env={"PERL_MM_USE_DEFAULT": "1"}, + run_detecting_problems( + self.session, + self._cmd([requirement]), + env=env, + user=user, ) if missing: raise UnsatisfiedRequirements(missing) -class HackageResolver(Resolver): - def __init__(self, session): +class RResolver(Resolver): + def __init__(self, session, repos, user_local=False): self.session = session + self.repos = repos + self.user_local = user_local + + def __str__(self): + return "cran" + + def __repr__(self): + return "%s(%r, %r)" % (type(self).__name__, self.session, self.repos) + + def _cmd(self, req): + # TODO(jelmer: Handle self.user_local + return [ + "R", + "-e", + "install.packages('%s', repos=%r)" % (req.package, self.repos), + ] + + def explain(self, requirements): + from ..requirements import RPackageRequirement + + rreqs = [] + for requirement in requirements: + if not isinstance(requirement, RPackageRequirement): + continue + rreqs.append(requirement) + if rreqs: + yield ([self._cmd(req) for req in rreqs]) + + def install(self, requirements): + from ..requirements import RPackageRequirement + + if self.user_local: + user = None + else: + user = "root" + + missing = [] + for requirement in requirements: + if not isinstance(requirement, RPackageRequirement): + missing.append(requirement) + continue + self.session.check_call(self._cmd(requirement), user=user) + if missing: + raise UnsatisfiedRequirements(missing) + + +class OctaveForgeResolver(Resolver): + def __init__(self, session, user_local=False): + self.session = session + self.user_local = user_local + + def __str__(self): + return "octave-forge" + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + + def _cmd(self, req): + # TODO(jelmer: Handle self.user_local + return ["octave-cli", "--eval", "pkg install -forge %s" % req.package] + + def explain(self, requirements): + from ..requirements import OctavePackageRequirement + + rreqs = [] + for requirement in requirements: + if not isinstance(requirement, OctavePackageRequirement): + continue + rreqs.append(requirement) + if rreqs: + yield ([self._cmd(req) for req in rreqs]) + + def install(self, requirements): + from ..requirements import OctavePackageRequirement + + if self.user_local: + user = None + else: + user = "root" + + missing = [] + for requirement in requirements: + if not isinstance(requirement, OctavePackageRequirement): + missing.append(requirement) + continue + self.session.check_call(self._cmd(requirement), user=user) + if missing: + raise UnsatisfiedRequirements(missing) + + +class CRANResolver(RResolver): + def __init__(self, session, user_local=False): + super(CRANResolver, self).__init__( + session, "http://cran.r-project.org", user_local=user_local + ) + + +class BioconductorResolver(RResolver): + def __init__(self, session, user_local=False): + super(BioconductorResolver, self).__init__( + session, "https://hedgehog.fhcrc.org/bioconductor", user_local=user_local + ) + + +class HackageResolver(Resolver): + def __init__(self, session, user_local=False): + self.session = session + self.user_local = user_local def __str__(self): return "hackage" @@ -86,17 +216,26 @@ class HackageResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def _cmd(self, reqs): + extra_args = [] + if self.user_local: + extra_args.append("--user") + return ["cabal", "install"] + extra_args + [req.package for req in reqs] + def install(self, requirements): from ..requirements import HaskellPackageRequirement + if self.user_local: + user = None + else: + user = "root" + missing = [] for requirement in requirements: if not isinstance(requirement, HaskellPackageRequirement): missing.append(requirement) continue - self.session.check_call( - ["cabal", "install", requirement.package] - ) + self.session.check_call(self._cmd([requirement]), user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -109,13 +248,13 @@ class HackageResolver(Resolver): continue haskellreqs.append(requirement) if haskellreqs: - yield (["cabal", "install"] + [req.package for req in haskellreqs], - haskellreqs) + yield (self._cmd(haskellreqs), haskellreqs) class PypiResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False): self.session = session + self.user_local = user_local def __str__(self): return "pypi" @@ -123,17 +262,27 @@ class PypiResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def _cmd(self, reqs): + extra_args = [] + if self.user_local: + extra_args.append("--user") + return ["pip", "install"] + extra_args + [req.package for req in reqs] + def install(self, requirements): from ..requirements import PythonPackageRequirement + if self.user_local: + user = None + else: + user = "root" + missing = [] for requirement in requirements: if not isinstance(requirement, PythonPackageRequirement): missing.append(requirement) continue try: - self.session.check_call( - ["pip", "install", requirement.package]) + self.session.check_call(self._cmd([requirement]), user=user) except subprocess.CalledProcessError: missing.append(requirement) if missing: @@ -148,14 +297,13 @@ class PypiResolver(Resolver): continue pyreqs.append(requirement) if pyreqs: - yield (["pip", "install"] + [req.package for req in pyreqs], - pyreqs) + yield (self._cmd(pyreqs), pyreqs) class GoResolver(Resolver): - - def __init__(self, session): + def __init__(self, session, user_local): self.session = session + self.user_local = user_local def __str__(self): return "go" @@ -166,12 +314,18 @@ class GoResolver(Resolver): def install(self, requirements): from ..requirements import GoPackageRequirement + if self.user_local: + env = {} + else: + # TODO(jelmer): Isn't this Debian-specific? + env = {"GOPATH": "/usr/share/gocode"} + missing = [] for requirement in requirements: if not isinstance(requirement, GoPackageRequirement): missing.append(requirement) continue - self.session.check_call(["go", "get", requirement.package]) + self.session.check_call(["go", "get", requirement.package], env=env) if missing: raise UnsatisfiedRequirements(missing) @@ -184,18 +338,20 @@ class GoResolver(Resolver): continue goreqs.append(requirement) if goreqs: - yield (["go", "get"] + [req.package for req in goreqs], - goreqs) + yield (["go", "get"] + [req.package for req in goreqs], goreqs) NPM_COMMAND_PACKAGES = { "del-cli": "del-cli", + "husky": "husky", } class NpmResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False): self.session = session + self.user_local = user_local + # TODO(jelmer): Handle user_local def __str__(self): return "npm" @@ -204,19 +360,35 @@ class NpmResolver(Resolver): return "%s(%r)" % (type(self).__name__, self.session) def install(self, requirements): - from ..requirements import NodePackageRequirement + from ..requirements import ( + NodePackageRequirement, + NodeModuleRequirement, + BinaryRequirement, + ) + + if self.user_local: + user = None + else: + user = "root" missing = [] for requirement in requirements: + if isinstance(requirement, BinaryRequirement): + try: + package = NPM_COMMAND_PACKAGES[requirement.binary_name] + except KeyError: + pass + else: + requirement = NodePackageRequirement(package) + if isinstance(requirement, NodeModuleRequirement): + # TODO: Is this legit? + requirement = NodePackageRequirement(requirement.module.split("/")[0]) if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - try: - package = NPM_COMMAND_PACKAGES[requirement.command] - except KeyError: - missing.append(requirement) - continue - self.session.check_call(["npm", "-g", "install", package]) + self.session.check_call( + ["npm", "-g", "install", requirement.package], user=user + ) if missing: raise UnsatisfiedRequirements(missing) @@ -248,6 +420,13 @@ class StackedResolver(Resolver): def __str__(self): return "[" + ", ".join(map(str, self.subs)) + "]" + def env(self): + ret = {} + # Reversed so earlier resolvers override later ones + for sub in reversed(self.subs): + ret.update(sub.env()) + return ret + def explain(self, requirements): for sub in self.subs: yield from sub.explain(requirements) @@ -260,6 +439,8 @@ class StackedResolver(Resolver): requirements = e.requirements else: return + if requirements: + raise UnsatisfiedRequirements(requirements) NATIVE_RESOLVER_CLS = [ @@ -268,35 +449,31 @@ NATIVE_RESOLVER_CLS = [ NpmResolver, GoResolver, HackageResolver, - ] + CRANResolver, + BioconductorResolver, + OctaveForgeResolver, +] -def native_resolvers(session): - return StackedResolver([kls(session) for kls in NATIVE_RESOLVER_CLS]) +def native_resolvers(session, user_local): + return StackedResolver([kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) -class ExplainResolver(Resolver): - def __init__(self, session): - self.session = session - - @classmethod - def from_session(cls, session): - return cls(session) - - def install(self, requirements): - raise UnsatisfiedRequirements(requirements) - - -def auto_resolver(session): +def auto_resolver(session, explain=False): # if session is SchrootSession or if we're root, use apt from .apt import AptResolver from ..session.schroot import SchrootSession + from ..session import get_user - user = session.check_output(["echo", "$USER"]).decode().strip() + user = get_user(session) resolvers = [] # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if # present? - if isinstance(session, SchrootSession) or user == "root": + if isinstance(session, SchrootSession) or user == "root" or explain: + user_local = False + else: + user_local = True + if not user_local: resolvers.append(AptResolver.from_session(session)) - resolvers.extend([kls(session) for kls in NATIVE_RESOLVER_CLS]) + resolvers.extend([kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) return StackedResolver(resolvers) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 08d95a4..618206c 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -19,6 +19,8 @@ from itertools import chain import logging import os import posixpath +import re +from typing import Optional, List from debian.changelog import Version from debian.deb822 import PkgRelation @@ -28,6 +30,7 @@ from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements from ..requirements import ( Requirement, + CargoCrateRequirement, BinaryRequirement, CHeaderRequirement, PkgConfigRequirement, @@ -36,30 +39,43 @@ from ..requirements import ( ValaPackageRequirement, RubyGemRequirement, GoPackageRequirement, + GoRequirement, DhAddonRequirement, PhpClassRequirement, + PhpPackageRequirement, RPackageRequirement, + NodeModuleRequirement, NodePackageRequirement, LibraryRequirement, RubyFileRequirement, XmlEntityRequirement, SprocketsFileRequirement, JavaClassRequirement, + CMakefileRequirement, HaskellPackageRequirement, MavenArtifactRequirement, GnomeCommonRequirement, JDKFileRequirement, + JDKRequirement, + JRERequirement, + QTRequirement, + X11Requirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, + CertificateAuthorityRequirement, + LibtoolRequirement, + VagueDependencyRequirement, ) class AptRequirement(Requirement): def __init__(self, relations): super(AptRequirement, self).__init__("apt") + if not isinstance(relations, list): + raise TypeError(relations) self.relations = relations @classmethod @@ -76,17 +92,53 @@ class AptRequirement(Requirement): def pkg_relation_str(self): return PkgRelation.str(self.relations) + def __hash__(self): + return hash((type(self), self.pkg_relation_str())) + + def __eq__(self, other): + return isinstance(self, type(other)) and self.relations == other.relations + def __str__(self): return "apt requirement: %s" % self.pkg_relation_str() - def touches_package(self, package): + def __repr__(self): + return "%s.from_str(%r)" % (type(self).__name__, self.pkg_relation_str()) + + def package_names(self): for rel in self.relations: for entry in rel: - if entry["name"] == package: - return True + yield entry["name"] + + def touches_package(self, package): + for name in self.package_names(): + if name == package: + return True return False +def find_package_names( + apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False +) -> List[str]: + if not isinstance(paths, list): + raise TypeError(paths) + return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) + + +def find_reqs_simple( + apt_mgr: AptManager, + paths: List[str], + regex: bool = False, + minimum_version=None, + case_insensitive=False, +) -> List[str]: + if not isinstance(paths, list): + raise TypeError(paths) + return [ + AptRequirement.simple(package, minimum_version=minimum_version) + for package in find_package_names(apt_mgr, paths, regex, case_insensitive) + ] + + def python_spec_to_apt_rels(pkg_name, specs): # TODO(jelmer): Dealing with epoch, etc? if not specs: @@ -94,99 +146,153 @@ def python_spec_to_apt_rels(pkg_name, specs): else: rels = [] for spec in specs: - c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "=": "="}[spec[0]] - rels.append([{"name": pkg_name, "version": (c, Version(spec[1]))}]) + deb_version = Version(spec[1]) + if spec[0] == "~=": + # PEP 440: For a given release identifier V.N , the compatible + # release clause is approximately equivalent to the pair of + # comparison clauses: >= V.N, == V.* + parts = spec[1].split(".") + parts.pop(-1) + parts[-1] = str(int(parts[-1]) + 1) + next_maj_deb_version = Version(".".join(parts)) + rels.extend( + [ + {"name": pkg_name, "version": (">=", deb_version)}, + {"name": pkg_name, "version": ("<<", next_maj_deb_version)}, + ] + ) + elif spec[0] == "!=": + rels.extend( + [ + {"name": pkg_name, "version": (">>", deb_version)}, + {"name": pkg_name, "version": ("<<", deb_version)}, + ] + ) + elif spec[1].endswith(".*") and spec[0] == "==": + s = spec[1].split(".") + s.pop(-1) + n = list(s) + n[-1] = str(int(n[-1]) + 1) + rels.extend( + [ + {"name": pkg_name, "version": (">=", Version(".".join(s)))}, + {"name": pkg_name, "version": ("<<", Version(".".join(n)))}, + ] + ) + else: + c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] + rels.append([{"name": pkg_name, "version": (c, deb_version)}]) return rels -def get_package_for_python_package(apt_mgr, package, python_version, specs=None): +def get_package_for_python_package( + apt_mgr, package, python_version: Optional[str], specs=None +): + pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape( + package.replace("-", "_") + ) + cpython2_regex = ( + "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" + % re.escape(package.replace("-", "_")) + ) + cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape( + package.replace("-", "_") + ) if python_version == "pypy": - pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % package.replace("-", "_")], - regex=True, - ) + paths = [pypy_regex] elif python_version == "cpython2": - pkg_name = apt_mgr.get_package_for_paths( - [ - "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" - % package.replace("-", "_") - ], - regex=True, - ) + paths = [cpython2_regex] elif python_version == "cpython3": - pkg_name = apt_mgr.get_package_for_paths( - [ - "/usr/lib/python3/dist-packages/%s-.*.egg-info" - % package.replace("-", "_") - ], - regex=True, - ) + paths = [cpython3_regex] + elif python_version is None: + paths = [cpython3_regex, cpython2_regex, pypy_regex] else: - raise NotImplementedError - if pkg_name is None: - return None - rels = python_spec_to_apt_rels(pkg_name, specs) - return AptRequirement(rels) + raise NotImplementedError("unsupported python version %s" % python_version) + names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True) + return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] def get_package_for_python_module(apt_mgr, module, python_version, specs): - if python_version == "python3": - paths = [ - posixpath.join( - "/usr/lib/python3/dist-packages", - module.replace(".", "/"), - "__init__.py", - ), - posixpath.join( - "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py" - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/lib-dynload", - module.replace(".", "/") + "\\.cpython-.*\\.so", - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py" - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py" - ), - ] - elif python_version == "python2": - paths = [ - posixpath.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - module.replace(".", "/"), - "__init__.py", - ), - posixpath.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - module.replace(".", "/") + ".py", - ), - posixpath.join( - "/usr/lib/python2.\\.[0-9]/lib-dynload", - module.replace(".", "/") + ".so", - ), - ] + cpython3_regexes = [ + posixpath.join( + "/usr/lib/python3/dist-packages", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + posixpath.join( + "/usr/lib/python3/dist-packages", + re.escape(module.replace(".", "/")) + ".py", + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/lib-dynload", + re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + ] + cpython2_regexes = [ + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist-packages", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist-packages", + re.escape(module.replace(".", "/")) + ".py", + ), + posixpath.join( + "/usr/lib/python2.\\.[0-9]/lib-dynload", + re.escape(module.replace(".", "/")) + ".so", + ), + ] + pypy_regexes = [ + posixpath.join( + "/usr/lib/pypy/dist-packages", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + posixpath.join( + "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" + ), + posixpath.join( + "/usr/lib/pypy/dist-packages", + re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", + ), + ] + if python_version == "cpython3": + paths = cpython3_regexes + elif python_version == "cpython2": + paths = cpython2_regexes elif python_version == "pypy": - paths = [ - posixpath.join( - "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py" - ), - posixpath.join( - "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py" - ), - posixpath.join( - "/usr/lib/pypy/dist-packages", - module.replace(".", "/") + "\\.pypy-.*\\.so", - ), - ] + paths = pypy_regexes + elif python_version is None: + paths = cpython3_regexes + cpython2_regexes + pypy_regexes else: raise AssertionError("unknown python version %r" % python_version) - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is None: - return None - rels = python_spec_to_apt_rels(pkg_name, specs) - return AptRequirement(rels) + names = find_package_names(apt_mgr, paths, regex=True) + return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] + + +vague_map = { + "the Gnu Scientific Library": "libgsl-dev", + "the required FreeType library": "libfreetype-dev", +} + + +def resolve_vague_dep_req(apt_mgr, req): + name = req.name + options = [] + if name in vague_map: + options.append(AptRequirement.simple(vague_map[name])) + for x in req.expand(): + options.extend(resolve_requirement_apt(apt_mgr, x)) + return options def resolve_binary_req(apt_mgr, req): @@ -196,149 +302,142 @@ def resolve_binary_req(apt_mgr, req): paths = [ posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] ] - pkg_name = apt_mgr.get_package_for_paths(paths) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths) def resolve_pkg_config_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")], + names = find_package_names( + apt_mgr, + [ + posixpath.join( + "/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc" + ) + ], + regex=True, ) - if package is None: - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], - regex=True, + if not names: + names = find_package_names( + apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")] ) - if package is not None: - return AptRequirement.simple(package, minimum_version=req.minimum_version) - return None + return [ + AptRequirement.simple(name, minimum_version=req.minimum_version) + for name in names + ] def resolve_path_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths([req.path]) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, [req.path]) def resolve_c_header_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/include", req.header)], regex=False + reqs = find_reqs_simple( + apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False ) - if package is None: - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/include", ".*", req.header)], regex=True + if not reqs: + reqs = find_reqs_simple( + apt_mgr, + [posixpath.join("/usr/include", ".*", re.escape(req.header))], + regex=True, ) - if package is None: - return None - return AptRequirement.simple(package) + return reqs def resolve_js_runtime_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - ["/usr/bin/node", "/usr/bin/duk"], regex=False - ) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, ["/usr/bin/node", "/usr/bin/duk"]) def resolve_vala_package_req(apt_mgr, req): - path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package - package = apt_mgr.get_package_for_paths([path], regex=True) - if package is not None: - return AptRequirement.simple(package) - return None + path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package) + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_ruby_gem_req(apt_mgr, req): paths = [ posixpath.join( "/usr/share/rubygems-integration/all/" - "specifications/%s-.*\\.gemspec" % req.gem + "specifications/%s-.*\\.gemspec" % re.escape(req.gem) ) ] - package = apt_mgr.get_package_for_paths(paths, regex=True) - if package is not None: - return AptRequirement.simple(package, minimum_version=req.minimum_version) - return None + return find_reqs_simple( + apt_mgr, paths, regex=True, minimum_version=req.minimum_version + ) def resolve_go_package_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/share/gocode/src", req.package, ".*")], regex=True + return find_reqs_simple( + apt_mgr, + [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], + regex=True, ) - if package is not None: - return AptRequirement.simple(package) - return None + + +def resolve_go_req(apt_mgr, req): + return [AptRequirement.simple("golang-go", minimum_version="2:%s" % req.version)] def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] - package = apt_mgr.get_package_for_paths(paths) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, paths) def resolve_php_class_req(apt_mgr, req): path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") - package = apt_mgr.get_package_for_paths([path]) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, [path]) + + +def resolve_php_package_req(apt_mgr, req): + return [ + AptRequirement.simple("php-%s" % req.package, minimum_version=req.min_version) + ] def resolve_r_package_req(apt_mgr, req): - paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)] - package = apt_mgr.get_package_for_paths(paths, regex=True) - if package is not None: - return AptRequirement.simple(package) - return None + paths = [ + posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package)) + ] + return find_reqs_simple(apt_mgr, paths, regex=True) + + +def resolve_node_module_req(apt_mgr, req): + paths = [ + "/usr/share/nodejs/.*/node_modules/%s/index.js" % re.escape(req.module), + "/usr/lib/nodejs/%s/index.js" % re.escape(req.module), + "/usr/share/nodejs/%s/index.js" % re.escape(req.module), + ] + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_node_package_req(apt_mgr, req): paths = [ - "/usr/share/nodejs/.*/node_modules/%s/package.json" % req.package, - "/usr/lib/nodejs/%s/package.json" % req.package, - "/usr/share/nodejs/%s/package.json" % req.package, + "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % re.escape(req.package), + "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), + "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_library_req(apt_mgr, req): paths = [ - posixpath.join("/usr/lib/lib%s.so$" % req.library), - posixpath.join("/usr/lib/.*/lib%s.so$" % req.library), - posixpath.join("/usr/lib/lib%s.a$" % req.library), - posixpath.join("/usr/lib/.*/lib%s.a$" % req.library), + posixpath.join("/usr/lib/lib%s.so$" % re.escape(req.library)), + posixpath.join("/usr/lib/.*/lib%s.so$" % re.escape(req.library)), + posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), + posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] - package = apt_mgr.get_package_for_paths(paths) - if package is not None: - return AptRequirement.simple(package) + reqs = find_reqs_simple(apt_mgr, paths, regex=False) + if reqs: + return reqs paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" - "lib/%s.rb" % req.filename + "lib/%s\\.rb" % re.escape(req.filename) ) ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_xml_entity_req(apt_mgr, req): @@ -354,28 +453,21 @@ def resolve_xml_entity_req(apt_mgr, req): else: return None - pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [search_path], regex=False) def resolve_sprockets_file_req(apt_mgr, req): if req.content_type == "application/javascript": - path = "/usr/share/.*/app/assets/javascripts/%s.js$" % req.name + path = "/usr/share/.*/app/assets/javascripts/%s\\.js$" % re.escape(req.name) else: logging.warning("unable to handle content type %s", req.content_type) return None - pkg_name = apt_mgr.get_package_for_paths([path], regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_java_class_req(apt_mgr, req): # Unfortunately this only finds classes in jars installed on the host # system :( - # TODO(jelmer): Call in session output = apt_mgr.session.check_output( ["java-propose-classpath", "-c" + req.classname] ) @@ -384,63 +476,70 @@ def resolve_java_class_req(apt_mgr, req): logging.warning("unable to find classpath for %s", req.classname) return False logging.info("Classpath for %s: %r", req.classname, classpath) - package = apt_mgr.get_package_for_paths(classpath) - if package is None: - logging.warning("no package for files in %r", classpath) - return None - return AptRequirement.simple(package) + return find_reqs_simple(apt_mgr, [classpath]) + + +def resolve_cmake_file_req(apt_mgr, req): + paths = ['/usr/lib/.*/cmake/.*/%s' % re.escape(req.filename)] + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_haskell_package_req(apt_mgr, req): - path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0] - pkg_name = apt_mgr.get_package_for_paths([path], regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_maven_artifact_req(apt_mgr, req): - artifact = req.artifacts[0] - parts = artifact.split(":") - if len(parts) == 4: - (group_id, artifact_id, kind, version) = parts - regex = False - elif len(parts) == 3: - (group_id, artifact_id, version) = parts - kind = "jar" - regex = False - elif len(parts) == 2: + if req.version is None: version = ".*" - (group_id, artifact_id) = parts - kind = "jar" regex = True + escape = re.escape else: - raise AssertionError("invalid number of parts to artifact %s" % artifact) - paths = [ - posixpath.join( - "/usr/share/maven-repo", - group_id.replace(".", "/"), - artifact_id, - version, - "%s-%s.%s" % (artifact_id, version, kind), - ) - ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + version = req.version + regex = False + + def escape(x): + return x + + kind = req.kind or "jar" + path = posixpath.join( + escape("/usr/share/maven-repo"), + escape(req.group_id.replace(".", "/")), + escape(req.artifact_id), + version, + escape("%s-") + version + escape("." + kind), + ) + + return find_reqs_simple(apt_mgr, [path], regex=regex) def resolve_gnome_common_req(apt_mgr, req): - return AptRequirement.simple("gnome-common") + return [AptRequirement.simple("gnome-common")] def resolve_jdk_file_req(apt_mgr, req): - path = req.jdk_path + ".*/" + req.filename - pkg_name = apt_mgr.get_package_for_paths([path], regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename) + return find_reqs_simple(apt_mgr, [path], regex=True) + + +def resolve_jdk_req(apt_mgr, req): + return [AptRequirement.simple("default-jdk")] + + +def resolve_jre_req(apt_mgr, req): + return [AptRequirement.simple("default-jre")] + + +def resolve_x11_req(apt_mgr, req): + return [AptRequirement.simple("libx11-dev")] + + +def resolve_qt_req(apt_mgr, req): + return find_reqs_simple(apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) + + +def resolve_libtool_req(apt_mgr, req): + return [AptRequirement.simple("libtool")] def resolve_perl_module_req(apt_mgr, req): @@ -455,28 +554,24 @@ def resolve_perl_module_req(apt_mgr, req): paths = [req.filename] else: paths = [posixpath.join(inc, req.filename) for inc in req.inc] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=False) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=False) def resolve_perl_file_req(apt_mgr, req): - pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [req.filename], regex=False) def _find_aclocal_fun(macro): # TODO(jelmer): Use the API for codesearch.debian.net instead? defun_prefix = b"AC_DEFUN([%s]," % macro.encode("ascii") + au_alias_prefix = b"AU_ALIAS([%s]," % macro.encode("ascii") + prefixes = [defun_prefix, au_alias_prefix] for entry in os.scandir("/usr/share/aclocal"): if not entry.is_file(): continue with open(entry.path, "rb") as f: for line in f: - if line.startswith(defun_prefix): + if any([line.startswith(prefix) for prefix in prefixes]): return entry.path raise KeyError @@ -487,17 +582,18 @@ def resolve_autoconf_macro_req(apt_mgr, req): except KeyError: logging.info("No local m4 file found defining %s", req.macro) return None - pkg_name = apt_mgr.get_package_for_paths([path]) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [path]) def resolve_python_module_req(apt_mgr, req): + if req.minimum_version: + specs = [(">=", req.minimum_version)] + else: + specs = [] if req.python_version == 2: - return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.specs) + return get_package_for_python_module(apt_mgr, req.module, "cpython2", specs) elif req.python_version in (None, 3): - return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.specs) + return get_package_for_python_module(apt_mgr, req.module, "cpython3", specs) else: return None @@ -515,8 +611,27 @@ def resolve_python_package_req(apt_mgr, req): return None +def resolve_cargo_crate_req(apt_mgr, req): + paths = ["/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml" % re.escape(req.crate)] + return find_reqs_simple(apt_mgr, paths, regex=True) + + +def resolve_ca_req(apt_mgr, req): + return [AptRequirement.simple("ca-certificates")] + + +def resolve_apt_req(apt_mgr, req): + # TODO(jelmer): This should be checking whether versions match as well. + for package_name in req.package_names(): + if not apt_mgr.package_exists(package_name): + return [] + return [req] + + APT_REQUIREMENT_RESOLVERS = [ + (AptRequirement, resolve_apt_req), (BinaryRequirement, resolve_binary_req), + (VagueDependencyRequirement, resolve_vague_dep_req), (PkgConfigRequirement, resolve_pkg_config_req), (PathRequirement, resolve_path_req), (CHeaderRequirement, resolve_c_header_req), @@ -524,44 +639,66 @@ APT_REQUIREMENT_RESOLVERS = [ (ValaPackageRequirement, resolve_vala_package_req), (RubyGemRequirement, resolve_ruby_gem_req), (GoPackageRequirement, resolve_go_package_req), + (GoRequirement, resolve_go_req), (DhAddonRequirement, resolve_dh_addon_req), (PhpClassRequirement, resolve_php_class_req), + (PhpPackageRequirement, resolve_php_package_req), (RPackageRequirement, resolve_r_package_req), + (NodeModuleRequirement, resolve_node_module_req), (NodePackageRequirement, resolve_node_package_req), (LibraryRequirement, resolve_library_req), (RubyFileRequirement, resolve_ruby_file_req), (XmlEntityRequirement, resolve_xml_entity_req), (SprocketsFileRequirement, resolve_sprockets_file_req), (JavaClassRequirement, resolve_java_class_req), + (CMakefileRequirement, resolve_cmake_file_req), (HaskellPackageRequirement, resolve_haskell_package_req), (MavenArtifactRequirement, resolve_maven_artifact_req), (GnomeCommonRequirement, resolve_gnome_common_req), (JDKFileRequirement, resolve_jdk_file_req), + (JDKRequirement, resolve_jdk_req), + (JRERequirement, resolve_jre_req), + (QTRequirement, resolve_qt_req), + (X11Requirement, resolve_x11_req), + (LibtoolRequirement, resolve_libtool_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), (AutoconfMacroRequirement, resolve_autoconf_macro_req), (PythonModuleRequirement, resolve_python_module_req), (PythonPackageRequirement, resolve_python_package_req), + (CertificateAuthorityRequirement, resolve_ca_req), + (CargoCrateRequirement, resolve_cargo_crate_req), ] -def resolve_requirement_apt(apt_mgr, req: Requirement) -> AptRequirement: +def resolve_requirement_apt(apt_mgr, req: Requirement) -> List[AptRequirement]: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): - return rr_fn(apt_mgr, req) + ret = rr_fn(apt_mgr, req) + if not ret: + return [] + if not isinstance(ret, list): + raise TypeError(ret) + return ret raise NotImplementedError(type(req)) class AptResolver(Resolver): - def __init__(self, apt): + def __init__(self, apt, tie_breakers=None): self.apt = apt + if tie_breakers is None: + tie_breakers = [] + self.tie_breakers = tie_breakers def __str__(self): return "apt" + def __repr__(self): + return "%s(%r, %r)" % (type(self).__name__, self.apt, self.tie_breakers) + @classmethod - def from_session(cls, session): - return cls(AptManager(session)) + def from_session(cls, session, tie_breakers=None): + return cls(AptManager.from_session(session), tie_breakers=tie_breakers) def install(self, requirements): missing = [] @@ -595,7 +732,29 @@ class AptResolver(Resolver): if apt_req is not None: apt_requirements.append((r, apt_req)) if apt_requirements: - yield (["apt", "satisfy"] + [PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))], [o for o, r in apt_requirements]) + yield ( + self.apt.satisfy_command( + [ + PkgRelation.str( + chain(*[r.relations for o, r in apt_requirements]) + ) + ] + ), + [o for o, r in apt_requirements], + ) def resolve(self, req: Requirement): - return resolve_requirement_apt(self.apt, req) + ret = resolve_requirement_apt(self.apt, req) + if not ret: + return None + if len(ret) == 1: + return ret[0] + logging.info("Need to break tie between %r with %r", ret, self.tie_breakers) + for tie_breaker in self.tie_breakers: + winner = tie_breaker(ret) + if winner is not None: + if not isinstance(winner, AptRequirement): + raise TypeError(winner) + return winner + logging.info("Unable to break tie over %r, picking first: %r", ret, ret[0]) + return ret[0] diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 7cdd15f..464381d 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -16,11 +16,25 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from typing import Optional, List, Dict +from typing import Optional, List, Dict, Tuple import sys import subprocess +class NoSessionOpen(Exception): + """There is no session open.""" + + def __init__(self, session): + self.session = session + + +class SessionAlreadyOpen(Exception): + """There is already a session open.""" + + def __init__(self, session): + self.session = session + + class Session(object): def __enter__(self) -> "Session": return self @@ -41,6 +55,7 @@ class Session(object): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, + close_fds: bool = True, ): raise NotImplementedError(self.check_call) @@ -74,12 +89,27 @@ class Session(object): def scandir(self, path: str): raise NotImplementedError(self.scandir) + def setup_from_vcs( + self, tree, include_controldir: Optional[bool] = None, subdir="package" + ) -> Tuple[str, str]: + raise NotImplementedError(self.setup_from_vcs) + + def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]: + raise NotImplementedError(self.setup_from_directory) + + def external_path(self, path: str) -> str: + raise NotImplementedError + + is_temporary: bool + class SessionSetupFailure(Exception): """Session failed to be set up.""" def run_with_tee(session: Session, args: List[str], **kwargs): + if "stdin" not in kwargs: + kwargs["stdin"] = subprocess.DEVNULL p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) contents = [] while p.poll() is None: @@ -88,3 +118,19 @@ def run_with_tee(session: Session, args: List[str], **kwargs): sys.stdout.buffer.flush() contents.append(line.decode("utf-8", "surrogateescape")) return p.returncode, contents + + +def get_user(session): + return session.check_output(["echo", "$USER"], cwd="/").decode().strip() + + +def which(session, name): + try: + ret = session.check_output(["which", name], cwd="/").decode().strip() + except subprocess.CalledProcessError as e: + if e.returncode == 1: + return None + raise + if not ret: + return None + return ret diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index b1f237c..0084d8d 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -16,10 +16,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from . import Session +from . import Session, NoSessionOpen, SessionAlreadyOpen +import contextlib import os import subprocess +import tempfile +from typing import Optional, Dict, List class PlainSession(Session): @@ -27,20 +30,63 @@ class PlainSession(Session): location = "/" + def __init__(self): + self.es = None + + def _prepend_user(self, user, args): + if self.es is None: + raise NoSessionOpen(self) + if user is not None: + import getpass + + if user != getpass.getuser(): + args = ["sudo", "-u", user] + args + return args + def __repr__(self): - return "%s()" % (type(self).__name__, ) + return "%s()" % (type(self).__name__,) + + def __enter__(self) -> "Session": + if self.es is not None: + raise SessionAlreadyOpen(self) + self.es = contextlib.ExitStack() + self.es.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.es is None: + raise NoSessionOpen(self) + self.es.__exit__(exc_type, exc_val, exc_tb) + self.es = None + return False def create_home(self): pass - def check_call(self, args): - return subprocess.check_call(args) + def check_call( + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + close_fds: bool = True, + ): + argv = self._prepend_user(user, argv) + return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds) - def check_output(self, args): - return subprocess.check_output(args) + def check_output( + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ) -> bytes: + argv = self._prepend_user(user, argv) + return subprocess.check_output(argv, cwd=cwd, env=env) - def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): - return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) + def Popen(self, args, stdout=None, stderr=None, stdin=None, user=None, cwd=None, env=None): + args = self._prepend_user(user, args) + return subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, cwd=cwd, env=env) def exists(self, path): return os.path.exists(path) @@ -50,3 +96,27 @@ class PlainSession(Session): def chdir(self, path): os.chdir(path) + + def external_path(self, path): + return os.path.abspath(path) + + def setup_from_vcs(self, tree, include_controldir=None, subdir="package"): + from ..vcs import dupe_vcs_tree, export_vcs_tree + + if include_controldir is False or ( + not hasattr(tree, "base") and include_controldir is None + ): + td = self.es.enter_context(tempfile.TemporaryDirectory()) + export_vcs_tree(tree, td) + return td, td + elif not hasattr(tree, "base"): + td = self.es.enter_context(tempfile.TemporaryDirectory()) + dupe_vcs_tree(tree, td) + return td, td + else: + return tree.base, tree.base + + def setup_from_directory(self, path): + return path, path + + is_temporary = False diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 3677262..f7f45b2 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -19,11 +19,12 @@ import logging import os import shlex import subprocess +import tempfile from typing import Optional, List, Dict -from . import Session, SessionSetupFailure +from . import Session, SessionSetupFailure, NoSessionOpen, SessionAlreadyOpen class SchrootSession(Session): @@ -31,6 +32,7 @@ class SchrootSession(Session): _cwd: Optional[str] _location: Optional[str] chroot: str + session_id: Optional[str] def __init__(self, chroot: str): if not isinstance(chroot, str): @@ -38,8 +40,11 @@ class SchrootSession(Session): self.chroot = chroot self._location = None self._cwd = None + self.session_id = None def _get_location(self) -> str: + if self.session_id is None: + raise NoSessionOpen(self) return ( subprocess.check_output( ["schroot", "--location", "-c", "session:" + self.session_id] @@ -48,10 +53,29 @@ class SchrootSession(Session): .decode() ) - def _end_session(self) -> None: - subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"]) + def _end_session(self) -> bool: + if self.session_id is None: + raise NoSessionOpen(self) + try: + subprocess.check_output( + ["schroot", "-c", "session:" + self.session_id, "-e"], + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as e: + for line in e.stderr.splitlines(False): + if line.startswith(b"E: "): + logging.error("%s", line[3:].decode(errors="replace")) + logging.warning( + "Failed to close schroot session %s, leaving stray.", self.session_id + ) + self.session_id = None + return False + self.session_id = None + return True def __enter__(self) -> "Session": + if self.session_id is not None: + raise SessionAlreadyOpen(self) try: self.session_id = ( subprocess.check_output(["schroot", "-c", self.chroot, "-b"]) @@ -86,6 +110,8 @@ class SchrootSession(Session): user: Optional[str] = None, env: Optional[Dict[str, str]] = None, ): + if self.session_id is None: + raise NoSessionOpen(self) base_argv = ["schroot", "-r", "-c", "session:" + self.session_id] if cwd is None: cwd = self._cwd @@ -113,9 +139,12 @@ class SchrootSession(Session): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, + close_fds: bool = True, ): try: - subprocess.check_call(self._run_argv(argv, cwd, user, env=env)) + subprocess.check_call( + self._run_argv(argv, cwd, user, env=env), close_fds=close_fds + ) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv) @@ -151,19 +180,49 @@ class SchrootSession(Session): .decode() .rstrip("\n") ) - logging.info("Creating directory %s", home) + logging.info("Creating directory %s in schroot session.", home) self.check_call(["mkdir", "-p", home], cwd="/", user="root") self.check_call(["chown", user, home], cwd="/", user="root") - def _fullpath(self, path: str) -> str: + def external_path(self, path: str) -> str: if self._cwd is None: raise ValueError("no cwd set") return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/")) def exists(self, path: str) -> bool: - fullpath = self._fullpath(path) + fullpath = self.external_path(path) return os.path.exists(fullpath) def scandir(self, path: str): - fullpath = self._fullpath(path) + fullpath = self.external_path(path) return os.scandir(fullpath) + + def setup_from_vcs( + self, tree, include_controldir: Optional[bool] = None, subdir="package" + ): + from ..vcs import dupe_vcs_tree, export_vcs_tree + + build_dir = os.path.join(self.location, "build") + + directory = tempfile.mkdtemp(dir=build_dir) + reldir = "/" + os.path.relpath(directory, self.location) + + export_directory = os.path.join(directory, subdir) + if not include_controldir: + export_vcs_tree(tree, export_directory) + else: + dupe_vcs_tree(tree, export_directory) + + return export_directory, os.path.join(reldir, subdir) + + def setup_from_directory(self, path, subdir="package"): + import shutil + + build_dir = os.path.join(self.location, "build") + directory = tempfile.mkdtemp(dir=build_dir) + reldir = "/" + os.path.relpath(directory, self.location) + export_directory = os.path.join(directory, subdir) + shutil.copytree(path, export_directory, dirs_exist_ok=True) + return export_directory, os.path.join(reldir, subdir) + + is_temporary = True diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index a06884a..e3288b5 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -35,8 +35,9 @@ from ..debian.fix_build import ( resolve_error, versioned_package_fixers, apt_fixers, - BuildDependencyContext, + DebianPackagingContext, ) +from breezy.commit import NullCommitReporter from breezy.tests import TestCaseWithTransport @@ -44,10 +45,17 @@ class DummyAptSearcher(FileSearcher): def __init__(self, files): self._apt_files = files - def search_files(self, path, regex=False): + def search_files(self, path, regex=False, case_insensitive=False): for p, pkg in sorted(self._apt_files.items()): + if case_insensitive: + flags = re.I + else: + flags = 0 if regex: - if re.match(path, p): + if re.match(path, p, flags): + yield pkg + elif case_insensitive: + if path.lower() == p.lower(): yield pkg else: if path == p: @@ -97,16 +105,15 @@ blah (0.1) UNRELEASED; urgency=medium session = PlainSession() apt = AptManager(session) apt._searchers = [DummyAptSearcher(self._apt_files)] - context = BuildDependencyContext( - ("build", ), + context = DebianPackagingContext( self.tree, - apt, subpath="", committer="ognibuild ", update_changelog=True, + commit_reporter=NullCommitReporter(), ) - fixers = versioned_package_fixers(session) + apt_fixers(apt) - return resolve_error(error, context, fixers) + fixers = versioned_package_fixers(session, context, apt) + apt_fixers(apt, context) + return resolve_error(error, ("build",), fixers) def get_build_deps(self): with open(self.tree.abspath("debian/control"), "r") as f: diff --git a/ognibuild/vcs.py b/ognibuild/vcs.py index b9bbf77..45b32c6 100644 --- a/ognibuild/vcs.py +++ b/ognibuild/vcs.py @@ -28,9 +28,9 @@ from buildlog_consultant.sbuild import ( from . import DetailedFailure -def export_vcs_tree(tree, directory): +def export_vcs_tree(tree, directory, subpath=""): try: - export(tree, directory, "dir", None) + export(tree, directory, "dir", None, subdir=(subpath or None)) except OSError as e: if e.errno == errno.ENOSPC: raise DetailedFailure(1, ["export"], NoSpaceOnDevice()) diff --git a/releaser.conf b/releaser.conf index deea576..c4c19a4 100644 --- a/releaser.conf +++ b/releaser.conf @@ -7,3 +7,8 @@ update_version { match: "^ version=\"(.*)\",$" new_line: " version=\"$VERSION\"," } +update_version { + path: "ognibuild/__init__.py" + match: "^__version__ = \\((.*)\\)$" + new_line: "__version__ = $TUPLED_VERSION" +} diff --git a/setup.cfg b/setup.cfg index 99a5d9d..07ea45f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,8 +2,12 @@ banned-modules = silver-platter = Should not use silver-platter [mypy] -# A number of ognibuilds' dependencies don't have type hints yet ignore_missing_imports = True [bdist_wheel] universal = 1 + +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py index d85eb85..d3676b2 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup(name="ognibuild", description="Detect and run any build system", - version="0.0.2", + version="0.0.3", maintainer="Jelmer Vernooij", maintainer_email="jelmer@jelmer.uk", license="GNU GPLv2 or later", @@ -29,12 +29,12 @@ setup(name="ognibuild", }, install_requires=[ 'breezy', - 'buildlog-consultant', + 'buildlog-consultant>=0.0.4', 'requirements-parser', ], extras_require={ 'debian': ['debmutate', 'python_debian', 'python_apt'], }, - tests_require=['python_debian', 'buildlog-consultant', 'breezy'], + tests_require=['python_debian', 'buildlog-consultant', 'breezy', 'testtools'], test_suite='ognibuild.tests.test_suite', )