From 0b9077f9643032f263c7dfb9c6d21669d1afa537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 17:55:44 +0000 Subject: [PATCH 001/252] Support for running apt as root. --- ognibuild/buildsystem.py | 4 ++++ ognibuild/debian/apt.py | 3 +++ ognibuild/resolver/apt.py | 2 +- ognibuild/session/plain.py | 8 ++++++++ setup.py | 2 +- 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 2b47cc0..bfee9cd 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -302,6 +302,10 @@ class Meson(BuildSystem): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build"], fixers) + def dist(self, session, resolver, fixers, quiet=False): + self._setup(session, fixers) + run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) + def test(self, session, resolver, fixers): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build", "test"], fixers) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 6dc89f7..f5480da 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -103,6 +103,9 @@ class AptManager(object): def satisfy(self, deps: List[str]) -> None: run_apt(self.session, ["satisfy"] + deps) + def satisfy_command(self, deps: List[str]) -> str: + return ["apt", "satisfy"] + deps + class ContentsFileNotFound(Exception): """The contents file was not found.""" diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 08d95a4..f500efe 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -595,7 +595,7 @@ class AptResolver(Resolver): if apt_req is not None: apt_requirements.append((r, apt_req)) if apt_requirements: - yield (["apt", "satisfy"] + [PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))], [o for o, r in apt_requirements]) + yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements]) def resolve(self, req: Requirement): return resolve_requirement_apt(self.apt, req) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index b1f237c..05f86a9 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -27,6 +27,13 @@ class PlainSession(Session): location = "/" + def _prepend_user(self, user, args): + if user is not None: + import getpass + if user != getpass.getuser(): + args = ["sudo", "-u", user] + args + return args + def __repr__(self): return "%s()" % (type(self).__name__, ) @@ -40,6 +47,7 @@ class PlainSession(Session): return subprocess.check_output(args) def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): + args = self._prepend_user(user, args) return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) def exists(self, path): diff --git a/setup.py b/setup.py index d85eb85..f8d358e 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup(name="ognibuild", description="Detect and run any build system", - version="0.0.2", + version="0.0.1", maintainer="Jelmer Vernooij", maintainer_email="jelmer@jelmer.uk", license="GNU GPLv2 or later", From 3e631796165ae93ba126568624aaee1c2cd09407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 2 Mar 2021 18:26:45 +0000 Subject: [PATCH 002/252] Fix use of reset_tree. --- ognibuild/debian/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 753a454..0d7c020 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -608,7 +608,7 @@ def build_incrementally( if max_iterations is not None and len(fixed_errors) > max_iterations: logging.warning("Last fix did not address the issue. Giving up.") raise - reset_tree(local_tree, local_tree.basis_tree(), subpath=subpath) + reset_tree(local_tree, subpath=subpath) if e.phase[0] == "build": context = BuildDependencyContext( e.phase, From 57a69725704658917d146cb2f2755a805a807a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 3 Mar 2021 00:00:35 +0000 Subject: [PATCH 003/252] Support resolving crates. --- ognibuild/buildsystem.py | 9 +++++++-- ognibuild/debian/apt.py | 2 +- ognibuild/requirements.py | 22 ++++++++++++++++++---- ognibuild/resolver/apt.py | 11 +++++++++++ 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index bfee9cd..6804b20 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -627,8 +627,13 @@ class Cargo(BuildSystem): def get_declared_dependencies(self): if "dependencies" in self.cargo: for name, details in self.cargo["dependencies"].items(): - # TODO(jelmer): Look at details['features'], details['version'] - yield "build", CargoCrateRequirement(name) + if isinstance(details, str): + details = {"version": details} + # TODO(jelmer): Look at details['version'] + yield "build", CargoCrateRequirement( + name, + features=details.get('features', []), + version=details.get("version")) def test(self, session, resolver, fixers): run_with_build_fixers(session, ["cargo", "test"], fixers) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index f5480da..405194f 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -308,7 +308,7 @@ def get_package_for_paths( if candidates: break if len(candidates) == 0: - logging.warning("No packages found that contain %r", paths) + logging.debug("No packages found that contain %r", paths) return None if len(candidates) > 1: logging.warning( diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index e2e5ff6..e0ad842 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -18,7 +18,7 @@ import posixpath import subprocess -from typing import Optional, List, Tuple +from typing import Optional, List, Tuple, Set from . import Requirement @@ -123,19 +123,33 @@ class NodePackageRequirement(Requirement): class CargoCrateRequirement(Requirement): crate: str + features: Set[str] + version: Optional[str] - def __init__(self, crate): + def __init__(self, crate, features=None, version=None): super(CargoCrateRequirement, self).__init__("cargo-crate") self.crate = crate + if features is None: + features = set() + self.features = features + self.version = version def __repr__(self): - return "%s(%r)" % ( + return "%s(%r, features=%r, version=%r)" % ( type(self).__name__, self.crate, + self.features, + self.version ) def __str__(self): - return "cargo crate: %s" % self.crate + if self.features: + return "cargo crate: %s %s (%s)" % ( + self.crate, self.version or '', + ', '.join(sorted(self.features))) + else: + return "cargo crate: %s %s" % ( + self.crate, self.version or '') class PkgConfigRequirement(Requirement): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index f500efe..7969786 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -28,6 +28,7 @@ from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements from ..requirements import ( Requirement, + CargoCrateRequirement, BinaryRequirement, CHeaderRequirement, PkgConfigRequirement, @@ -515,6 +516,15 @@ def resolve_python_package_req(apt_mgr, req): return None +def resolve_cargo_crate_req(apt_mgr, req): + paths = [ + '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % req.crate] + pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) + if pkg_name is None: + return None + return AptRequirement.simple(pkg_name) + + APT_REQUIREMENT_RESOLVERS = [ (BinaryRequirement, resolve_binary_req), (PkgConfigRequirement, resolve_pkg_config_req), @@ -542,6 +552,7 @@ APT_REQUIREMENT_RESOLVERS = [ (AutoconfMacroRequirement, resolve_autoconf_macro_req), (PythonModuleRequirement, resolve_python_module_req), (PythonPackageRequirement, resolve_python_package_req), + (CargoCrateRequirement, resolve_cargo_crate_req), ] From 7a6e1af492c4ae545b80569e9f5be41d2d5f6a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 3 Mar 2021 00:31:46 +0000 Subject: [PATCH 004/252] Fix typing. --- ognibuild/debian/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 405194f..c2eaf95 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -103,7 +103,7 @@ class AptManager(object): def satisfy(self, deps: List[str]) -> None: run_apt(self.session, ["satisfy"] + deps) - def satisfy_command(self, deps: List[str]) -> str: + def satisfy_command(self, deps: List[str]) -> List[str]: return ["apt", "satisfy"] + deps From 8f39cc839b4904aa90ac3550f3d0e36b74a81794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 3 Mar 2021 01:18:33 +0000 Subject: [PATCH 005/252] Install debian extra. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 123fa45..05fc643 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -21,8 +21,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip flake8 cython - python -m pip install git+https://github.com/jelmer/buildlog-consultant python setup.py develop + python -m pip install -e ".[debian]" mkdir -p ~/.config/breezy/plugins brz branch lp:brz-debian ~/.config/breezy/plugins/debian - name: Style checks From da2605ab255b7d0eba4ae954500fa27049e1530f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 4 Mar 2021 16:10:38 +0000 Subject: [PATCH 006/252] Improve Java support. --- ognibuild/buildlog.py | 4 ++++ ognibuild/buildsystem.py | 43 +++++++++++++++++++++++++++++++++------ ognibuild/fix_build.py | 3 ++- ognibuild/requirements.py | 6 ++++++ ognibuild/resolver/apt.py | 6 ++++++ 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index bb596b8..16ed253 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -34,6 +34,7 @@ from buildlog_consultant.common import ( MissingPerlModule, MissingXmlEntity, MissingJDKFile, + MissingJDK, MissingNodeModule, MissingPhpClass, MissingRubyGem, @@ -75,6 +76,7 @@ from .requirements import ( MavenArtifactRequirement, GnomeCommonRequirement, JDKFileRequirement, + JDKRequirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, @@ -129,6 +131,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return GnomeCommonRequirement() elif isinstance(problem, MissingJDKFile): return JDKFileRequirement(problem.jdk_path, problem.filename) + elif isinstance(problem, MissingJDK): + return JDKRequirement() elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": return BinaryRequirement("glib-gettextize") diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 6804b20..4611595 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -262,20 +262,51 @@ class Gradle(BuildSystem): name = "gradle" - def __init__(self, path): + def __init__(self, path, executable="gradle"): self.path = path + self.executable = executable def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) + @classmethod + def exists(cls, path): + return ( + os.path.exists(os.path.join(path, "build.gradle")) or + os.path.exists(os.path.join(path, "build.gradle.kts"))) + + @classmethod + def from_path(cls, path): + if os.path.exists(os.path.join(path, "gradlew")): + return cls(path, "./gradlew") + return cls(path) + + def setup(self, resolver): + if not self.executable.startswith('./'): + resolver.install([BinaryRequirement(self.executable)]) + def clean(self, session, resolver, fixers): - run_with_build_fixers(session, ["gradle", "clean"], fixers) + self.setup(resolver) + run_with_build_fixers(session, [self.executable, "clean"], fixers) def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["gradle", "build"], fixers) + self.setup(resolver) + run_with_build_fixers(session, [self.executable, "build"], fixers) def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["gradle", "test"], fixers) + self.setup(resolver) + run_with_build_fixers(session, [self.executable, "test"], fixers) + + def dist(self, session, resolver, fixers, quiet=False): + self.setup(resolver) + run_with_build_fixers(session, [self.executable, "distTar"], fixers) + + def install(self, session, resolver, fixers, install_target): + raise NotImplementedError + self.setup(resolver) + # TODO(jelmer): installDist just creates files under build/install/... + run_with_build_fixers( + session, [self.executable, "installDist"], fixers) class Meson(BuildSystem): @@ -733,9 +764,9 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 logging.debug("Found Cargo.toml, assuming rust cargo package.") yield Cargo("Cargo.toml") - if os.path.exists(os.path.join(path, "build.gradle")): + if Gradle.exists(path): logging.debug("Found build.gradle, assuming gradle package.") - yield Gradle("build.gradle") + yield Gradle.from_path(path) if os.path.exists(os.path.join(path, "meson.build")): logging.debug("Found meson.build, assuming meson package.") diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index d7dca7f..abbebc1 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -66,9 +66,10 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF logging.info("Running %r", args) fixed_errors = [] while True: - retcode, lines = run_with_tee(session, args) + retcode, contents = run_with_tee(session, args) if retcode == 0: return + lines = ''.join(contents).splitlines(True) match, error = find_build_failure_description(lines) if error is None: if match: diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index e0ad842..c131f67 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -334,6 +334,12 @@ class JDKFileRequirement(Requirement): return posixpath.join(self.jdk_path, self.filename) +class JDKRequirement(Requirement): + + def __init__(self): + super(JDKRequirement, self).__init__("jdk") + + class PerlFileRequirement(Requirement): filename: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 7969786..3d1679e 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -50,6 +50,7 @@ from ..requirements import ( MavenArtifactRequirement, GnomeCommonRequirement, JDKFileRequirement, + JDKRequirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, @@ -444,6 +445,10 @@ def resolve_jdk_file_req(apt_mgr, req): return None +def resolve_jdk_req(apt_mgr, req): + return AptRequirement.simple('default-jdk') + + def resolve_perl_module_req(apt_mgr, req): DEFAULT_PERL_PATHS = ["/usr/share/perl5"] @@ -547,6 +552,7 @@ APT_REQUIREMENT_RESOLVERS = [ (MavenArtifactRequirement, resolve_maven_artifact_req), (GnomeCommonRequirement, resolve_gnome_common_req), (JDKFileRequirement, resolve_jdk_file_req), + (JDKRequirement, resolve_jdk_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), (AutoconfMacroRequirement, resolve_autoconf_macro_req), From 456b47bb975fe0bfa5036f32b3ef507ea445ac9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 13:33:54 +0000 Subject: [PATCH 007/252] Fix python module assignment. --- ognibuild/requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index c131f67..17ba1fe 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -366,6 +366,7 @@ class PythonModuleRequirement(Requirement): def __init__(self, module, python_version=None, minimum_version=None): super(PythonModuleRequirement, self).__init__("python-module") + self.module = module self.python_version = python_version self.minimum_version = minimum_version From 05bad21f1fda1fa897ad97473745fae895b917e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 14:35:09 +0000 Subject: [PATCH 008/252] Split out Contents file reading. --- ognibuild/debian/apt.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index c2eaf95..8be48d0 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -111,6 +111,14 @@ class ContentsFileNotFound(Exception): """The contents file was not found.""" +def read_contents_file(f): + for line in f: + (path, rest) = line.rsplit(maxsplit=1) + package = rest.split(b"/")[-1] + decoded_path = "/" + path.decode("utf-8", "surrogateescape") + yield decoded_path, package.decode("utf-8") + + class AptContentsFileSearcher(FileSearcher): def __init__(self): self._db = {} @@ -145,11 +153,8 @@ class AptContentsFileSearcher(FileSearcher): yield pkg def load_file(self, f): - for line in f: - (path, rest) = line.rsplit(maxsplit=1) - package = rest.split(b"/")[-1] - decoded_path = "/" + path.decode("utf-8", "surrogateescape") - self[decoded_path] = package.decode("utf-8") + for path, package in read_contents_file(f): + self[path] = package @classmethod def _load_cache_file(cls, url, cache_dir): From ae493f85c9c521b934fc6fd8364234bbcad687ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 14:44:10 +0000 Subject: [PATCH 009/252] Fix debian install. --- .github/workflows/pythonpackage.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 05fc643..b8e4b5a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -22,9 +22,14 @@ jobs: run: | python -m pip install --upgrade pip flake8 cython python setup.py develop + - name: Install Debian-specific dependencies + run: | + apt install libapt-pkg-dev + python -m pip install git+https://salsa.debian.org/apt-team/python-apt python -m pip install -e ".[debian]" mkdir -p ~/.config/breezy/plugins brz branch lp:brz-debian ~/.config/breezy/plugins/debian + if: "matrix.os == 'ubuntu-latest'" - name: Style checks run: | python -m flake8 From cfbca2330235578d17836ce9dbd1646d0013e6a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 14:56:52 +0000 Subject: [PATCH 010/252] Add list of supported systems. Fixes #1 --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index 30d9861..286489b 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,50 @@ Ognibuild has a number of subcommands: It also includes a subcommand that can fix up the build dependencies for Debian packages, called deb-fix-build. +Status +------ + +Ognibuild is functional, but sometimes rough around the edges. If you run into +issues (or lack of support for a particular ecosystem), please file a bug. + +Supported Build Systems +~~~~~~~~~~~~~~~~~~~~~~~ + +* Cabal +* Cargo +* Golang +* Gradle +* Make, including various makefile generators: + + autoconf/automake + + CMake + + Makefile.PL +* Maven +* ninja, including ninja generators: + + meson +* Node +* PHP Pear +* Python - setup.py/setup.cfg/pyproject.toml +* Ruby gems +* Waf + +Supported package repositories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Package repositories are used to install missing dependencies. + +The following "native" repositories are supported: + +* pypi +* cpan +* hackage +* npm +* cargo +* golang\* + +As well one distribution repository: + +* apt + License ------- From 516ad0dc47312198347e427615dfbe2643042140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 14:58:25 +0000 Subject: [PATCH 011/252] Fix syntax. --- README.md | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 286489b..8def755 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -ognibuild -========= +# ognibuild Ognibuild is a simple wrapper with a common interface for invoking any kind of build tool. @@ -10,8 +9,7 @@ parameters. It can also detect and install missing dependencies. -Goals ------ +## Goals The goal of ognibuild is to provide a consistent CLI that can be used for any software package. It is mostly useful for automated building of @@ -20,8 +18,7 @@ large sets of diverse packages (e.g. different programming languages). It is not meant to expose all functionality that is present in the underlying build systems. To use that, invoke those build systems directly. -Usage ------ +## Usage Ognibuild has a number of subcommands: @@ -34,14 +31,12 @@ Ognibuild has a number of subcommands: It also includes a subcommand that can fix up the build dependencies for Debian packages, called deb-fix-build. -Status ------- +## Status Ognibuild is functional, but sometimes rough around the edges. If you run into issues (or lack of support for a particular ecosystem), please file a bug. -Supported Build Systems -~~~~~~~~~~~~~~~~~~~~~~~ +### Supported Build Systems * Cabal * Cargo @@ -60,8 +55,7 @@ Supported Build Systems * Ruby gems * Waf -Supported package repositories -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### Supported package repositories Package repositories are used to install missing dependencies. @@ -78,7 +72,6 @@ As well one distribution repository: * apt -License -------- +## License Ognibuild is licensed under the GNU GPL, v2 or later. From 579efb46faec9ce90d1e105736d4303ec9774a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 14:59:16 +0000 Subject: [PATCH 012/252] Fix list syntax... --- README.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 8def755..c85ba1c 100644 --- a/README.md +++ b/README.md @@ -38,22 +38,22 @@ issues (or lack of support for a particular ecosystem), please file a bug. ### Supported Build Systems -* Cabal -* Cargo -* Golang -* Gradle -* Make, including various makefile generators: +- Cabal +- Cargo +- Golang +- Gradle +- Make, including various makefile generators: + autoconf/automake + CMake + Makefile.PL -* Maven -* ninja, including ninja generators: +- Maven +- ninja, including ninja generators: + meson -* Node -* PHP Pear -* Python - setup.py/setup.cfg/pyproject.toml -* Ruby gems -* Waf +- Node +- PHP Pear +- Python - setup.py/setup.cfg/pyproject.toml +- Ruby gems +- Waf ### Supported package repositories @@ -61,16 +61,16 @@ Package repositories are used to install missing dependencies. The following "native" repositories are supported: -* pypi -* cpan -* hackage -* npm -* cargo -* golang\* +- pypi +- cpan +- hackage +- npm +- cargo +- golang\* As well one distribution repository: -* apt +- apt ## License From a227ffa07f023aad73c9610590ae1d891a0c5d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 5 Mar 2021 14:59:34 +0000 Subject: [PATCH 013/252] Fix sublist syntax... --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c85ba1c..8d9b19d 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,12 @@ issues (or lack of support for a particular ecosystem), please file a bug. - Golang - Gradle - Make, including various makefile generators: - + autoconf/automake - + CMake - + Makefile.PL + - autoconf/automake + - CMake + - Makefile.PL - Maven -- ninja, including ninja generators: - + meson +- ninja, including ninja file generators: + - meson - Node - PHP Pear - Python - setup.py/setup.cfg/pyproject.toml From d3afa5f6ebb32f412dd8c813fdccb2807ef05a43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 14 Mar 2021 22:42:05 +0000 Subject: [PATCH 014/252] Factor out url_to_cache_filename. --- ognibuild/debian/apt.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 8be48d0..144d917 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -119,6 +119,12 @@ def read_contents_file(f): yield decoded_path, package.decode("utf-8") +def url_to_cache_filename(url): + from urllib.parse import urlparse + parsed = urlparse(url) + return parsed.hostname + parsed.path.replace("/", "_") + + class AptContentsFileSearcher(FileSearcher): def __init__(self): self._db = {} @@ -143,14 +149,16 @@ class AptContentsFileSearcher(FileSearcher): self._db[path] = package def search_files(self, path, regex=False): - c = re.compile(path) - for p, pkg in sorted(self._db.items()): - if regex: + if regex: + c = re.compile(path) + for p, pkg in sorted(self._db.items()): if c.match(p): yield pkg - else: - if path == p: - yield pkg + else: + try: + return self._db[path] + except KeyError: + pass def load_file(self, f): for path, package in read_contents_file(f): @@ -158,12 +166,8 @@ class AptContentsFileSearcher(FileSearcher): @classmethod def _load_cache_file(cls, url, cache_dir): - from urllib.parse import urlparse - - parsed = urlparse(url) - p = os.path.join( - cache_dir, parsed.hostname + parsed.path.replace("/", "_") + ".lz4" - ) + fn = url_to_cache_filename(url) + p = os.path.join(cache_dir, fn + ".lz4") if not os.path.exists(p): return None logging.debug("Loading cached contents file %s", p) From b302b9039024c103fcc1dd7472ddb148716feaed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 15 Mar 2021 03:07:52 +0000 Subject: [PATCH 015/252] Fix python specs. --- ognibuild/resolver/apt.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 3d1679e..83cd3fa 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -500,10 +500,14 @@ def resolve_autoconf_macro_req(apt_mgr, req): def resolve_python_module_req(apt_mgr, req): + if req.minimum_version: + specs = [(">=", req.minimum_version)] + else: + specs = [] if req.python_version == 2: - return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.specs) + return get_package_for_python_module(apt_mgr, req.module, "cpython2", specs) elif req.python_version in (None, 3): - return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.specs) + return get_package_for_python_module(apt_mgr, req.module, "cpython3", specs) else: return None From 95c2ed458310c2377b90b395c4c8934644645765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 15 Mar 2021 19:31:54 +0000 Subject: [PATCH 016/252] Fix python version match. --- ognibuild/resolver/apt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 83cd3fa..6089850 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -132,7 +132,7 @@ def get_package_for_python_package(apt_mgr, package, python_version, specs=None) def get_package_for_python_module(apt_mgr, module, python_version, specs): - if python_version == "python3": + if python_version == "cpython3": paths = [ posixpath.join( "/usr/lib/python3/dist-packages", @@ -153,7 +153,7 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py" ), ] - elif python_version == "python2": + elif python_version == "cpython2": paths = [ posixpath.join( "/usr/lib/python2\\.[0-9]/dist-packages", From ed7238d396b289a373b2458df8ff70c21e82334d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 15 Mar 2021 20:24:32 +0000 Subject: [PATCH 017/252] s/pythonX/cpythonX/g --- ognibuild/debian/fix_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 0d7c020..8fb6c7e 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -392,8 +392,8 @@ def fix_missing_python_module(error, context): specs = [] pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", specs) - py2_pkg = get_package_for_python_module(context.apt, error.module, "python2", specs) - py3_pkg = get_package_for_python_module(context.apt, error.module, "python3", specs) + py2_pkg = get_package_for_python_module(context.apt, error.module, "cpython2", specs) + py3_pkg = get_package_for_python_module(context.apt, error.module, "cpython3", specs) extra_build_deps = [] if error.python_version == 2: From 1acda140cf7c32a832341c2d0c8aab560b0b89ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 15 Mar 2021 21:30:05 +0000 Subject: [PATCH 018/252] Use sudo to run apt. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index b8e4b5a..83b445c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,7 +24,7 @@ jobs: python setup.py develop - name: Install Debian-specific dependencies run: | - apt install libapt-pkg-dev + sudo apt install libapt-pkg-dev python -m pip install git+https://salsa.debian.org/apt-team/python-apt python -m pip install -e ".[debian]" mkdir -p ~/.config/breezy/plugins From 1d6b83b2be08ecf54667ad347129f90e588bbd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 04:29:27 +0000 Subject: [PATCH 019/252] Factor out contents_urls_from_sourceslist. --- ognibuild/debian/apt.py | 72 ++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 144d917..a877828 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -125,6 +125,40 @@ def url_to_cache_filename(url): return parsed.hostname + parsed.path.replace("/", "_") +def contents_urls_from_sourceslist(sl, arch): + # TODO(jelmer): Verify signatures, etc. + arches = [(arch, True), ("all", False)] + for source in sl.list: + if source.invalid or source.disabled: + continue + if source.type == "deb-src": + continue + if source.type != "deb": + logging.warning("Invalid line in sources: %r", source) + continue + base_url = source.uri.rstrip("/") + name = source.dist.rstrip("/") + components = source.comps + if components: + dists_url = base_url + "/dists" + else: + dists_url = base_url + if components: + for component in components: + for arch, mandatory in arches: + yield ( + "%s/%s/%s/Contents-%s" + % (dists_url, name, component, arch), + mandatory, + ) + else: + for arch, mandatory in arches: + yield ( + "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), + mandatory, + ) + + class AptContentsFileSearcher(FileSearcher): def __init__(self): self._db = {} @@ -207,42 +241,8 @@ class AptContentsFileSearcher(FileSearcher): # TODO(jelmer): Use aptsources.sourceslist.SourcesList from .build import get_build_architecture - # TODO(jelmer): Verify signatures, etc. - urls = [] - arches = [(get_build_architecture(), True), ("all", False)] - for source in sl.list: - if source.invalid or source.disabled: - continue - if source.type == "deb-src": - continue - if source.type != "deb": - logging.warning("Invalid line in sources: %r", source) - continue - base_url = source.uri.rstrip("/") - name = source.dist.rstrip("/") - components = source.comps - if components: - dists_url = base_url + "/dists" - else: - dists_url = base_url - if components: - for component in components: - for arch, mandatory in arches: - urls.append( - ( - "%s/%s/%s/Contents-%s" - % (dists_url, name, component, arch), - mandatory, - ) - ) - else: - for arch, mandatory in arches: - urls.append( - ( - "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), - mandatory, - ) - ) + urls = list( + contents_urls_from_sourceslist(sl, get_build_architecture())) return cls.from_urls(urls, cache_dirs=cache_dirs) @staticmethod From ceb052332b6657e2458ecca8f263130e0aa85fc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 04:34:58 +0000 Subject: [PATCH 020/252] Factor out load_contents_url. --- ognibuild/debian/apt.py | 72 ++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index a877828..32b2a7e 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -29,6 +29,9 @@ from .. import DetailedFailure, UnidentifiedError from ..session import Session, run_with_tee +USER_AGENT = "Ognibuild" + + def run_apt(session: Session, args: List[str]) -> None: """Run apt.""" args = ["apt", "-y"] + args @@ -159,6 +162,40 @@ def contents_urls_from_sourceslist(sl, arch): ) +def load_contents_url(url): + from urllib.error import HTTPError + from urllib.request import urlopen, Request + + for ext in [".xz", ".gz", ""]: + try: + request = Request( + url + ext, headers={"User-Agent": USER_AGENT}) + response = urlopen(request) + except HTTPError as e: + if e.status == 404: + continue + raise + break + else: + raise ContentsFileNotFound(url) + if ext == ".gz": + import gzip + + f = gzip.GzipFile(fileobj=response) + elif ext == ".xz": + import lzma + from io import BytesIO + + f = BytesIO(lzma.decompress(response.read())) + elif response.headers.get_content_type() == "text/plain": + f = response + else: + raise Exception( + "Unknown content type %r" % response.headers.get_content_type() + ) + return f + + class AptContentsFileSearcher(FileSearcher): def __init__(self): self._db = {} @@ -245,41 +282,8 @@ class AptContentsFileSearcher(FileSearcher): contents_urls_from_sourceslist(sl, get_build_architecture())) return cls.from_urls(urls, cache_dirs=cache_dirs) - @staticmethod - def _get(url): - from urllib.request import urlopen, Request - - request = Request(url, headers={"User-Agent": "Debian Janitor"}) - return urlopen(request) - def load_url(self, url, allow_cache=True): - from urllib.error import HTTPError - - for ext in [".xz", ".gz", ""]: - try: - response = self._get(url + ext) - except HTTPError as e: - if e.status == 404: - continue - raise - break - else: - raise ContentsFileNotFound(url) - if ext == ".gz": - import gzip - - f = gzip.GzipFile(fileobj=response) - elif ext == ".xz": - import lzma - from io import BytesIO - - f = BytesIO(lzma.decompress(response.read())) - elif response.headers.get_content_type() == "text/plain": - f = response - else: - raise Exception( - "Unknown content type %r" % response.headers.get_content_type() - ) + f = load_contents_url(url) self.load_file(f) From 6aaa2df421548628898262a5398710a549d24984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 15:26:52 +0000 Subject: [PATCH 021/252] Factor out file search. --- ognibuild/__init__.py | 3 + ognibuild/debian/apt.py | 236 +------------------------ ognibuild/debian/file_search.py | 293 ++++++++++++++++++++++++++++++++ 3 files changed, 299 insertions(+), 233 deletions(-) create mode 100644 ognibuild/debian/file_search.py diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 6e210c2..366f1f8 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -20,6 +20,9 @@ import os import stat +USER_AGENT = "Ognibuild" + + class DetailedFailure(Exception): def __init__(self, retcode, argv, error): self.retcode = retcode diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 32b2a7e..eb318b6 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -18,7 +18,7 @@ import logging import re -from typing import List, Iterator, Optional, Set +from typing import List, Optional, Set import os from buildlog_consultant.apt import ( @@ -27,9 +27,7 @@ from buildlog_consultant.apt import ( from .. import DetailedFailure, UnidentifiedError from ..session import Session, run_with_tee - - -USER_AGENT = "Ognibuild" +from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_package_for_paths def run_apt(session: Session, args: List[str]) -> None: @@ -46,11 +44,6 @@ def run_apt(session: Session, args: List[str]) -> None: raise UnidentifiedError(retcode, args, lines, secondary=match) -class FileSearcher(object): - def search_files(self, path: str, regex: bool = False) -> Iterator[str]: - raise NotImplementedError(self.search_files) - - class AptManager(object): session: Session @@ -64,7 +57,7 @@ class AptManager(object): def searchers(self): if self._searchers is None: self._searchers = [ - AptContentsFileSearcher.from_session(self.session), + AptCachedContentsFileSearcher.from_session(self.session), GENERATED_FILE_SEARCHER, ] return self._searchers @@ -108,226 +101,3 @@ class AptManager(object): def satisfy_command(self, deps: List[str]) -> List[str]: return ["apt", "satisfy"] + deps - - -class ContentsFileNotFound(Exception): - """The contents file was not found.""" - - -def read_contents_file(f): - for line in f: - (path, rest) = line.rsplit(maxsplit=1) - package = rest.split(b"/")[-1] - decoded_path = "/" + path.decode("utf-8", "surrogateescape") - yield decoded_path, package.decode("utf-8") - - -def url_to_cache_filename(url): - from urllib.parse import urlparse - parsed = urlparse(url) - return parsed.hostname + parsed.path.replace("/", "_") - - -def contents_urls_from_sourceslist(sl, arch): - # TODO(jelmer): Verify signatures, etc. - arches = [(arch, True), ("all", False)] - for source in sl.list: - if source.invalid or source.disabled: - continue - if source.type == "deb-src": - continue - if source.type != "deb": - logging.warning("Invalid line in sources: %r", source) - continue - base_url = source.uri.rstrip("/") - name = source.dist.rstrip("/") - components = source.comps - if components: - dists_url = base_url + "/dists" - else: - dists_url = base_url - if components: - for component in components: - for arch, mandatory in arches: - yield ( - "%s/%s/%s/Contents-%s" - % (dists_url, name, component, arch), - mandatory, - ) - else: - for arch, mandatory in arches: - yield ( - "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), - mandatory, - ) - - -def load_contents_url(url): - from urllib.error import HTTPError - from urllib.request import urlopen, Request - - for ext in [".xz", ".gz", ""]: - try: - request = Request( - url + ext, headers={"User-Agent": USER_AGENT}) - response = urlopen(request) - except HTTPError as e: - if e.status == 404: - continue - raise - break - else: - raise ContentsFileNotFound(url) - if ext == ".gz": - import gzip - - f = gzip.GzipFile(fileobj=response) - elif ext == ".xz": - import lzma - from io import BytesIO - - f = BytesIO(lzma.decompress(response.read())) - elif response.headers.get_content_type() == "text/plain": - f = response - else: - raise Exception( - "Unknown content type %r" % response.headers.get_content_type() - ) - return f - - -class AptContentsFileSearcher(FileSearcher): - def __init__(self): - self._db = {} - - @classmethod - def from_session(cls, session): - logging.info("Loading apt contents information") - # TODO(jelmer): what about sources.list.d? - from aptsources.sourceslist import SourcesList - - sl = SourcesList() - sl.load(os.path.join(session.location, "etc/apt/sources.list")) - return cls.from_sources_list( - sl, - cache_dirs=[ - os.path.join(session.location, "var/lib/apt/lists"), - "/var/lib/apt/lists", - ], - ) - - def __setitem__(self, path, package): - self._db[path] = package - - def search_files(self, path, regex=False): - if regex: - c = re.compile(path) - for p, pkg in sorted(self._db.items()): - if c.match(p): - yield pkg - else: - try: - return self._db[path] - except KeyError: - pass - - def load_file(self, f): - for path, package in read_contents_file(f): - self[path] = package - - @classmethod - def _load_cache_file(cls, url, cache_dir): - fn = url_to_cache_filename(url) - p = os.path.join(cache_dir, fn + ".lz4") - if not os.path.exists(p): - return None - logging.debug("Loading cached contents file %s", p) - import lz4.frame - - return lz4.frame.open(p, mode="rb") - - @classmethod - def from_urls(cls, urls, cache_dirs=None): - self = cls() - for url, mandatory in urls: - for cache_dir in cache_dirs or []: - f = cls._load_cache_file(url, cache_dir) - if f is not None: - self.load_file(f) - break - else: - if not mandatory and self._db: - logging.debug( - "Not attempting to fetch optional contents " "file %s", url - ) - else: - logging.debug("Fetching contents file %s", url) - try: - self.load_url(url) - except ContentsFileNotFound: - if mandatory: - logging.warning("Unable to fetch contents file %s", url) - else: - logging.debug( - "Unable to fetch optional contents file %s", url - ) - return self - - @classmethod - def from_sources_list(cls, sl, cache_dirs=None): - # TODO(jelmer): Use aptsources.sourceslist.SourcesList - from .build import get_build_architecture - - urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture())) - return cls.from_urls(urls, cache_dirs=cache_dirs) - - def load_url(self, url, allow_cache=True): - f = load_contents_url(url) - self.load_file(f) - - -class GeneratedFileSearcher(FileSearcher): - def __init__(self, db): - self._db = db - - def search_files(self, path: str, regex: bool = False) -> Iterator[str]: - for p, pkg in sorted(self._db.items()): - if regex: - if re.match(path, p): - yield pkg - else: - if path == p: - yield pkg - - -# TODO(jelmer): read from a file -GENERATED_FILE_SEARCHER = GeneratedFileSearcher( - { - "/etc/locale.gen": "locales", - # Alternative - "/usr/bin/rst2html": "/usr/share/docutils/scripts/python3/rst2html", - } -) - - -def get_package_for_paths( - paths: List[str], searchers: List[FileSearcher], regex: bool = False -) -> Optional[str]: - candidates: Set[str] = set() - for path in paths: - for searcher in searchers: - candidates.update(searcher.search_files(path, regex=regex)) - if candidates: - break - if len(candidates) == 0: - logging.debug("No packages found that contain %r", paths) - return None - if len(candidates) > 1: - logging.warning( - "More than 1 packages found that contain %r: %r", path, candidates - ) - # Euhr. Pick the one with the shortest name? - return sorted(candidates, key=len)[0] - else: - return candidates.pop() diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py new file mode 100644 index 0000000..5a0b105 --- /dev/null +++ b/ognibuild/debian/file_search.py @@ -0,0 +1,293 @@ +#!/usr/bin/python +# Copyright (C) 2019-2020 Jelmer Vernooij +# encoding: utf-8 +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from datetime import datetime +import os +import re +from typing import Iterator, List, Optional, Set +import logging + + +from .. import USER_AGENT + + +class FileSearcher(object): + def search_files(self, path: str, regex: bool = False) -> Iterator[str]: + raise NotImplementedError(self.search_files) + + +class ContentsFileNotFound(Exception): + """The contents file was not found.""" + + +def read_contents_file(f): + for line in f: + (path, rest) = line.rsplit(maxsplit=1) + yield path, rest + + +def url_to_cache_filename(url): + from urllib.parse import urlparse + parsed = urlparse(url) + return parsed.hostname + parsed.path.replace("/", "_") + + +def contents_urls_from_sourceslist(sl, arch): + # TODO(jelmer): Verify signatures, etc. + arches = [(arch, True), ("all", False)] + for source in sl.list: + if source.invalid or source.disabled: + continue + if source.type == "deb-src": + continue + if source.type != "deb": + logging.warning("Invalid line in sources: %r", source) + continue + base_url = source.uri.rstrip("/") + name = source.dist.rstrip("/") + components = source.comps + if components: + dists_url = base_url + "/dists" + else: + dists_url = base_url + if components: + for component in components: + for arch, mandatory in arches: + yield ( + "%s/%s/%s/Contents-%s" + % (dists_url, name, component, arch), + mandatory, + ) + else: + for arch, mandatory in arches: + yield ( + "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), + mandatory, + ) + + +def load_contents_url(url): + from urllib.error import HTTPError + from urllib.request import urlopen, Request + + for ext in [".xz", ".gz", ""]: + try: + request = Request( + url + ext, headers={"User-Agent": USER_AGENT}) + response = urlopen(request) + except HTTPError as e: + if e.status == 404: + continue + raise + break + else: + raise ContentsFileNotFound(url) + if ext == ".gz": + import gzip + + f = gzip.GzipFile(fileobj=response) + elif ext == ".xz": + import lzma + from io import BytesIO + + f = BytesIO(lzma.decompress(response.read())) + elif response.headers.get_content_type() == "text/plain": + f = response + else: + raise Exception( + "Unknown content type %r" % response.headers.get_content_type() + ) + return f + + +def load_apt_cache_file(cache_dir, url): + fn = url_to_cache_filename(url) + p = os.path.join(cache_dir, fn + ".lz4") + if not os.path.exists(p): + return None + logging.debug("Loading cached contents file %s", p) + #return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) + import lz4.frame + return lz4.frame.open(p, mode="rb") + + +class AptCachedContentsFileSearcher(FileSearcher): + def __init__(self): + self._db = {} + + @classmethod + def from_session(cls, session): + logging.info("Loading apt contents information") + + self = cls() + self.load_from_session(session) + return self + + def load_local(self): + # TODO(jelmer): what about sources.list.d? + from aptsources.sourceslist import SourcesList + + sl = SourcesList() + sl.load("/etc/apt/sources.list") + + from .build import get_build_architecture + + urls = list( + contents_urls_from_sourceslist(sl, get_build_architecture())) + cache_dirs = set(["/var/lib/apt/lists"]) + self._load_urls(urls, cache_dirs) + + def load_from_session(self, session): + # TODO(jelmer): what about sources.list.d? + from aptsources.sourceslist import SourcesList + + sl = SourcesList() + sl.load(os.path.join(session.location, "etc/apt/sources.list")) + + from .build import get_build_architecture + + urls = list( + contents_urls_from_sourceslist(sl, get_build_architecture())) + cache_dirs = [ + os.path.join(session.location, "var/lib/apt/lists"), + "/var/lib/apt/lists", + ] + self._load_urls(urls, cache_dirs) + + def _load_urls(self, urls, cache_dirs): + for url, mandatory in urls: + for cache_dir in cache_dirs: + f = load_apt_cache_file(cache_dir, url) + if f is not None: + self.load_file(f, url) + break + else: + if not mandatory and self._db: + logging.debug( + "Not attempting to fetch optional contents " "file %s", url + ) + else: + logging.debug("Fetching contents file %s", url) + try: + f = load_contents_url(url) + self.load_file(f, url) + except ContentsFileNotFound: + if mandatory: + logging.warning("Unable to fetch contents file %s", url) + else: + logging.debug( + "Unable to fetch optional contents file %s", url + ) + + def __setitem__(self, path, package): + self._db[path] = package + + def search_files(self, path, regex=False): + path = path.lstrip('/').encode('utf-8', 'surrogateescape') + if regex: + c = re.compile(path) + ret = [] + for p, rest in self._db.items(): + if c.match(p): + pkg = rest.split(b"/")[-1] + ret.append((p, pkg.decode('utf-8'))) + for p, pkg in sorted(ret): + yield pkg + else: + try: + yield self._db[path].split(b"/")[-1].decode('utf-8') + except KeyError: + pass + + def load_file(self, f, url): + start_time = datetime.now() + for path, rest in read_contents_file(f.readlines()): + self[path] = rest + logging.debug('Read %s in %s', url, datetime.now() - start_time) + + +class GeneratedFileSearcher(FileSearcher): + def __init__(self, db): + self._db = db + + def search_files(self, path: str, regex: bool = False) -> Iterator[str]: + for p, pkg in sorted(self._db.items()): + if regex: + if re.match(path, p): + yield pkg + else: + if path == p: + yield pkg + + +# TODO(jelmer): read from a file +GENERATED_FILE_SEARCHER = GeneratedFileSearcher( + { + "/etc/locale.gen": "locales", + # Alternative + "/usr/bin/rst2html": "/usr/share/docutils/scripts/python3/rst2html", + } +) + + +def get_package_for_paths( + paths: List[str], searchers: List[FileSearcher], regex: bool = False +) -> Optional[str]: + candidates: Set[str] = set() + for path in paths: + for searcher in searchers: + candidates.update(searcher.search_files(path, regex=regex)) + if candidates: + break + if len(candidates) == 0: + logging.debug("No packages found that contain %r", paths) + return None + if len(candidates) > 1: + logging.warning( + "More than 1 packages found that contain %r: %r", path, candidates + ) + # Euhr. Pick the one with the shortest name? + return sorted(candidates, key=len)[0] + else: + return candidates.pop() + + +def main(argv): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('path', help='Path to search for.', type=str, nargs='*') + parser.add_argument('--regex', '-x', help='Search for regex.', action='store_true') + parser.add_argument('--debug', action='store_true') + args = parser.parse_args() + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + main_searcher = AptCachedContentsFileSearcher() + main_searcher.load_local() + searchers = [main_searcher, GENERATED_FILE_SEARCHER] + + package = get_package_for_paths(args.path, searchers=searchers, regex=args.regex) + print(package) + + +if __name__ == '__main__': + import sys + sys.exit(main(sys.argv)) From 79de95e787b5a12d837c8070002db5a0f842d051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 17:09:23 +0000 Subject: [PATCH 022/252] Run sudo. --- ognibuild/__main__.py | 2 +- ognibuild/debian/apt.py | 27 ++++++++++++++++++++------- ognibuild/resolver/__init__.py | 19 ++++--------------- ognibuild/resolver/apt.py | 2 +- ognibuild/session/__init__.py | 4 ++++ 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index a889962..e0bed9c 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -159,7 +159,7 @@ def main(): # noqa: C901 elif args.resolve == "native": resolver = native_resolvers(session) elif args.resolve == "auto": - resolver = auto_resolver(session) + resolver = auto_resolver(session, explain=args.explain) logging.info("Using requirement resolver: %s", resolver) os.chdir(args.directory) try: diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index eb318b6..6daefa5 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -26,13 +26,15 @@ from buildlog_consultant.apt import ( ) from .. import DetailedFailure, UnidentifiedError -from ..session import Session, run_with_tee +from ..session import Session, run_with_tee, get_user from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_package_for_paths -def run_apt(session: Session, args: List[str]) -> None: +def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None: """Run apt.""" - args = ["apt", "-y"] + args + if prefix is None: + prefix = [] + args = prefix = ["apt", "-y"] + args retcode, lines = run_with_tee(session, args, cwd="/", user="root") if retcode == 0: return @@ -49,10 +51,21 @@ class AptManager(object): session: Session _searchers: Optional[List[FileSearcher]] - def __init__(self, session): + def __init__(self, session, prefix=None): self.session = session self._apt_cache = None self._searchers = None + if prefix is None: + prefix = [] + self.prefix = prefix + + @classmethod + def from_session(cls, session): + if get_user(session) != "root": + prefix = ["sudo"] + else: + prefix = [] + return cls(session, prefix=prefix) def searchers(self): if self._searchers is None: @@ -94,10 +107,10 @@ class AptManager(object): logging.info("Installing using apt: %r", packages) packages = self.missing(packages) if packages: - run_apt(self.session, ["install"] + packages) + run_apt(self.session, ["install"] + packages, prefix=self.prefix) def satisfy(self, deps: List[str]) -> None: - run_apt(self.session, ["satisfy"] + deps) + run_apt(self.session, ["satisfy"] + deps, prefix=self.prefix) def satisfy_command(self, deps: List[str]) -> List[str]: - return ["apt", "satisfy"] + deps + return self.prefix + ["apt", "satisfy"] + deps diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 51e0467..4dacd35 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -275,28 +275,17 @@ def native_resolvers(session): return StackedResolver([kls(session) for kls in NATIVE_RESOLVER_CLS]) -class ExplainResolver(Resolver): - def __init__(self, session): - self.session = session - - @classmethod - def from_session(cls, session): - return cls(session) - - def install(self, requirements): - raise UnsatisfiedRequirements(requirements) - - -def auto_resolver(session): +def auto_resolver(session, explain=False): # if session is SchrootSession or if we're root, use apt from .apt import AptResolver from ..session.schroot import SchrootSession + from ..session import get_user - user = session.check_output(["echo", "$USER"]).decode().strip() + user = get_user(session) resolvers = [] # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if # present? - if isinstance(session, SchrootSession) or user == "root": + if isinstance(session, SchrootSession) or user == "root" or explain: resolvers.append(AptResolver.from_session(session)) resolvers.extend([kls(session) for kls in NATIVE_RESOLVER_CLS]) return StackedResolver(resolvers) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 6089850..5827ab6 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -582,7 +582,7 @@ class AptResolver(Resolver): @classmethod def from_session(cls, session): - return cls(AptManager(session)) + return cls(AptManager.from_session(session)) def install(self, requirements): missing = [] diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 7cdd15f..23021c1 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -88,3 +88,7 @@ def run_with_tee(session: Session, args: List[str], **kwargs): sys.stdout.buffer.flush() contents.append(line.decode("utf-8", "surrogateescape")) return p.returncode, contents + + +def get_user(session): + return session.check_output(["echo", "$USER"]).decode().strip() From 09d74cf827434091b22b43230bb966b116c6845f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 18:28:53 +0000 Subject: [PATCH 023/252] Use apt_pkg function. --- ognibuild/debian/file_search.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 5a0b105..ecab66a 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -16,6 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import apt_pkg from datetime import datetime import os import re @@ -41,12 +42,6 @@ def read_contents_file(f): yield path, rest -def url_to_cache_filename(url): - from urllib.parse import urlparse - parsed = urlparse(url) - return parsed.hostname + parsed.path.replace("/", "_") - - def contents_urls_from_sourceslist(sl, arch): # TODO(jelmer): Verify signatures, etc. arches = [(arch, True), ("all", False)] @@ -116,7 +111,7 @@ def load_contents_url(url): def load_apt_cache_file(cache_dir, url): - fn = url_to_cache_filename(url) + fn = apt_pkg.uri_to_filename(url) p = os.path.join(cache_dir, fn + ".lz4") if not os.path.exists(p): return None From b6414238bff15c64e135f8d2e09ea68f167e6a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 20:02:52 +0000 Subject: [PATCH 024/252] Some refactoring. --- ognibuild/debian/file_search.py | 64 +++++++++++++++++---------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index ecab66a..5205785 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -42,38 +42,42 @@ def read_contents_file(f): yield path, rest +def contents_urls_from_sources_entry(source, arches): + if source.invalid or source.disabled: + return + if source.type == "deb-src": + return + if source.type != "deb": + logging.warning("Invalid line in sources: %r", source) + return + base_url = source.uri.rstrip("/") + name = source.dist.rstrip("/") + components = source.comps + if components: + dists_url = base_url + "/dists" + else: + dists_url = base_url + if components: + for component in components: + for arch, mandatory in arches: + yield ( + "%s/%s/%s/Contents-%s" + % (dists_url, name, component, arch), + mandatory, + ) + else: + for arch, mandatory in arches: + yield ( + "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), + mandatory, + ) + + def contents_urls_from_sourceslist(sl, arch): # TODO(jelmer): Verify signatures, etc. arches = [(arch, True), ("all", False)] for source in sl.list: - if source.invalid or source.disabled: - continue - if source.type == "deb-src": - continue - if source.type != "deb": - logging.warning("Invalid line in sources: %r", source) - continue - base_url = source.uri.rstrip("/") - name = source.dist.rstrip("/") - components = source.comps - if components: - dists_url = base_url + "/dists" - else: - dists_url = base_url - if components: - for component in components: - for arch, mandatory in arches: - yield ( - "%s/%s/%s/Contents-%s" - % (dists_url, name, component, arch), - mandatory, - ) - else: - for arch, mandatory in arches: - yield ( - "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), - mandatory, - ) + yield from contents_urls_from_sources_entry(source, arches) def load_contents_url(url): @@ -158,10 +162,10 @@ class AptCachedContentsFileSearcher(FileSearcher): urls = list( contents_urls_from_sourceslist(sl, get_build_architecture())) - cache_dirs = [ + cache_dirs = set([ os.path.join(session.location, "var/lib/apt/lists"), "/var/lib/apt/lists", - ] + ]) self._load_urls(urls, cache_dirs) def _load_urls(self, urls, cache_dirs): From b7206647b134f67cdbea9365ed08a44b1e38123c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 20:07:42 +0000 Subject: [PATCH 025/252] Simplify cache handling. --- ognibuild/debian/file_search.py | 85 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 5205785..9ee2b2e 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -42,7 +42,7 @@ def read_contents_file(f): yield path, rest -def contents_urls_from_sources_entry(source, arches): +def contents_urls_from_sources_entry(source, arches, load_url): if source.invalid or source.disabled: return if source.type == "deb-src": @@ -59,28 +59,26 @@ def contents_urls_from_sources_entry(source, arches): dists_url = base_url if components: for component in components: - for arch, mandatory in arches: + for arch in arches: yield ( "%s/%s/%s/Contents-%s" - % (dists_url, name, component, arch), - mandatory, + % (dists_url, name, component, arch) ) else: - for arch, mandatory in arches: + for arch in arches: yield ( - "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch), - mandatory, + "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch) ) -def contents_urls_from_sourceslist(sl, arch): +def contents_urls_from_sourceslist(sl, arch, load_url): # TODO(jelmer): Verify signatures, etc. - arches = [(arch, True), ("all", False)] + arches = [arch, "all"] for source in sl.list: - yield from contents_urls_from_sources_entry(source, arches) + yield from contents_urls_from_sources_entry(source, arches, load_url) -def load_contents_url(url): +def load_direct_url(url): from urllib.error import HTTPError from urllib.request import urlopen, Request @@ -114,11 +112,20 @@ def load_contents_url(url): return f -def load_apt_cache_file(cache_dir, url): +def load_url_with_cache(url, cache_dirs): + for cache_dir in cache_dirs: + try: + return load_apt_cache_file(url, cache_dir) + except FileNotFoundError: + pass + return load_direct_url(url) + + +def load_apt_cache_file(url, cache_dir): fn = apt_pkg.uri_to_filename(url) p = os.path.join(cache_dir, fn + ".lz4") if not os.path.exists(p): - return None + raise FileNotFoundError(p) logging.debug("Loading cached contents file %s", p) #return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) import lz4.frame @@ -145,11 +152,15 @@ class AptCachedContentsFileSearcher(FileSearcher): sl.load("/etc/apt/sources.list") from .build import get_build_architecture + cache_dirs = set(["/var/lib/apt/lists"]) + + def load_url(url): + return load_url_with_cache(url, cache_dirs) urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture())) - cache_dirs = set(["/var/lib/apt/lists"]) - self._load_urls(urls, cache_dirs) + contents_urls_from_sourceslist(sl, get_build_architecture(), + load_url)) + self._load_urls(urls, cache_dirs, load_url) def load_from_session(self, session): # TODO(jelmer): what about sources.list.d? @@ -160,38 +171,26 @@ class AptCachedContentsFileSearcher(FileSearcher): from .build import get_build_architecture - urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture())) cache_dirs = set([ os.path.join(session.location, "var/lib/apt/lists"), "/var/lib/apt/lists", ]) - self._load_urls(urls, cache_dirs) - def _load_urls(self, urls, cache_dirs): - for url, mandatory in urls: - for cache_dir in cache_dirs: - f = load_apt_cache_file(cache_dir, url) - if f is not None: - self.load_file(f, url) - break - else: - if not mandatory and self._db: - logging.debug( - "Not attempting to fetch optional contents " "file %s", url - ) - else: - logging.debug("Fetching contents file %s", url) - try: - f = load_contents_url(url) - self.load_file(f, url) - except ContentsFileNotFound: - if mandatory: - logging.warning("Unable to fetch contents file %s", url) - else: - logging.debug( - "Unable to fetch optional contents file %s", url - ) + def load_url(url): + return load_url_with_cache(url, cache_dirs) + + urls = list( + contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)) + self._load_urls(urls, cache_dirs, load_url) + + def _load_urls(self, urls, cache_dirs, load_url): + for url in urls: + logging.debug("Fetching contents file %s", url) + try: + f = load_url(url) + self.load_file(f, url) + except ContentsFileNotFound: + logging.warning("Unable to fetch contents file %s", url) def __setitem__(self, path, package): self._db[path] = package From 891e097d3cc4fd5c7e5f69767a02fe9b2bc21425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 20:21:11 +0000 Subject: [PATCH 026/252] More streamlining. --- ognibuild/debian/file_search.py | 88 ++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 9ee2b2e..e450e56 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -18,6 +18,7 @@ import apt_pkg from datetime import datetime +from debian.deb822 import Release import os import re from typing import Iterator, List, Optional, Set @@ -57,18 +58,36 @@ def contents_urls_from_sources_entry(source, arches, load_url): dists_url = base_url + "/dists" else: dists_url = base_url + inrelease_url = "%s/%s/InRelease" % (dists_url, name) + try: + response = load_url(inrelease_url) + except FileNotFoundError: + release_url = "%s/%s/Release" % (dists_url, name) + try: + response = load_url(release_url) + except FileNotFoundError as e: + logging.warning('Unable to download %s or %s: %s', inrelease_url, release_url, e) + return + + existing_names = {} + release = Release(response.read()) + for hn in ['MD5Sum', 'SHA1Sum', 'SHA256Sum']: + for entry in release.get(hn, []): + existing_names[os.path.splitext(entry['name'])[0]] = entry['name'] + + contents_files = set() if components: for component in components: for arch in arches: - yield ( - "%s/%s/%s/Contents-%s" - % (dists_url, name, component, arch) - ) + contents_files.add("%s/Contents-%s" % (component, arch)) else: for arch in arches: - yield ( - "%s/%s/Contents-%s" % (dists_url, name.rstrip("/"), arch) - ) + contents_files.add("Contents-%s" % (arch,)) + + for fn in contents_files: + if fn in existing_names: + url = "%s/%s/%s" % (dists_url, name, fn) + yield url def contents_urls_from_sourceslist(sl, arch, load_url): @@ -78,6 +97,20 @@ def contents_urls_from_sourceslist(sl, arch, load_url): yield from contents_urls_from_sources_entry(source, arches, load_url) +def _unwrap(f, ext): + if ext == ".gz": + import gzip + + return gzip.GzipFile(fileobj=f) + elif ext == ".xz": + import lzma + from io import BytesIO + + f = BytesIO(lzma.decompress(f.read())) + else: + return f + + def load_direct_url(url): from urllib.error import HTTPError from urllib.request import urlopen, Request @@ -93,23 +126,8 @@ def load_direct_url(url): raise break else: - raise ContentsFileNotFound(url) - if ext == ".gz": - import gzip - - f = gzip.GzipFile(fileobj=response) - elif ext == ".xz": - import lzma - from io import BytesIO - - f = BytesIO(lzma.decompress(response.read())) - elif response.headers.get_content_type() == "text/plain": - f = response - else: - raise Exception( - "Unknown content type %r" % response.headers.get_content_type() - ) - return f + raise FileNotFoundError(url) + return _unwrap(response, ext) def load_url_with_cache(url, cache_dirs): @@ -123,13 +141,17 @@ def load_url_with_cache(url, cache_dirs): def load_apt_cache_file(url, cache_dir): fn = apt_pkg.uri_to_filename(url) - p = os.path.join(cache_dir, fn + ".lz4") - if not os.path.exists(p): - raise FileNotFoundError(p) - logging.debug("Loading cached contents file %s", p) - #return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) - import lz4.frame - return lz4.frame.open(p, mode="rb") + for ext in ['.xz', '.gz', '.lz4', '']: + p = os.path.join(cache_dir, fn + ext) + if not os.path.exists(p): + continue + #return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) + logging.debug("Loading cached contents file %s", p) + if ext == '.lz4': + import lz4.frame + return lz4.frame.open(p, mode="rb") + return _unwrap(open(p, 'rb'), ext) + raise FileNotFoundError(url) class AptCachedContentsFileSearcher(FileSearcher): @@ -158,8 +180,8 @@ class AptCachedContentsFileSearcher(FileSearcher): return load_url_with_cache(url, cache_dirs) urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture(), - load_url)) + contents_urls_from_sourceslist( + sl, get_build_architecture(), load_url)) self._load_urls(urls, cache_dirs, load_url) def load_from_session(self, session): From 8bd4de0b4e2bf68db86575cbc7c4a9e5b3956379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 20:29:39 +0000 Subject: [PATCH 027/252] Drop logging. --- ognibuild/debian/file_search.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index e450e56..0d6ead5 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -207,7 +207,6 @@ class AptCachedContentsFileSearcher(FileSearcher): def _load_urls(self, urls, cache_dirs, load_url): for url in urls: - logging.debug("Fetching contents file %s", url) try: f = load_url(url) self.load_file(f, url) From d2f7f3eac6424ad6f136986b6247c131b72ca7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 21:36:03 +0000 Subject: [PATCH 028/252] Skip apt tests with pypy. --- .github/workflows/pythonpackage.yml | 59 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 83b445c..24fb02a 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -13,33 +13,32 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip flake8 cython - python setup.py develop - - name: Install Debian-specific dependencies - run: | - sudo apt install libapt-pkg-dev - python -m pip install git+https://salsa.debian.org/apt-team/python-apt - python -m pip install -e ".[debian]" - mkdir -p ~/.config/breezy/plugins - brz branch lp:brz-debian ~/.config/breezy/plugins/debian - if: "matrix.os == 'ubuntu-latest'" - - name: Style checks - run: | - python -m flake8 - - name: Typing checks - run: | - pip install -U mypy - python -m mypy ognibuild - if: "matrix.python-version != 'pypy3'" - - name: Test suite run - run: | - python -m unittest ognibuild.tests.test_suite - env: - PYTHONHASHSEED: random + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip flake8 cython + python setup.py develop + - name: Install Debian-specific dependencies + run: | + python -m pip install git+https://salsa.debian.org/apt-team/python-apt + python -m pip install -e ".[debian]" + mkdir -p ~/.config/breezy/plugins + brz branch lp:brz-debian ~/.config/breezy/plugins/debian + if: "matrix.python-version != 'pypy3' and matrix.os == 'ubuntu-latest'" + - name: Style checks + run: | + python -m flake8 + - name: Typing checks + run: | + pip install -U mypy + python -m mypy ognibuild + if: "matrix.python-version != 'pypy3'" + - name: Test suite run + run: | + python -m unittest ognibuild.tests.test_suite + env: + PYTHONHASHSEED: random From b945a0f6770c52ebac001f22d8ae6bad91a0fa16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 21:44:12 +0000 Subject: [PATCH 029/252] Fix syntax. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 24fb02a..af21fa5 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -28,7 +28,7 @@ jobs: python -m pip install -e ".[debian]" mkdir -p ~/.config/breezy/plugins brz branch lp:brz-debian ~/.config/breezy/plugins/debian - if: "matrix.python-version != 'pypy3' and matrix.os == 'ubuntu-latest'" + if: "matrix.python-version != 'pypy3' && matrix.os == 'ubuntu-latest'" - name: Style checks run: | python -m flake8 From 5c6383a3f63e29880baf4908e4ac79672bb10b34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 21:47:40 +0000 Subject: [PATCH 030/252] Install apt-pkg-dev. --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index af21fa5..a9bfdd8 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -24,6 +24,7 @@ jobs: python setup.py develop - name: Install Debian-specific dependencies run: | + sudo apt install libapt-pkg-dev python -m pip install git+https://salsa.debian.org/apt-team/python-apt python -m pip install -e ".[debian]" mkdir -p ~/.config/breezy/plugins From 94418e06941f55550ea0dad411e2827708f3a001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 16 Mar 2021 22:06:51 +0000 Subject: [PATCH 031/252] Try to install wheel. --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a9bfdd8..9c0a80c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -25,6 +25,7 @@ jobs: - name: Install Debian-specific dependencies run: | sudo apt install libapt-pkg-dev + python -m pip install wheel python -m pip install git+https://salsa.debian.org/apt-team/python-apt python -m pip install -e ".[debian]" mkdir -p ~/.config/breezy/plugins From b988dd23246259e6f5e9285b54b94f3f6e13bab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 01:30:10 +0000 Subject: [PATCH 032/252] Fix relative paths. --- ognibuild/buildsystem.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 4611595..2a620ba 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -681,6 +681,9 @@ class Golang(BuildSystem): name = "golang" + def __init__(self, path): + self.path = path + def __repr__(self): return "%s()" % (type(self).__name__) @@ -740,29 +743,29 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 """Detect build systems.""" if os.path.exists(os.path.join(path, "package.xml")): logging.debug("Found package.xml, assuming pear package.") - yield Pear("package.xml") + yield Pear(os.path.join(path, "package.xml")) if os.path.exists(os.path.join(path, "setup.py")): logging.debug("Found setup.py, assuming python project.") - yield SetupPy("setup.py") + yield SetupPy(os.path.join(path, "setup.py")) elif os.path.exists(os.path.join(path, "pyproject.toml")): logging.debug("Found pyproject.toml, assuming python project.") - yield PyProject("pyproject.toml") + yield PyProject(os.path.join(path, "pyproject.toml")) elif os.path.exists(os.path.join(path, "setup.cfg")): logging.debug("Found setup.cfg, assuming python project.") - yield SetupCfg("setup.cfg") + yield SetupCfg(os.path.join(path, "setup.cfg")) if os.path.exists(os.path.join(path, "package.json")): logging.debug("Found package.json, assuming node package.") - yield Npm("package.json") + yield Npm(os.path.join(path, "package.json")) if os.path.exists(os.path.join(path, "waf")): logging.debug("Found waf, assuming waf package.") - yield Waf("waf") + yield Waf(os.path.join(path, "waf")) if os.path.exists(os.path.join(path, "Cargo.toml")): logging.debug("Found Cargo.toml, assuming rust cargo package.") - yield Cargo("Cargo.toml") + yield Cargo(os.path.join(path, "Cargo.toml")) if Gradle.exists(path): logging.debug("Found build.gradle, assuming gradle package.") @@ -770,22 +773,22 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 if os.path.exists(os.path.join(path, "meson.build")): logging.debug("Found meson.build, assuming meson package.") - yield Meson("meson.build") + yield Meson(os.path.join(path, "meson.build")) if os.path.exists(os.path.join(path, "Setup.hs")): logging.debug("Found Setup.hs, assuming haskell package.") - yield Cabal("Setup.hs") + yield Cabal(os.path.join(path, "Setup.hs")) if os.path.exists(os.path.join(path, "pom.xml")): logging.debug("Found pom.xml, assuming maven package.") - yield Maven("pom.xml") + yield Maven(os.path.join(path, "pom.xml")) if os.path.exists(os.path.join(path, "dist.ini")) and not os.path.exists( os.path.join(path, "Makefile.PL") ): - yield DistInkt("dist.ini") + yield DistInkt(os.path.join(path, "dist.ini")) - gemfiles = [entry.name for entry in os.scandir(path) if entry.name.endswith(".gem")] + gemfiles = [entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")] if gemfiles: yield Gem(gemfiles[0]) @@ -810,7 +813,7 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 if os.path.exists(os.path.join(path, ".travis.yml")): import ruamel.yaml.reader - with open(".travis.yml", "rb") as f: + with open(os.path.join(path, ".travis.yml"), "rb") as f: try: data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) except ruamel.yaml.reader.ReaderError as e: @@ -818,13 +821,13 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 else: language = data.get("language") if language == "go": - yield Golang() + yield Golang(path) seen_golang = True if not seen_golang: for entry in os.scandir(path): if entry.name.endswith(".go"): - yield Golang() + yield Golang(path) break From 5b872c10a318a6d0ef97d31d79c4dbb178d2c1cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 03:15:45 +0000 Subject: [PATCH 033/252] Pass in subpath. --- ognibuild/debian/fix_build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 8fb6c7e..62170fd 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -297,8 +297,8 @@ def commit_debian_changes( return True -def targeted_python_versions(tree: Tree) -> Set[str]: - with tree.get_file("debian/control") as f: +def targeted_python_versions(tree: Tree, subpath: str) -> Set[str]: + with tree.get_file(os.path.join(subpath, "debian/control")) as f: control = Deb822(f) build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) all_build_deps: Set[str] = set() @@ -315,7 +315,7 @@ def targeted_python_versions(tree: Tree) -> Set[str]: def fix_missing_python_distribution(error, context): # noqa: C901 - targeted = targeted_python_versions(context.tree) + targeted = targeted_python_versions(context.tree, context.subpath) default = not targeted pypy_pkg = context.apt.get_package_for_paths( @@ -381,7 +381,7 @@ def fix_missing_python_distribution(error, context): # noqa: C901 def fix_missing_python_module(error, context): if getattr(context, "tree", None) is not None: - targeted = targeted_python_versions(context.tree) + targeted = targeted_python_versions(context.tree, context.subpath) else: targeted = set() default = not targeted From 0f6e8786de0352166d5ee363bd2d9d698f5d8dff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 03:21:15 +0000 Subject: [PATCH 034/252] Set chdir. --- ognibuild/buildsystem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 2a620ba..a4c0dd2 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -127,6 +127,8 @@ def run_setup(script_name, script_args=None, stop_after="run"): save_argv = sys.argv.copy() g = {"__file__": script_name, "__name__": "__main__"} try: + old_cwd = os.getcwd() + os.chdir(os.path.dirname(script_name)) try: sys.argv[0] = script_name if script_args is not None: @@ -134,6 +136,7 @@ def run_setup(script_name, script_args=None, stop_after="run"): with open(script_name, "rb") as f: exec(f.read(), g) finally: + os.chdir(old_cwd) sys.argv = save_argv core._setup_stop_after = None except SystemExit: From 2392907f8e048eeb02c41ef79e495be8cf411f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 03:50:53 +0000 Subject: [PATCH 035/252] Fix installation of poetry. --- ognibuild/buildsystem.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index a4c0dd2..65f29cf 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -371,13 +371,7 @@ class PyProject(BuildSystem): logging.debug( "Found pyproject.toml with poetry section, " "assuming poetry project." ) - resolver.install( - [ - PythonPackageRequirement("venv"), - PythonPackageRequirement("poetry"), - ] - ) - session.check_call(["poetry", "build", "-f", "sdist"]) + run_with_build_fixers(session, ["poetry", "build", "-f", "sdist"], fixers) return raise AssertionError("no supported section in pyproject.toml") From 9635b6864884ed2550e5c1fc506512c7450ef6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 04:11:58 +0000 Subject: [PATCH 036/252] There's no such thing as session.mkdir. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 65f29cf..35c4147 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -325,7 +325,7 @@ class Meson(BuildSystem): def _setup(self, session, fixers): if session.exists("build"): return - session.mkdir("build") + os.mkdir(os.path.join(session.location, "build")) run_with_build_fixers(session, ["meson", "setup", "build"], fixers) def clean(self, session, resolver, fixers): From 14e3726ae8748e08a59790a948889d6bccfb7ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 04:39:34 +0000 Subject: [PATCH 037/252] Handle missing command. --- ognibuild/fix_build.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index abbebc1..5023f9e 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -20,6 +20,7 @@ from typing import List, Optional from buildlog_consultant.common import ( find_build_failure_description, + MissingCommand, ) from breezy.mutabletree import MutableTree @@ -66,18 +67,22 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF logging.info("Running %r", args) fixed_errors = [] while True: - retcode, contents = run_with_tee(session, args) - if retcode == 0: - return - lines = ''.join(contents).splitlines(True) - match, error = find_build_failure_description(lines) - if error is None: - if match: - logging.warning("Build failed with unidentified error:") - logging.warning("%s", match.line.rstrip("\n")) - else: - logging.warning("Build failed and unable to find cause. Giving up.") - raise UnidentifiedError(retcode, args, lines, secondary=match) + try: + retcode, contents = run_with_tee(session, args) + except FileNotFoundError as e: + error = MissingCommand(e.args[0]) + else: + if retcode == 0: + return + lines = ''.join(contents).splitlines(True) + match, error = find_build_failure_description(lines) + if error is None: + if match: + logging.warning("Build failed with unidentified error:") + logging.warning("%s", match.line.rstrip("\n")) + else: + logging.warning("Build failed and unable to find cause. Giving up.") + raise UnidentifiedError(retcode, args, lines, secondary=match) logging.info("Identified error: %r", error) if error in fixed_errors: From 7238bb0caf7a0ffb149d1bb5aa315914fde36f47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 04:44:48 +0000 Subject: [PATCH 038/252] Handle missing make target. --- ognibuild/buildsystem.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 35c4147..7f03f91 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -569,9 +569,9 @@ class Make(BuildSystem): run_with_build_fixers(session, ["make", "dist"], fixers) except UnidentifiedError as e: if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: - pass + raise NotImplementedError elif "make[1]: *** No rule to make target 'dist'. Stop.\n" in e.lines: - pass + raise NotImplementedError elif ( "Reconfigure the source tree " "(via './config' or 'perl Configure'), please.\n" @@ -614,8 +614,6 @@ class Make(BuildSystem): run_with_build_fixers(session, ["make", "dist"], fixers) else: raise - else: - return def get_declared_dependencies(self): # TODO(jelmer): Split out the perl-specific stuff? From d46f45debc6f20075b56ad238e0b085bc3862794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 04:48:24 +0000 Subject: [PATCH 039/252] Use absolute path for dist.ini. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 7f03f91..ea67ca6 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -463,7 +463,7 @@ class DistInkt(BuildSystem): self.path = path self.name = "dist-zilla" self.dist_inkt_class = None - with open("dist.ini", "rb") as f: + with open(self.path, "rb") as f: for line in f: if not line.startswith(b";;"): continue From 4c75eb34c484c006f4c98d45dc630a3d368e1f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 04:52:54 +0000 Subject: [PATCH 040/252] Obtain username in root, since cwd might not exist. --- ognibuild/session/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 23021c1..cebe227 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -91,4 +91,4 @@ def run_with_tee(session: Session, args: List[str], **kwargs): def get_user(session): - return session.check_output(["echo", "$USER"]).decode().strip() + return session.check_output(["echo", "$USER"], cwd="/").decode().strip() From 159dc9879c4e30c8520f5e2726a686dbdf787603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 05:02:09 +0000 Subject: [PATCH 041/252] Missing args. --- ognibuild/session/plain.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 05f86a9..74d28d0 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -20,6 +20,7 @@ from . import Session import os import subprocess +from typing import Optional, Dict, List class PlainSession(Session): @@ -40,11 +41,21 @@ class PlainSession(Session): def create_home(self): pass - def check_call(self, args): - return subprocess.check_call(args) + def check_call( + self, argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None): + argv = self._prepend_user(user, argv) + return subprocess.check_call(argv, cwd=cwd, env=env) - def check_output(self, args): - return subprocess.check_output(args) + def check_output( + self, argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None) -> bytes: + argv = self._prepend_user(user, argv) + return subprocess.check_output(argv, cwd=cwd, env=env) def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): args = self._prepend_user(user, args) From c6afe4eb40e1a53a66fc08f5591c56e641d9273d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 12:24:43 +0000 Subject: [PATCH 042/252] Don't rely on cwd. --- ognibuild/buildsystem.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index ea67ca6..b9d7259 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -226,7 +226,7 @@ class SetupPy(BuildSystem): self._run_setup(session, resolver, ["install"] + extra_args, fixers) def _run_setup(self, session, resolver, args, fixers): - interpreter = shebang_binary("setup.py") + interpreter = shebang_binary(self.path) if interpreter is not None: resolver.install([BinaryRequirement(interpreter)]) run_with_build_fixers(session, ["./setup.py"] + args, fixers) @@ -502,24 +502,24 @@ class Make(BuildSystem): name = "make" + def __init__(self, path): + self.path = path + def __repr__(self): - return "%s()" % type(self).__name__ + return "%s(%r)" % (type(self).__name__, self.path) def setup(self, session, resolver, fixers): - resolver.install([BinaryRequirement("make")]) - def makefile_exists(): return any( [session.exists(p) for p in ["Makefile", "GNUmakefile", "makefile"]] ) if session.exists("Makefile.PL") and not makefile_exists(): - resolver.install([BinaryRequirement("perl")]) run_with_build_fixers(session, ["perl", "Makefile.PL"], fixers) if not makefile_exists() and not session.exists("configure"): if session.exists("autogen.sh"): - if shebang_binary("autogen.sh") is None: + if shebang_binary(os.path.join(self.path, "autogen.sh")) is None: run_with_build_fixers(session, ["/bin/sh", "./autogen.sh"], fixers) try: run_with_build_fixers(session, ["./autogen.sh"], fixers) @@ -802,7 +802,7 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 ] ] ): - yield Make() + yield Make(path) seen_golang = False if os.path.exists(os.path.join(path, ".travis.yml")): From b3a9fc3aac92fba261ec870bb8ab4b288167aae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 13:12:56 +0000 Subject: [PATCH 043/252] Add schroot arg. --- ognibuild/buildsystem.py | 4 +++ ognibuild/debian/fix_build.py | 50 ++++++++++++++++++++++++++--------- ognibuild/dist.py | 13 +++------ ognibuild/session/schroot.py | 18 +++++++++++++ 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index b9d7259..4991dd3 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -241,6 +241,10 @@ class SetupPy(BuildSystem): for require in self.result.get_requires(): yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages + if getattr(self.result, "setup_requires", []): + for require in self.result.setup_requires: + yield "build", PythonPackageRequirement.from_requirement_str(require) + # Not present for distutils-only packages if getattr(self.result, "install_requires", []): for require in self.result.install_requires: yield "core", PythonPackageRequirement.from_requirement_str(require) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 62170fd..3933fa6 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -696,16 +696,19 @@ def main(argv=None): help="force updating of the changelog", default=None, ) + parser.add_argument( + '--schroot', + type=str, + help='chroot to use.') args = parser.parse_args() from breezy.workingtree import WorkingTree from .apt import AptManager from ..session.plain import PlainSession + from ..session.schroot import SchrootSession import tempfile import contextlib - apt = AptManager(PlainSession()) - logging.basicConfig(level=logging.INFO, format="%(message)s") with contextlib.ExitStack() as es: @@ -716,17 +719,38 @@ def main(argv=None): output_directory = args.output_directory tree = WorkingTree.open(".") - build_incrementally( - tree, - apt, - args.suffix, - args.suite, - output_directory, - args.build_command, - None, - committer=args.committer, - update_changelog=args.update_changelog, - ) + if args.schroot: + session = SchrootSession(args.schroot) + es.enter_context(session) + else: + session = PlainSession() + + apt = AptManager(session) + + try: + build_incrementally( + tree, + apt, + args.suffix, + args.suite, + output_directory, + args.build_command, + None, + committer=args.committer, + update_changelog=args.update_changelog, + ) + except SbuildFailure as e: + if e.phase is None: + phase = 'unknown phase' + elif len(e.phase) == 1: + phase = e.phase[0] + else: + phase = '%s (%s)' % (e.phase[0], e.phase[1]) + if e.error: + logging.fatal('Error during %s: %s', phase, e.error) + else: + logging.fatal('Error during %s: %s', phase, e.description) + return 1 if __name__ == "__main__": diff --git a/ognibuild/dist.py b/ognibuild/dist.py index cfdb5db..578ee05 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -37,7 +37,6 @@ from buildlog_consultant.common import ( from . import DetailedFailure from .buildsystem import NoBuildToolsFound from .session.schroot import SchrootSession -from .vcs import dupe_vcs_tree, export_vcs_tree SUPPORTED_DIST_EXTENSIONS = [ @@ -142,20 +141,14 @@ def create_dist_schroot( from .debian import satisfy_build_deps satisfy_build_deps(session, packaging_tree) - build_dir = os.path.join(session.location, "build") try: - directory = tempfile.mkdtemp(dir=build_dir) + export_directory, reldir = session.setup_from_vcs( + tree, include_controldir=include_controldir, subdir=subdir) except OSError as e: if e.errno == errno.ENOSPC: raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) - reldir = "/" + os.path.relpath(directory, session.location) - - export_directory = os.path.join(directory, subdir) - if not include_controldir: - export_vcs_tree(tree, export_directory) - else: - dupe_vcs_tree(tree, export_directory) + raise buildsystems = list(detect_buildsystems(export_directory)) resolver = AptResolver.from_session(session) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 3677262..ba1a281 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -19,6 +19,7 @@ import logging import os import shlex import subprocess +import tempfile from typing import Optional, List, Dict @@ -167,3 +168,20 @@ class SchrootSession(Session): def scandir(self, path: str): fullpath = self._fullpath(path) return os.scandir(fullpath) + + def setup_from_vcs( + self, tree, include_controldir=False, subdir="package"): + from ..vcs import dupe_vcs_tree, export_vcs_tree + build_dir = os.path.join(self.location, "build") + + directory = tempfile.mkdtemp(dir=build_dir) + reldir = "/" + os.path.relpath(directory, self.location) + os.chdir(reldir) + + export_directory = os.path.join(directory, subdir) + if not include_controldir: + export_vcs_tree(tree, export_directory) + else: + dupe_vcs_tree(tree, export_directory) + + return export_directory, reldir From 40aa791ec1daee89dbb5921ff7255a0880beb7ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 13:19:46 +0000 Subject: [PATCH 044/252] Drop custom code. --- ognibuild/buildsystem.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 4991dd3..3e16542 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -181,23 +181,6 @@ class SetupPy(BuildSystem): setup_cfg_contents = f.read() except FileNotFoundError: setup_cfg_contents = "" - if "setuptools" in setup_py_contents: - logging.debug("Reference to setuptools found, installing.") - resolver.install([PythonPackageRequirement("setuptools")]) - if ( - "setuptools_scm" in setup_py_contents - or "setuptools_scm" in setup_cfg_contents - ): - logging.debug("Reference to setuptools-scm found, installing.") - resolver.install( - [ - PythonPackageRequirement("setuptools_scm"), - BinaryRequirement("git"), - BinaryRequirement("hg"), - ] - ) - - # TODO(jelmer): Install setup_requires def test(self, session, resolver, fixers): self.setup(resolver) From 9792ddb865c468fe97ea08d6acab27bcdcbfce67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 13:26:14 +0000 Subject: [PATCH 045/252] Read requirements from pyproject.toml. --- ognibuild/buildsystem.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 3e16542..c366ba8 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -362,6 +362,11 @@ class PyProject(BuildSystem): return raise AssertionError("no supported section in pyproject.toml") + def get_declared_dependencies(self): + if "build-system" in self.pyproject: + for require in self.pyproject['build-system'].get("requires", []): + yield "build", PythonPackageRequirement.from_requirement_str(require) + class SetupCfg(BuildSystem): @@ -730,12 +735,12 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 if os.path.exists(os.path.join(path, "setup.py")): logging.debug("Found setup.py, assuming python project.") yield SetupPy(os.path.join(path, "setup.py")) - elif os.path.exists(os.path.join(path, "pyproject.toml")): - logging.debug("Found pyproject.toml, assuming python project.") - yield PyProject(os.path.join(path, "pyproject.toml")) elif os.path.exists(os.path.join(path, "setup.cfg")): logging.debug("Found setup.cfg, assuming python project.") yield SetupCfg(os.path.join(path, "setup.cfg")) + if os.path.exists(os.path.join(path, "pyproject.toml")): + logging.debug("Found pyproject.toml, assuming python project.") + yield PyProject(os.path.join(path, "pyproject.toml")) if os.path.exists(os.path.join(path, "package.json")): logging.debug("Found package.json, assuming node package.") From cc85d9d7a3cefa3813848539face9ec721ee0c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 14:15:54 +0000 Subject: [PATCH 046/252] Add code for exporting. --- ognibuild/__main__.py | 5 ++++- ognibuild/buildsystem.py | 8 +------- ognibuild/dist.py | 9 ++------- ognibuild/session/__init__.py | 10 +++++++++- ognibuild/session/plain.py | 29 +++++++++++++++++++++++++++++ ognibuild/session/schroot.py | 15 ++++++++++++--- 6 files changed, 57 insertions(+), 19 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index e0bed9c..f9a5fc6 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -154,6 +154,10 @@ def main(): # noqa: C901 session = PlainSession() with session: + logging.info("Preparing directory %s", args.directory) + external_dir, internal_dir = session.setup_from_directory(args.directory) + session.chdir(internal_dir) + os.chdir(external_dir) if args.resolve == "apt": resolver = AptResolver.from_session(session) elif args.resolve == "native": @@ -161,7 +165,6 @@ def main(): # noqa: C901 elif args.resolve == "auto": resolver = auto_resolver(session, explain=args.explain) logging.info("Using requirement resolver: %s", resolver) - os.chdir(args.directory) try: bss = list(detect_buildsystems(args.directory)) logging.info( diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index c366ba8..532c841 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -174,13 +174,7 @@ class SetupPy(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def setup(self, resolver): - with open(self.path, "r") as f: - setup_py_contents = f.read() - try: - with open("setup.cfg", "r") as f: - setup_cfg_contents = f.read() - except FileNotFoundError: - setup_cfg_contents = "" + pass def test(self, session, resolver, fixers): self.setup(resolver) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 578ee05..c094230 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -155,13 +155,8 @@ def create_dist_schroot( fixers = [InstallFixer(resolver)] with DistCatcher(export_directory) as dc: - oldcwd = os.getcwd() - os.chdir(export_directory) - try: - session.chdir(os.path.join(reldir, subdir)) - run_dist(session, buildsystems, resolver, fixers) - finally: - os.chdir(oldcwd) + session.chdir(reldir) + run_dist(session, buildsystems, resolver, fixers) for path in dc.files: shutil.copy(path, target_dir) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index cebe227..aa93f29 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from typing import Optional, List, Dict +from typing import Optional, List, Dict, Tuple import sys import subprocess @@ -74,6 +74,14 @@ class Session(object): def scandir(self, path: str): raise NotImplementedError(self.scandir) + def setup_from_vcs( + self, tree, include_controldir: Optional[bool] = None, + subdir="package") -> Tuple[str, str]: + raise NotImplementedError(self.setup_from_vcs) + + def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]: + raise NotImplementedError(self.setup_from_directory) + class SessionSetupFailure(Exception): """Session failed to be set up.""" diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 74d28d0..58cdfb7 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -18,8 +18,10 @@ from . import Session +import contextlib import os import subprocess +import tempfile from typing import Optional, Dict, List @@ -38,6 +40,15 @@ class PlainSession(Session): def __repr__(self): return "%s()" % (type(self).__name__, ) + def __enter__(self) -> "Session": + self.es = contextlib.ExitStack() + self.es.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.es.__exit__(exc_type, exc_val, exc_tb) + return False + def create_home(self): pass @@ -69,3 +80,21 @@ class PlainSession(Session): def chdir(self, path): os.chdir(path) + + def setup_from_vcs( + self, tree, include_controldir=None, subdir="package"): + from ..vcs import dupe_vcs_tree, export_vcs_tree + if include_controldir is False or ( + not hasattr(tree, 'base') and include_controldir is None): + td = self.es.enter_context(tempfile.TemporaryDirectory()) + export_vcs_tree(tree, td) + return td, td + elif not hasattr(tree, 'base'): + td = self.es.enter_context(tempfile.TemporaryDirectory()) + dupe_vcs_tree(tree, td) + return td, td + else: + return tree.base, tree.base + + def setup_from_directory(self, path): + return path, path diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index ba1a281..6e8c236 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -170,13 +170,13 @@ class SchrootSession(Session): return os.scandir(fullpath) def setup_from_vcs( - self, tree, include_controldir=False, subdir="package"): + self, tree, include_controldir: Optional[bool] = None, + subdir="package"): from ..vcs import dupe_vcs_tree, export_vcs_tree build_dir = os.path.join(self.location, "build") directory = tempfile.mkdtemp(dir=build_dir) reldir = "/" + os.path.relpath(directory, self.location) - os.chdir(reldir) export_directory = os.path.join(directory, subdir) if not include_controldir: @@ -184,4 +184,13 @@ class SchrootSession(Session): else: dupe_vcs_tree(tree, export_directory) - return export_directory, reldir + return export_directory, os.path.join(reldir, subdir) + + def setup_from_directory(self, path, subdir="package"): + import shutil + build_dir = os.path.join(self.location, "build") + directory = tempfile.mkdtemp(dir=build_dir) + reldir = "/" + os.path.relpath(directory, self.location) + export_directory = os.path.join(directory, subdir) + shutil.copytree(path, export_directory, dirs_exist_ok=True) + return export_directory, os.path.join(reldir, subdir) From 997a643b019e165ff1c1af809ba3e832ed398e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 15:06:02 +0000 Subject: [PATCH 047/252] Handle ~= for python. --- ognibuild/resolver/apt.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 5827ab6..ffc6934 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -96,8 +96,20 @@ def python_spec_to_apt_rels(pkg_name, specs): else: rels = [] for spec in specs: - c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "=": "="}[spec[0]] - rels.append([{"name": pkg_name, "version": (c, Version(spec[1]))}]) + deb_version = Version(spec[1]) + if spec[0] == '~=': + # PEP 440: For a given release identifier V.N , the compatible + # release clause is approximately equivalent to the pair of + # comparison clauses: >= V.N, == V.* + parts = spec[1].split('.') + parts.pop(-1) + parts[-1] = str(int(parts[-1])+1) + next_maj_deb_version = Version('.'.join(parts)) + rels.extend([{"name": pkg_name, "version": ('>=', deb_version)}, + {"name": pkg_name, "version": ('<<', next_maj_deb_version)}]) + else: + c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "=": "="}[spec[0]] + rels.append([{"name": pkg_name, "version": (c, deb_version)}]) return rels From ccc7b934a253d212b9b0c4728473123858f4f887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 15:06:15 +0000 Subject: [PATCH 048/252] Enter context for PlainSession. --- ognibuild/debian/fix_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 3933fa6..855d880 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -721,10 +721,11 @@ def main(argv=None): tree = WorkingTree.open(".") if args.schroot: session = SchrootSession(args.schroot) - es.enter_context(session) else: session = PlainSession() + es.enter_context(session) + apt = AptManager(session) try: From 0c1abd29e36363da0369b68f876188e6e562a44d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 16:02:41 +0000 Subject: [PATCH 049/252] Fix path to META.yml. --- ognibuild/buildsystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 532c841..5bd51a6 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -603,13 +603,13 @@ class Make(BuildSystem): def get_declared_dependencies(self): # TODO(jelmer): Split out the perl-specific stuff? - if os.path.exists("META.yml"): + if os.path.exists(os.path.join(self.path, "META.yml")): # See http://module-build.sourceforge.net/META-spec-v1.4.html for # the specification of the format. import ruamel.yaml import ruamel.yaml.reader - with open("META.yml", "rb") as f: + with open(os.path.join(self.path, "META.yml"), "rb") as f: try: data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) except ruamel.yaml.reader.ReaderError as e: From b15dbe7c028353d8aa3b604f7c73963671433817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 16:11:43 +0000 Subject: [PATCH 050/252] Probing. --- ognibuild/buildsystem.py | 142 +++++++++++++++++++++++---------------- 1 file changed, 85 insertions(+), 57 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 5bd51a6..4b99aff 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -110,6 +110,12 @@ class Pear(BuildSystem): self.setup(resolver) run_with_build_fixers(session, ["pear", "install", self.path], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "package.xml")): + logging.debug("Found package.xml, assuming pear package.") + return cls(os.path.join(path, "package.xml")) + # run_setup, but setting __name__ # Imported from Python's distutils.core, Copyright (C) PSF @@ -265,6 +271,12 @@ class Gradle(BuildSystem): return cls(path, "./gradlew") return cls(path) + @classmethod + def probe(cls, path): + if cls.exists(path): + logging.debug("Found build.gradle, assuming gradle package.") + return cls.from_path(path) + def setup(self, resolver): if not self.executable.startswith('./'): resolver.install([BinaryRequirement(self.executable)]) @@ -329,6 +341,12 @@ class Meson(BuildSystem): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build", "install"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "meson.build")): + logging.debug("Found meson.build, assuming meson package.") + return Meson(os.path.join(path, "meson.build")) + class PyProject(BuildSystem): @@ -404,6 +422,12 @@ class Npm(BuildSystem): self.setup(resolver) run_with_build_fixers(session, ["npm", "pack"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "package.json")): + logging.debug("Found package.json, assuming node package.") + return cls(os.path.join(path, "package.json")) + class Waf(BuildSystem): @@ -423,6 +447,12 @@ class Waf(BuildSystem): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["./waf", "test"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "waf")): + logging.debug("Found waf, assuming waf package.") + return cls(os.path.join(path, "waf")) + class Gem(BuildSystem): @@ -443,6 +473,12 @@ class Gem(BuildSystem): logging.warning("More than one gemfile. Trying the first?") run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) + @classmethod + def probe(cls, path): + gemfiles = [entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")] + if gemfiles: + return cls(gemfiles[0]) + class DistInkt(BuildSystem): def __init__(self, path): @@ -483,6 +519,14 @@ class DistInkt(BuildSystem): resolver.install([PerlModuleRequirement("Dist::Zilla")]) run_with_build_fixers(session, ["dzil", "build", "--in", ".."], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "dist.ini")) and not os.path.exists( + os.path.join(path, "Makefile.PL") + ): + return cls(os.path.join(path, "dist.ini")) + + class Make(BuildSystem): @@ -620,6 +664,25 @@ class Make(BuildSystem): else: raise NotImplementedError + @classmethod + def probe(cls, path): + if any( + [ + os.path.exists(os.path.join(path, p)) + for p in [ + "Makefile", + "GNUmakefile", + "makefile", + "Makefile.PL", + "CMakeLists.txt", + "autogen.sh", + "configure.ac", + "configure.in", + ] + ] + ): + return cls(path) + class Cargo(BuildSystem): @@ -656,6 +719,12 @@ class Cargo(BuildSystem): def build(self, session, resolver, fixers): run_with_build_fixers(session, ["cargo", "build"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "Cargo.toml")): + logging.debug("Found Cargo.toml, assuming rust cargo package.") + return Cargo(os.path.join(path, "Cargo.toml")) + class Golang(BuildSystem): """Go builds.""" @@ -688,6 +757,12 @@ class Maven(BuildSystem): def __init__(self, path): self.path = path + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "pom.xml")): + logging.debug("Found pom.xml, assuming maven package.") + return cls(os.path.join(path, "pom.xml")) + class Cabal(BuildSystem): @@ -719,12 +794,19 @@ class Cabal(BuildSystem): def dist(self, session, resolver, fixers, quiet=False): self._run(session, ["sdist"], fixers) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "Setup.hs")): + logging.debug("Found Setup.hs, assuming haskell package.") + return cls(os.path.join(path, "Setup.hs")) + def detect_buildsystems(path, trust_package=False): # noqa: C901 """Detect build systems.""" - if os.path.exists(os.path.join(path, "package.xml")): - logging.debug("Found package.xml, assuming pear package.") - yield Pear(os.path.join(path, "package.xml")) + for bs_cls in [Pear, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, DistInkt, Gem, Make]: + bs = bs_cls.probe(path) + if bs is not None: + yield bs if os.path.exists(os.path.join(path, "setup.py")): logging.debug("Found setup.py, assuming python project.") @@ -736,60 +818,6 @@ def detect_buildsystems(path, trust_package=False): # noqa: C901 logging.debug("Found pyproject.toml, assuming python project.") yield PyProject(os.path.join(path, "pyproject.toml")) - if os.path.exists(os.path.join(path, "package.json")): - logging.debug("Found package.json, assuming node package.") - yield Npm(os.path.join(path, "package.json")) - - if os.path.exists(os.path.join(path, "waf")): - logging.debug("Found waf, assuming waf package.") - yield Waf(os.path.join(path, "waf")) - - if os.path.exists(os.path.join(path, "Cargo.toml")): - logging.debug("Found Cargo.toml, assuming rust cargo package.") - yield Cargo(os.path.join(path, "Cargo.toml")) - - if Gradle.exists(path): - logging.debug("Found build.gradle, assuming gradle package.") - yield Gradle.from_path(path) - - if os.path.exists(os.path.join(path, "meson.build")): - logging.debug("Found meson.build, assuming meson package.") - yield Meson(os.path.join(path, "meson.build")) - - if os.path.exists(os.path.join(path, "Setup.hs")): - logging.debug("Found Setup.hs, assuming haskell package.") - yield Cabal(os.path.join(path, "Setup.hs")) - - if os.path.exists(os.path.join(path, "pom.xml")): - logging.debug("Found pom.xml, assuming maven package.") - yield Maven(os.path.join(path, "pom.xml")) - - if os.path.exists(os.path.join(path, "dist.ini")) and not os.path.exists( - os.path.join(path, "Makefile.PL") - ): - yield DistInkt(os.path.join(path, "dist.ini")) - - gemfiles = [entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")] - if gemfiles: - yield Gem(gemfiles[0]) - - if any( - [ - os.path.exists(os.path.join(path, p)) - for p in [ - "Makefile", - "GNUmakefile", - "makefile", - "Makefile.PL", - "CMakeLists.txt", - "autogen.sh", - "configure.ac", - "configure.in", - ] - ] - ): - yield Make(path) - seen_golang = False if os.path.exists(os.path.join(path, ".travis.yml")): import ruamel.yaml.reader From 703d1cda9461c652f488c811aff886af6215b603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 16:29:02 +0000 Subject: [PATCH 051/252] More probing, unify python code. --- ognibuild/buildsystem.py | 178 +++++++++++++++----------------- ognibuild/debian/apt.py | 3 +- ognibuild/debian/file_search.py | 2 +- ognibuild/dist.py | 1 - 4 files changed, 87 insertions(+), 97 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 4b99aff..c532e8d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -168,13 +168,29 @@ class SetupPy(BuildSystem): def __init__(self, path): self.path = path - # TODO(jelmer): Perhaps run this in session, so we can install - # missing dependencies? + if os.path.exists(os.path.join(self.path, 'setup.py')): + self.has_setup_py = True + # TODO(jelmer): Perhaps run this in session, so we can install + # missing dependencies? + try: + self.distribution = run_setup(os.path.abspath(os.path.join(self.path, 'setup.py')), stop_after="init") + except RuntimeError as e: + logging.warning("Unable to load setup.py metadata: %s", e) + self.distribution = None + else: + self.has_setup_py = False + self.distribution = None + try: - self.result = run_setup(os.path.abspath(path), stop_after="init") - except RuntimeError as e: - logging.warning("Unable to load setup.py metadata: %s", e) - self.result = None + self.pyproject = self.load_toml() + except FileNotFoundError: + self.pyproject = None + + def load_toml(self): + import toml + + with open(os.path.join(self.path, "pyproject.toml"), "r") as pf: + return toml.load(pf) def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) @@ -184,32 +200,54 @@ class SetupPy(BuildSystem): def test(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["test"], fixers) + if self.has_setup_py: + self._run_setup(session, resolver, ["test"], fixers) + else: + raise NotImplementedError def build(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["build"], fixers) + if self.has_setup_py: + self._run_setup(session, resolver, ["build"], fixers) + else: + raise NotImplementedError def dist(self, session, resolver, fixers, quiet=False): self.setup(resolver) - preargs = [] - if quiet: - preargs.append("--quiet") - self._run_setup(session, resolver, preargs + ["sdist"], fixers) + if self.has_setup_py: + preargs = [] + if quiet: + preargs.append("--quiet") + self._run_setup(session, resolver, preargs + ["sdist"], fixers) + return + elif self.pyproject: + if "poetry" in self.pyproject.get("tool", []): + logging.debug( + "Found pyproject.toml with poetry section, " "assuming poetry project." + ) + run_with_build_fixers(session, ["poetry", "build", "-f", "sdist"], fixers) + return + raise AssertionError("no supported section in pyproject.toml") def clean(self, session, resolver, fixers): self.setup(resolver) - self._run_setup(session, resolver, ["clean"], fixers) + if self.has_setup_py: + self._run_setup(session, resolver, ["clean"], fixers) + else: + raise NotImplementedError def install(self, session, resolver, fixers, install_target): self.setup(resolver) - extra_args = [] - if install_target.user: - extra_args.append("--user") - self._run_setup(session, resolver, ["install"] + extra_args, fixers) + if self.has_setup_py: + extra_args = [] + if install_target.user: + extra_args.append("--user") + self._run_setup(session, resolver, ["install"] + extra_args, fixers) + else: + raise NotImplementedError def _run_setup(self, session, resolver, args, fixers): - interpreter = shebang_binary(self.path) + interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) if interpreter is not None: resolver.install([BinaryRequirement(interpreter)]) run_with_build_fixers(session, ["./setup.py"] + args, fixers) @@ -219,34 +257,50 @@ class SetupPy(BuildSystem): run_with_build_fixers(session, ["python3", "./setup.py"] + args, fixers) def get_declared_dependencies(self): - if self.result is None: + if self.distribution is None: raise NotImplementedError - for require in self.result.get_requires(): + for require in self.distribution.get_requires(): yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.result, "setup_requires", []): - for require in self.result.setup_requires: + if getattr(self.distribution, "setup_requires", []): + for require in self.distribution.setup_requires: yield "build", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.result, "install_requires", []): - for require in self.result.install_requires: + if getattr(self.distribution, "install_requires", []): + for require in self.distribution.install_requires: yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.result, "tests_require", []): - for require in self.result.tests_require: + if getattr(self.distribution, "tests_require", []): + for require in self.distribution.tests_require: yield "test", PythonPackageRequirement.from_requirement_str(require) + if self.pyproject: + if "build-system" in self.pyproject: + for require in self.pyproject['build-system'].get("requires", []): + yield "build", PythonPackageRequirement.from_requirement_str(require) def get_declared_outputs(self): - if self.result is None: + if self.distribution is None: raise NotImplementedError - for script in self.result.scripts or []: + for script in self.distribution.scripts or []: yield BinaryOutput(os.path.basename(script)) - entry_points = getattr(self.result, "entry_points", None) or {} + entry_points = getattr(self.distribution, "entry_points", None) or {} for script in entry_points.get("console_scripts", []): yield BinaryOutput(script.split("=")[0]) - for package in self.result.packages or []: + for package in self.distribution.packages or []: yield PythonPackageOutput(package, python_version="cpython3") + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "setup.py")): + logging.debug("Found setup.py, assuming python project.") + return cls(path) + if os.path.exists(os.path.join(path, "setup.cfg")): + logging.debug("Found setup.py, assuming python project.") + return cls(path) + if os.path.exists(os.path.join(path, "pyproject.toml")): + logging.debug("Found pyproject.toml, assuming python project.") + return cls(path) + class Gradle(BuildSystem): @@ -348,57 +402,6 @@ class Meson(BuildSystem): return Meson(os.path.join(path, "meson.build")) -class PyProject(BuildSystem): - - name = "pyproject" - - def __init__(self, path): - self.path = path - self.pyproject = self.load_toml() - - def __repr__(self): - return "%s(%r)" % (type(self).__name__, self.path) - - def load_toml(self): - import toml - - with open(self.path, "r") as pf: - return toml.load(pf) - - def dist(self, session, resolver, fixers, quiet=False): - if "poetry" in self.pyproject.get("tool", []): - logging.debug( - "Found pyproject.toml with poetry section, " "assuming poetry project." - ) - run_with_build_fixers(session, ["poetry", "build", "-f", "sdist"], fixers) - return - raise AssertionError("no supported section in pyproject.toml") - - def get_declared_dependencies(self): - if "build-system" in self.pyproject: - for require in self.pyproject['build-system'].get("requires", []): - yield "build", PythonPackageRequirement.from_requirement_str(require) - - -class SetupCfg(BuildSystem): - - name = "setup.cfg" - - def __init__(self, path): - self.path = path - - def setup(self, resolver): - resolver.install( - [ - PythonPackageRequirement("pep517"), - ] - ) - - def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) - session.check_call(["python3", "-m", "pep517.build", "-s", "."]) - - class Npm(BuildSystem): name = "npm" @@ -527,7 +530,6 @@ class DistInkt(BuildSystem): return cls(os.path.join(path, "dist.ini")) - class Make(BuildSystem): name = "make" @@ -803,21 +805,11 @@ class Cabal(BuildSystem): def detect_buildsystems(path, trust_package=False): # noqa: C901 """Detect build systems.""" - for bs_cls in [Pear, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, DistInkt, Gem, Make]: + for bs_cls in [Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, DistInkt, Gem, Make]: bs = bs_cls.probe(path) if bs is not None: yield bs - if os.path.exists(os.path.join(path, "setup.py")): - logging.debug("Found setup.py, assuming python project.") - yield SetupPy(os.path.join(path, "setup.py")) - elif os.path.exists(os.path.join(path, "setup.cfg")): - logging.debug("Found setup.cfg, assuming python project.") - yield SetupCfg(os.path.join(path, "setup.cfg")) - if os.path.exists(os.path.join(path, "pyproject.toml")): - logging.debug("Found pyproject.toml, assuming python project.") - yield PyProject(os.path.join(path, "pyproject.toml")) - seen_golang = False if os.path.exists(os.path.join(path, ".travis.yml")): import ruamel.yaml.reader diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 6daefa5..4b0bb70 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -17,8 +17,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging -import re -from typing import List, Optional, Set +from typing import List, Optional import os from buildlog_consultant.apt import ( diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 0d6ead5..ca9aa00 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -145,7 +145,7 @@ def load_apt_cache_file(url, cache_dir): p = os.path.join(cache_dir, fn + ext) if not os.path.exists(p): continue - #return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) + # return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) logging.debug("Loading cached contents file %s", p) if ext == '.lz4': import lz4.frame diff --git a/ognibuild/dist.py b/ognibuild/dist.py index c094230..25d2d84 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -20,7 +20,6 @@ import logging import os import shutil import sys -import tempfile import time from typing import Optional From 9ad9309267376140263cecb3f7af09073825b2a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 16:49:31 +0000 Subject: [PATCH 052/252] Add qmake support. --- ognibuild/buildsystem.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index c532e8d..650b51a 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -579,6 +579,9 @@ class Make(BuildSystem): if not makefile_exists() and session.exists("configure"): run_with_build_fixers(session, ["./configure"], fixers) + if not makefile_exists() and any([n.endswith('.pro') for n in session.scandir(".")]): + run_with_build_fixers(session, ["qmake"], fixers) + def build(self, session, resolver, fixers): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "all"], fixers) @@ -684,6 +687,10 @@ class Make(BuildSystem): ] ): return cls(path) + for n in os.scandir(path): + # qmake + if n.name.endswith('.pro'): + return cls(path) class Cargo(BuildSystem): From 4c40bc69d1a75540ff28b9a38b60eaa9c9c05363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 16:49:50 +0000 Subject: [PATCH 053/252] list cmake. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8d9b19d..c595135 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ issues (or lack of support for a particular ecosystem), please file a bug. - autoconf/automake - CMake - Makefile.PL + - qmake - Maven - ninja, including ninja file generators: - meson From cb1adaa8a04d02ea3915e628eb388ff54ace7096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 16:57:44 +0000 Subject: [PATCH 054/252] Support Module::Build::Tiny. --- README.md | 2 ++ ognibuild/buildsystem.py | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c595135..39b2ddc 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ issues (or lack of support for a particular ecosystem), please file a bug. - ninja, including ninja file generators: - meson - Node +- Perl + - Module::Build::Tiny - PHP Pear - Python - setup.py/setup.cfg/pyproject.toml - Ruby gems diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 650b51a..a3dd10f 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -810,9 +810,47 @@ class Cabal(BuildSystem): return cls(os.path.join(path, "Setup.hs")) +class PerlBuildTiny(BuildSystem): + + name = "perl-build-tiny" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def setup(self, session, fixers): + run_with_build_fixers(session, ["perl", "Build.PL"], fixers) + + def test(self, session, resolver, fixers): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "test"], fixers) + + def build(self, session, resolver, fixers): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "build"], fixers) + + def clean(self, session, resolver, fixers): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "clean"], fixers) + + def install(self, session, resolver, fixers, install_target): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "install"], fixers) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "Build.PL")): + logging.debug("Found Build.PL, assuming Module::Build::Tiny package.") + return cls(path) + + def detect_buildsystems(path, trust_package=False): # noqa: C901 """Detect build systems.""" - for bs_cls in [Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, DistInkt, Gem, Make]: + for bs_cls in [ + Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, + DistInkt, Gem, Make, PerlBuildTiny]: bs = bs_cls.probe(path) if bs is not None: yield bs From 5ec4638487e93fe6af9697dde1417d33968c3bb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 17:01:10 +0000 Subject: [PATCH 055/252] Make relfilename a property. --- ognibuild/requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 17ba1fe..afd6fc8 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -107,6 +107,7 @@ class PerlModuleRequirement(Requirement): self.filename = filename self.inc = inc + @property def relfilename(self): return self.module.replace("::", "/") + ".pm" From ba8f6d442c32904698c793d3897a39ca413de742 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 17:13:55 +0000 Subject: [PATCH 056/252] Don't treat DirEntry as a string. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index a3dd10f..d30b397 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -579,7 +579,7 @@ class Make(BuildSystem): if not makefile_exists() and session.exists("configure"): run_with_build_fixers(session, ["./configure"], fixers) - if not makefile_exists() and any([n.endswith('.pro') for n in session.scandir(".")]): + if not makefile_exists() and any([n.name.endswith('.pro') for n in session.scandir(".")]): run_with_build_fixers(session, ["qmake"], fixers) def build(self, session, resolver, fixers): From 65612f8234255a6ce778e9cdd526c143aaf97e54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 17:16:25 +0000 Subject: [PATCH 057/252] make sure retcode is defined. --- ognibuild/fix_build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 5023f9e..7542b2b 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -71,6 +71,7 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF retcode, contents = run_with_tee(session, args) except FileNotFoundError as e: error = MissingCommand(e.args[0]) + retcode = 1 else: if retcode == 0: return From f174e7ce7bb1f8e03eb5d9a9eb79d6e155003cc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 17:26:12 +0000 Subject: [PATCH 058/252] Fix arg. --- ognibuild/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 7542b2b..3676534 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -70,7 +70,7 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF try: retcode, contents = run_with_tee(session, args) except FileNotFoundError as e: - error = MissingCommand(e.args[0]) + error = MissingCommand(args[0]) retcode = 1 else: if retcode == 0: From f2515552e4d49d0264b2891c59b1ab8e6a543d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 17:47:16 +0000 Subject: [PATCH 059/252] Add some __repr__. --- ognibuild/requirements.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index afd6fc8..92730e8 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -111,6 +111,9 @@ class PerlModuleRequirement(Requirement): def relfilename(self): return self.module.replace("::", "/") + ".pm" + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.module) + class NodePackageRequirement(Requirement): @@ -120,6 +123,9 @@ class NodePackageRequirement(Requirement): super(NodePackageRequirement, self).__init__("npm-package") self.package = package + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.package) + class CargoCrateRequirement(Requirement): From eade730fc3b10b57f00c54d8a2964245b9160087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 18:36:02 +0000 Subject: [PATCH 060/252] Fix == in python version string matching. --- ognibuild/buildsystem.py | 6 ++++++ ognibuild/resolver/apt.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index d30b397..439b99e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -529,6 +529,12 @@ class DistInkt(BuildSystem): ): return cls(os.path.join(path, "dist.ini")) + def get_declared_dependencies(self): + import subprocess + out = subprocess.check_output(["dzil", "authordeps"]) + for entry in out.splitlines(): + yield "build", PerlModuleRequirement(entry.decode()) + class Make(BuildSystem): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index ffc6934..a1d33e5 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -108,7 +108,7 @@ def python_spec_to_apt_rels(pkg_name, specs): rels.extend([{"name": pkg_name, "version": ('>=', deb_version)}, {"name": pkg_name, "version": ('<<', next_maj_deb_version)}]) else: - c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "=": "="}[spec[0]] + c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] rels.append([{"name": pkg_name, "version": (c, deb_version)}]) return rels From a448f81761e5b3c06da245bf6a0c9e171f1b8798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 19:43:31 +0000 Subject: [PATCH 061/252] Fix call to mkdir. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 439b99e..085aaee 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -372,7 +372,7 @@ class Meson(BuildSystem): def _setup(self, session, fixers): if session.exists("build"): return - os.mkdir(os.path.join(session.location, "build")) + session.check_call(['mkdir', 'build']) run_with_build_fixers(session, ["meson", "setup", "build"], fixers) def clean(self, session, resolver, fixers): From 4759ae87bb0b57f93d033ca543b3690377f61c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 17 Mar 2021 20:09:56 +0000 Subject: [PATCH 062/252] Shorten __repr__. --- ognibuild/debian/fix_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 855d880..2f171c2 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -536,7 +536,8 @@ class SimpleBuildFixer(BuildFixer): self._fn = fn def __repr__(self): - return "%s(%r, %r)" % (type(self).__name__, self._problem_cls, self._fn) + return "%s(%r, %r)" % ( + type(self).__name__, self._problem_cls.__name__, self._fn.__name__) def can_fix(self, problem: Problem): return isinstance(problem, self._problem_cls) From 6321f09c664608f1039c01e3afb355d44ac78123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 12:08:29 +0000 Subject: [PATCH 063/252] Fix style. --- ognibuild/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 3676534..56a495b 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -69,7 +69,7 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF while True: try: retcode, contents = run_with_tee(session, args) - except FileNotFoundError as e: + except FileNotFoundError: error = MissingCommand(args[0]) retcode = 1 else: From 7b6fecc2acd178cce96c3ff9861535e65303ffc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 12:08:36 +0000 Subject: [PATCH 064/252] Use probe for golang. --- ognibuild/buildsystem.py | 44 +++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 085aaee..f25cee5 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -764,6 +764,25 @@ class Golang(BuildSystem): def clean(self, session, resolver, fixers): session.check_call(["go", "clean"]) + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, ".travis.yml")): + import ruamel.yaml.reader + + with open(os.path.join(path, ".travis.yml"), "rb") as f: + try: + data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) + except ruamel.yaml.reader.ReaderError as e: + warnings.warn("Unable to parse .travis.yml: %s" % (e,)) + else: + language = data.get("language") + if language == "go": + return Golang(path) + + for entry in os.scandir(path): + if entry.name.endswith(".go"): + return Golang(path) + class Maven(BuildSystem): @@ -852,36 +871,15 @@ class PerlBuildTiny(BuildSystem): return cls(path) -def detect_buildsystems(path, trust_package=False): # noqa: C901 +def detect_buildsystems(path, trust_package=False): """Detect build systems.""" for bs_cls in [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistInkt, Gem, Make, PerlBuildTiny]: + DistInkt, Gem, Make, PerlBuildTiny, Golang]: bs = bs_cls.probe(path) if bs is not None: yield bs - seen_golang = False - if os.path.exists(os.path.join(path, ".travis.yml")): - import ruamel.yaml.reader - - with open(os.path.join(path, ".travis.yml"), "rb") as f: - try: - data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) - except ruamel.yaml.reader.ReaderError as e: - warnings.warn("Unable to parse .travis.yml: %s" % (e,)) - else: - language = data.get("language") - if language == "go": - yield Golang(path) - seen_golang = True - - if not seen_golang: - for entry in os.scandir(path): - if entry.name.endswith(".go"): - yield Golang(path) - break - def get_buildsystem(path, trust_package=False): for buildsystem in detect_buildsystems(path, trust_package=trust_package): From bd1d18a890909b954c0da3e7d15b3a64a97794bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 13:56:53 +0000 Subject: [PATCH 065/252] Probe from travis config. --- ognibuild/buildsystem.py | 54 ++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f25cee5..597dbf4 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -79,6 +79,14 @@ class BuildSystem(object): def get_declared_outputs(self): raise NotImplementedError(self.get_declared_outputs) + @classmethod + def probe(cls, path): + return None + + @classmethod + def probe_from_travis(cls, path, data): + return None + class Pear(BuildSystem): @@ -766,22 +774,19 @@ class Golang(BuildSystem): @classmethod def probe(cls, path): - if os.path.exists(os.path.join(path, ".travis.yml")): - import ruamel.yaml.reader - - with open(os.path.join(path, ".travis.yml"), "rb") as f: - try: - data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) - except ruamel.yaml.reader.ReaderError as e: - warnings.warn("Unable to parse .travis.yml: %s" % (e,)) - else: - language = data.get("language") - if language == "go": - return Golang(path) - for entry in os.scandir(path): if entry.name.endswith(".go"): return Golang(path) + if entry.is_dir(): + for entry in os.scandir(entry.path): + if entry.name.endswith(".go"): + return Golang(path) + + @classmethod + def probe_from_travis(cls, path, data): + language = data.get("language") + if language == "go": + return cls(path) class Maven(BuildSystem): @@ -871,15 +876,32 @@ class PerlBuildTiny(BuildSystem): return cls(path) +BUILDSYSTEM_CLSES = [ + Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, + DistInkt, Gem, Make, PerlBuildTiny, Golang] + + def detect_buildsystems(path, trust_package=False): """Detect build systems.""" - for bs_cls in [ - Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistInkt, Gem, Make, PerlBuildTiny, Golang]: + for bs_cls in BUILDSYSTEM_CLSES: bs = bs_cls.probe(path) if bs is not None: yield bs + if os.path.exists(os.path.join(path, ".travis.yml")): + import ruamel.yaml.reader + + with open(os.path.join(path, ".travis.yml"), "rb") as f: + try: + data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) + except ruamel.yaml.reader.ReaderError as e: + warnings.warn("Unable to parse .travis.yml: %s" % (e,)) + else: + for bs_cls in BUILDSYSTEM_CLSES: + bs = bs_cls.probe_from_travis(path, data) + if bs is not None: + yield bs + def get_buildsystem(path, trust_package=False): for buildsystem in detect_buildsystems(path, trust_package=trust_package): From 37f56f855628e7256fe9f10df5ac33bd25cf018d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 13:58:32 +0000 Subject: [PATCH 066/252] Drop travis probing. --- ognibuild/buildsystem.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 597dbf4..7c0e90c 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -83,10 +83,6 @@ class BuildSystem(object): def probe(cls, path): return None - @classmethod - def probe_from_travis(cls, path, data): - return None - class Pear(BuildSystem): @@ -782,12 +778,6 @@ class Golang(BuildSystem): if entry.name.endswith(".go"): return Golang(path) - @classmethod - def probe_from_travis(cls, path, data): - language = data.get("language") - if language == "go": - return cls(path) - class Maven(BuildSystem): @@ -888,20 +878,6 @@ def detect_buildsystems(path, trust_package=False): if bs is not None: yield bs - if os.path.exists(os.path.join(path, ".travis.yml")): - import ruamel.yaml.reader - - with open(os.path.join(path, ".travis.yml"), "rb") as f: - try: - data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) - except ruamel.yaml.reader.ReaderError as e: - warnings.warn("Unable to parse .travis.yml: %s" % (e,)) - else: - for bs_cls in BUILDSYSTEM_CLSES: - bs = bs_cls.probe_from_travis(path, data) - if bs is not None: - yield bs - def get_buildsystem(path, trust_package=False): for buildsystem in detect_buildsystems(path, trust_package=trust_package): From 84c8a1abda3a9c2590813e9d8a6bbb410d4a5301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 14:16:52 +0000 Subject: [PATCH 067/252] Try the next level. --- ognibuild/buildsystem.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 7c0e90c..b1a9320 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -873,6 +873,19 @@ BUILDSYSTEM_CLSES = [ def detect_buildsystems(path, trust_package=False): """Detect build systems.""" + ret = [] + ret.extend(_detect_buildsystems_directory(path)) + + if not ret: + # Nothing found. Try the next level? + for entry in os.scandir(path): + if entry.is_dir(): + ret.extend(_detect_buildsystems_directory(entry.path)) + + return ret + + +def _detect_buildsystems_directory(path): for bs_cls in BUILDSYSTEM_CLSES: bs = bs_cls.probe(path) if bs is not None: From 97da5ea6bda42636f5c0e6802672dfea66f8b65c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 14:59:57 +0000 Subject: [PATCH 068/252] Add R/CRAN support. --- README.md | 2 ++ ognibuild/buildsystem.py | 24 ++++++++++++++++++++- ognibuild/resolver/__init__.py | 39 ++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 39b2ddc..61dae9e 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ issues (or lack of support for a particular ecosystem), please file a bug. - Module::Build::Tiny - PHP Pear - Python - setup.py/setup.cfg/pyproject.toml +- R - Ruby gems - Waf @@ -69,6 +70,7 @@ The following "native" repositories are supported: - hackage - npm - cargo +- cran - golang\* As well one distribution repository: diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index b1a9320..fe245a9 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -363,6 +363,28 @@ class Gradle(BuildSystem): session, [self.executable, "installDist"], fixers) +class R(BuildSystem): + + name = "R" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + def build(self, session, resolver, fixers): + run_with_build_fixers(session, ["R", "CMD", "build", "."], fixers) + + def test(self, session, resolver, fixers): + run_with_build_fixers(session, ["R", "CMD", "test", "."], fixers) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, 'DESCRIPTION')): + return cls(path) + + class Meson(BuildSystem): name = "meson" @@ -868,7 +890,7 @@ class PerlBuildTiny(BuildSystem): BUILDSYSTEM_CLSES = [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistInkt, Gem, Make, PerlBuildTiny, Golang] + DistInkt, Gem, Make, PerlBuildTiny, Golang, R] def detect_buildsystems(path, trust_package=False): diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 4dacd35..a0c0e71 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -76,6 +76,44 @@ class CPANResolver(Resolver): raise UnsatisfiedRequirements(missing) +class CRANResolver(Resolver): + def __init__(self, session, repos='"http://cran.r-project.org'): + self.session = session + self.repos = repos + + def __str__(self): + return "cran" + + def __repr__(self): + return "%s(%r, %r)" % (type(self).__name__, self.session, self.repos) + + def _cmd(self, req): + return ["R", "-e", "install.packages('%s', repos=%r)" % (req.package, self.repos)] + + def explain(self, requirements): + from ..requirements import RPackageRequirement + + rreqs = [] + for requirement in requirements: + if not isinstance(requirement, RPackageRequirement): + continue + rreqs.append(requirement) + if rreqs: + yield ([self._cmd(req) for req in rreqs]) + + def install(self, requirements): + from ..requirements import RPackageRequirement + + missing = [] + for requirement in requirements: + if not isinstance(requirement, RPackageRequirement): + missing.append(requirement) + continue + self.session.check_call(self._cmd(requirement)) + if missing: + raise UnsatisfiedRequirements(missing) + + class HackageResolver(Resolver): def __init__(self, session): self.session = session @@ -268,6 +306,7 @@ NATIVE_RESOLVER_CLS = [ NpmResolver, GoResolver, HackageResolver, + CRANResolver, ] From c9e2db373f706d8e3a723a1e97317efe78d0568c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 17:06:36 +0000 Subject: [PATCH 069/252] Don't use quotes. --- ognibuild/debian/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 2f171c2..71ac72b 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -536,7 +536,7 @@ class SimpleBuildFixer(BuildFixer): self._fn = fn def __repr__(self): - return "%s(%r, %r)" % ( + return "%s(%s, %s)" % ( type(self).__name__, self._problem_cls.__name__, self._fn.__name__) def can_fix(self, problem: Problem): From 3aeb984147b512bcbfe824427ef968c9109438a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 19:56:44 +0000 Subject: [PATCH 070/252] Install missing dependencies when figuring out declared dependencies. --- ognibuild/__main__.py | 8 +- ognibuild/buildsystem.py | 191 ++++++++++++++++++++++++---------- ognibuild/info.py | 4 +- ognibuild/session/__init__.py | 1 + ognibuild/session/plain.py | 5 +- ognibuild/session/schroot.py | 3 +- 6 files changed, 148 insertions(+), 64 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index f9a5fc6..47e2f75 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -47,12 +47,12 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(session, resolver, buildsystems, stages, explain=False): +def install_necessary_declared_requirements(session, resolver, fixers, buildsystems, stages, explain=False): relevant = [] declared_reqs = [] for buildsystem in buildsystems: try: - declared_reqs.extend(buildsystem.get_declared_dependencies()) + declared_reqs.extend(buildsystem.get_declared_dependencies(session, fixers)) except NotImplementedError: logging.warning( "Unable to determine declared dependencies from %r", buildsystem @@ -169,17 +169,17 @@ def main(): # noqa: C901 bss = list(detect_buildsystems(args.directory)) logging.info( "Detected buildsystems: %s", ', '.join(map(str, bss))) + fixers = determine_fixers(session, resolver, explain=args.explain) if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: logging.info("Checking that declared requirements are present") try: install_necessary_declared_requirements( - session, resolver, bss, stages, explain=args.explain) + session, resolver, fixers, bss, stages, explain=args.explain) except ExplainInstall as e: display_explain_commands(e.commands) return 1 - fixers = determine_fixers(session, resolver, explain=args.explain) if args.subcommand == "dist": from .dist import run_dist diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index fe245a9..a3c7a74 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -73,10 +73,10 @@ class BuildSystem(object): def install(self, session, resolver, fixers, install_target): raise NotImplementedError(self.install) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): raise NotImplementedError(self.get_declared_dependencies) - def get_declared_outputs(self): + def get_declared_outputs(self, session, fixers=None): raise NotImplementedError(self.get_declared_outputs) @classmethod @@ -124,7 +124,6 @@ class Pear(BuildSystem): # run_setup, but setting __name__ # Imported from Python's distutils.core, Copyright (C) PSF - def run_setup(script_name, script_args=None, stop_after="run"): from distutils import core import sys @@ -154,36 +153,64 @@ def run_setup(script_name, script_args=None, stop_after="run"): # (ie. error)? pass - if core._setup_distribution is None: - raise RuntimeError( - ( - "'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?" - ) - % script_name - ) - return core._setup_distribution +_setup_wrapper = """\ +from distutils import core +import sys + +script_name = %(script_name)s + +save_argv = sys.argv.copy() +g = {"__file__": script_name, "__name__": "__main__"} +try: + core._setup_stop_after = "init" + sys.argv[0] = script_name + with open(script_name, "rb") as f: + exec(f.read(), g) +except SystemExit: + # Hmm, should we do something if exiting with a non-zero code + # (ie. error)? + pass + +if core._setup_distribution is None: + raise RuntimeError( + ( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" + ) + % script_name + ) + +d = core._setup_distribution +r = { + 'setup_requires': getattr(d, "setup_requires", []), + 'install_requires': getattr(d, "install_requires", []), + 'tests_require': getattr(d, "tests_require", []), + 'scripts': getattr(d, "scripts", []) or [], + 'entry_points': getattr(d, "entry_points", None) or {}, + 'packages': getattr(d, "packages", []) or [], + 'requires': d.get_requires() or [], + } +import os +import json +with open(%(output_path)s, 'w') as f: + json.dump(r, f) +""" + + class SetupPy(BuildSystem): name = "setup.py" + DEFAULT_PYTHON = 'python3' def __init__(self, path): self.path = path if os.path.exists(os.path.join(self.path, 'setup.py')): self.has_setup_py = True - # TODO(jelmer): Perhaps run this in session, so we can install - # missing dependencies? - try: - self.distribution = run_setup(os.path.abspath(os.path.join(self.path, 'setup.py')), stop_after="init") - except RuntimeError as e: - logging.warning("Unable to load setup.py metadata: %s", e) - self.distribution = None else: self.has_setup_py = False - self.distribution = None try: self.pyproject = self.load_toml() @@ -196,28 +223,79 @@ class SetupPy(BuildSystem): with open(os.path.join(self.path, "pyproject.toml"), "r") as pf: return toml.load(pf) + def _extract_setup(self, session=None, fixers=None): + if session is None: + return self._extract_setup_direct() + else: + return self._extract_setup_in_session(session, fixers) + + def _extract_setup_direct(self): + p = os.path.join(self.path, 'setup.py') + try: + d = run_setup(os.path.abspath(p), stop_after="init") + except RuntimeError as e: + logging.warning("Unable to load setup.py metadata: %s", e) + return None + if d is None: + logging.warning( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p)) + return None + + return { + 'setup_requires': getattr(d, "setup_requires", []), + 'install_requires': getattr(d, "install_requires", []), + 'tests_require': getattr(d, "tests_require", []), + 'scripts': getattr(d, "scripts", []), + 'entry_points': getattr(d, "entry_points", None) or {}, + 'packages': getattr(d, "packages", []), + 'requires': d.get_requires() or [], + } + + def _extract_setup_in_session(self, session, fixers=None): + import tempfile + import json + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) + if interpreter is None: + interpreter = self.DEFAULT_PYTHON + output_f = tempfile.NamedTemporaryFile( + dir=os.path.join(session.location, 'tmp'), mode='w+t') + with output_f: + # TODO(jelmer): Perhaps run this in session, so we can install + # missing dependencies? + argv = [interpreter, "-c", + _setup_wrapper + .replace('%(script_name)s', '"setup.py"') + .replace('%(output_path)s', + '"/' + os.path.relpath(output_f.name, session.location) + + '"')] + try: + if fixers is not None: + run_with_build_fixers(session, argv, fixers) + else: + session.check_call(argv, close_fds=False) + except RuntimeError as e: + logging.warning("Unable to load setup.py metadata: %s", e) + return None + output_f.seek(0) + return json.load(output_f) + def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def setup(self, resolver): - pass - def test(self, session, resolver, fixers): - self.setup(resolver) if self.has_setup_py: self._run_setup(session, resolver, ["test"], fixers) else: raise NotImplementedError def build(self, session, resolver, fixers): - self.setup(resolver) if self.has_setup_py: self._run_setup(session, resolver, ["build"], fixers) else: raise NotImplementedError def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) if self.has_setup_py: preargs = [] if quiet: @@ -234,14 +312,12 @@ class SetupPy(BuildSystem): raise AssertionError("no supported section in pyproject.toml") def clean(self, session, resolver, fixers): - self.setup(resolver) if self.has_setup_py: self._run_setup(session, resolver, ["clean"], fixers) else: raise NotImplementedError def install(self, session, resolver, fixers, install_target): - self.setup(resolver) if self.has_setup_py: extra_args = [] if install_target.user: @@ -253,44 +329,50 @@ class SetupPy(BuildSystem): def _run_setup(self, session, resolver, args, fixers): interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) if interpreter is not None: - resolver.install([BinaryRequirement(interpreter)]) run_with_build_fixers(session, ["./setup.py"] + args, fixers) else: # Just assume it's Python 3 - resolver.install([BinaryRequirement("python3")]) - run_with_build_fixers(session, ["python3", "./setup.py"] + args, fixers) + run_with_build_fixers(session, [self.DEFAULT_PYTHON, "./setup.py"] + args, fixers) - def get_declared_dependencies(self): - if self.distribution is None: + def get_declared_dependencies(self, session, fixers=None): + distribution = self._extract_setup(session, fixers) + if distribution is None: raise NotImplementedError - for require in self.distribution.get_requires(): + for require in distribution['requires']: yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.distribution, "setup_requires", []): - for require in self.distribution.setup_requires: - yield "build", PythonPackageRequirement.from_requirement_str(require) + for require in distribution['setup_requires']: + yield "build", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.distribution, "install_requires", []): - for require in self.distribution.install_requires: - yield "core", PythonPackageRequirement.from_requirement_str(require) + for require in distribution['install_requires']: + yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.distribution, "tests_require", []): - for require in self.distribution.tests_require: - yield "test", PythonPackageRequirement.from_requirement_str(require) + for require in distribution['tests_require']: + yield "test", PythonPackageRequirement.from_requirement_str(require) if self.pyproject: if "build-system" in self.pyproject: for require in self.pyproject['build-system'].get("requires", []): yield "build", PythonPackageRequirement.from_requirement_str(require) - def get_declared_outputs(self): - if self.distribution is None: + def get_declared_outputs(self, session, fixers=None): + distribution = self._extract_setup(session, fixers) + if distribution is None: raise NotImplementedError - for script in self.distribution.scripts or []: + for script in distribution['scripts']: yield BinaryOutput(os.path.basename(script)) - entry_points = getattr(self.distribution, "entry_points", None) or {} - for script in entry_points.get("console_scripts", []): + for script in distribution["entry_points"].get("console_scripts", []): yield BinaryOutput(script.split("=")[0]) - for package in self.distribution.packages or []: + packages = set() + for package in sorted(distribution['packages']): + pts = package.split('.') + b = [] + for e in pts: + b.append(e) + if '.'.join(b) in packages: + break + else: + packages.add(package) + for package in packages: yield PythonPackageOutput(package, python_version="cpython3") @classmethod @@ -438,7 +520,7 @@ class Npm(BuildSystem): with open(path, "r") as f: self.package = json.load(f) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): if "devDependencies" in self.package: for name, unused_version in self.package["devDependencies"].items(): # TODO(jelmer): Look at version @@ -555,9 +637,8 @@ class DistInkt(BuildSystem): ): return cls(os.path.join(path, "dist.ini")) - def get_declared_dependencies(self): - import subprocess - out = subprocess.check_output(["dzil", "authordeps"]) + def get_declared_dependencies(self, session, fixers=None): + out = session.check_output(["dzil", "authordeps"]) for entry in out.splitlines(): yield "build", PerlModuleRequirement(entry.decode()) @@ -682,7 +763,7 @@ class Make(BuildSystem): else: raise - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): # TODO(jelmer): Split out the perl-specific stuff? if os.path.exists(os.path.join(self.path, "META.yml")): # See http://module-build.sourceforge.net/META-spec-v1.4.html for @@ -740,7 +821,7 @@ class Cargo(BuildSystem): with open(path, "r") as f: self.cargo = load(f) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): if "dependencies" in self.cargo: for name, details in self.cargo["dependencies"].items(): if isinstance(details, str): diff --git a/ognibuild/info.py b/ognibuild/info.py index 553b620..0ee1cfd 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -21,7 +21,7 @@ def run_info(session, buildsystems): print("%r:" % buildsystem) deps = {} try: - for kind, dep in buildsystem.get_declared_dependencies(): + for kind, dep in buildsystem.get_declared_dependencies(session): deps.setdefault(kind, []).append(dep) except NotImplementedError: print( @@ -35,7 +35,7 @@ def run_info(session, buildsystems): print("\t\t\t%s" % dep) print("") try: - outputs = list(buildsystem.get_declared_outputs()) + outputs = list(buildsystem.get_declared_outputs(session)) except NotImplementedError: print("\tUnable to detect declared outputs for this type of build system") outputs = [] diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index aa93f29..1ef59d5 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -41,6 +41,7 @@ class Session(object): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, + close_fds: bool = True ): raise NotImplementedError(self.check_call) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 58cdfb7..924577a 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -56,9 +56,10 @@ class PlainSession(Session): self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None, - env: Optional[Dict[str, str]] = None): + env: Optional[Dict[str, str]] = None, + close_fds: bool = True): argv = self._prepend_user(user, argv) - return subprocess.check_call(argv, cwd=cwd, env=env) + return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds) def check_output( self, argv: List[str], diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 6e8c236..760c8e6 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -114,9 +114,10 @@ class SchrootSession(Session): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, + close_fds: bool = True ): try: - subprocess.check_call(self._run_argv(argv, cwd, user, env=env)) + subprocess.check_call(self._run_argv(argv, cwd, user, env=env), close_fds=close_fds) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv) From 2ceef03c8adb930315b5078d88158874f3c6f78f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 18 Mar 2021 21:11:21 +0000 Subject: [PATCH 071/252] Add subpath argument. --- ognibuild/vcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/vcs.py b/ognibuild/vcs.py index b9bbf77..45b32c6 100644 --- a/ognibuild/vcs.py +++ b/ognibuild/vcs.py @@ -28,9 +28,9 @@ from buildlog_consultant.sbuild import ( from . import DetailedFailure -def export_vcs_tree(tree, directory): +def export_vcs_tree(tree, directory, subpath=""): try: - export(tree, directory, "dir", None) + export(tree, directory, "dir", None, subdir=(subpath or None)) except OSError as e: if e.errno == errno.ENOSPC: raise DetailedFailure(1, ["export"], NoSpaceOnDevice()) From 4fbc36c7ff5ce2a3114993a7a72c8fecb9426f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 14:41:43 +0000 Subject: [PATCH 072/252] More R package support. --- ognibuild/buildsystem.py | 29 ++++++++++++++++++++++++++++- ognibuild/outputs.py | 12 ++++++++++++ ognibuild/requirements.py | 25 +++++++++++++++++++++++++ ognibuild/resolver/__init__.py | 18 ++++++++++++++++-- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index a3c7a74..7a03938 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -27,6 +27,7 @@ from . import shebang_binary, UnidentifiedError from .outputs import ( BinaryOutput, PythonPackageOutput, + RPackageOutput, ) from .requirements import ( BinaryRequirement, @@ -34,6 +35,7 @@ from .requirements import ( PerlModuleRequirement, NodePackageRequirement, CargoCrateRequirement, + RPackageRequirement, ) from .fix_build import run_with_build_fixers @@ -456,16 +458,41 @@ class R(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def build(self, session, resolver, fixers): + pass + + def dist(self, session, resolver, fixers, quiet=False): run_with_build_fixers(session, ["R", "CMD", "build", "."], fixers) + def install(self, session, resolver, fixers, install_target): + run_with_build_fixers(session, ["R", "CMD", "INSTALL", "."], fixers) + def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["R", "CMD", "test", "."], fixers) + run_with_build_fixers(session, ["R", "CMD", "check", "."], fixers) @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, 'DESCRIPTION')): return cls(path) + def _read_description(self): + path = os.path.join(self.path, 'DESCRIPTION') + from email.parser import BytesParser + with open(path, 'rb') as f: + return BytesParser().parse(f) + + def get_declared_dependencies(self, session, fixers=None): + description = self._read_description() + if 'Suggests' in description: + suggests = [s.strip() for s in description['Suggests'].split(',') if s.strip()] + for s in suggests: + # TODO(jelmer): Look at version + yield "build", RPackageRequirement.from_str(s) + + def get_declared_outputs(self, session, fixers=None): + description = self._read_description() + if 'Package' in description: + yield RPackageOutput(description['Package']) + class Meson(BuildSystem): diff --git a/ognibuild/outputs.py b/ognibuild/outputs.py index ba1bf85..a39caa9 100644 --- a/ognibuild/outputs.py +++ b/ognibuild/outputs.py @@ -46,3 +46,15 @@ class PythonPackageOutput(UpstreamOutput): self.name, self.python_version, ) + + +class RPackageOutput(UpstreamOutput): + def __init__(self, name): + super(RPackageOutput, self).__init__("r-package") + self.name = name + + def __str__(self): + return "R package: %s" % self.name + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.name) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 92730e8..b789845 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -17,6 +17,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import posixpath +import re import subprocess from typing import Optional, List, Tuple, Set @@ -249,6 +250,30 @@ class RPackageRequirement(Requirement): self.package = package self.minimum_version = minimum_version + def __repr__(self): + return "%s(%r, minimum_version=%r)" % ( + type(self).__name__, + self.package, + self.minimum_version, + ) + + def __str__(self): + if self.minimum_version: + return "R package: %s (>= %s)" % (self.package, self.minimum_version) + else: + return "R package: %s" % (self.package,) + + @classmethod + def from_str(cls, text): + # TODO(jelmer): More complex parser + m = re.fullmatch(r'(.*) \(>= (.*)\)', text) + if m: + return cls(m.group(1), m.group(2)) + m = re.fullmatch(r'([^ ]+)', text) + if m: + return cls(m.group(1)) + raise ValueError(text) + class LibraryRequirement(Requirement): diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index a0c0e71..d3ebee6 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -76,8 +76,8 @@ class CPANResolver(Resolver): raise UnsatisfiedRequirements(missing) -class CRANResolver(Resolver): - def __init__(self, session, repos='"http://cran.r-project.org'): +class RResolver(Resolver): + def __init__(self, session, repos): self.session = session self.repos = repos @@ -114,6 +114,19 @@ class CRANResolver(Resolver): raise UnsatisfiedRequirements(missing) +class CRANResolver(RResolver): + + def __init__(self, session): + super(CRANResolver, self).__init__(session, 'http://cran.r-project.org') + + +class BioconductorResolver(RResolver): + + def __init__(self, session): + super(BioconductorResolver, self).__init__( + session, 'https://hedgehog.fhcrc.org/bioconductor') + + class HackageResolver(Resolver): def __init__(self, session): self.session = session @@ -307,6 +320,7 @@ NATIVE_RESOLVER_CLS = [ GoResolver, HackageResolver, CRANResolver, + BioconductorResolver, ] From a3f7f8ba211c7231751a5c48762ace4ef48928a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 15:00:35 +0000 Subject: [PATCH 073/252] Support Suggests/Depends/Imports in DESCRIPTION. --- ognibuild/buildsystem.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 7a03938..8767951 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -449,6 +449,8 @@ class Gradle(BuildSystem): class R(BuildSystem): + # https://r-pkgs.org/description.html + name = "R" def __init__(self, path): @@ -481,11 +483,17 @@ class R(BuildSystem): return BytesParser().parse(f) def get_declared_dependencies(self, session, fixers=None): + def parse_list(t): + return [s.strip() for s in t.split(',') if s.strip()] description = self._read_description() if 'Suggests' in description: - suggests = [s.strip() for s in description['Suggests'].split(',') if s.strip()] - for s in suggests: - # TODO(jelmer): Look at version + for s in parse_list(description['Suggests']): + yield "build", RPackageRequirement.from_str(s) + if 'Depends' in description: + for s in parse_list(description['Depends']): + yield "build", RPackageRequirement.from_str(s) + if 'Imports' in description: + for s in parse_list(description['Imports']): yield "build", RPackageRequirement.from_str(s) def get_declared_outputs(self, session, fixers=None): From 57746c83521e51f1f9da15b977088f36c9281d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 17:05:47 +0000 Subject: [PATCH 074/252] Add scan_buildsystems. --- ognibuild/buildsystem.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 8767951..990dec9 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1009,29 +1009,30 @@ BUILDSYSTEM_CLSES = [ DistInkt, Gem, Make, PerlBuildTiny, Golang, R] -def detect_buildsystems(path, trust_package=False): +def scan_buildsystems(path): """Detect build systems.""" ret = [] - ret.extend(_detect_buildsystems_directory(path)) + ret.extend([('.', bs) for bs in detect_buildsystems(path)]) if not ret: # Nothing found. Try the next level? for entry in os.scandir(path): if entry.is_dir(): - ret.extend(_detect_buildsystems_directory(entry.path)) + ret.extend( + [(entry.name, bs) for bs in detect_buildsystems(entry.path)]) return ret -def _detect_buildsystems_directory(path): +def detect_buildsystems(path): for bs_cls in BUILDSYSTEM_CLSES: bs = bs_cls.probe(path) if bs is not None: yield bs -def get_buildsystem(path, trust_package=False): - for buildsystem in detect_buildsystems(path, trust_package=trust_package): +def get_buildsystem(path): + for buildsystem in detect_buildsystems(path): return buildsystem raise NoBuildToolsFound() From 3d0519d83e0bee532cfb6f151e0db6cdba52be04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 17:09:38 +0000 Subject: [PATCH 075/252] Return a tuple from get_buildsystem. --- ognibuild/buildsystem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 990dec9..e245074 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,7 +20,7 @@ import logging import os import re -from typing import Optional +from typing import Optional, Tuple import warnings from . import shebang_binary, UnidentifiedError @@ -1031,8 +1031,8 @@ def detect_buildsystems(path): yield bs -def get_buildsystem(path): - for buildsystem in detect_buildsystems(path): - return buildsystem +def get_buildsystem(path: str) -> Tuple[str, BuildSystem]: + for subpath, buildsystem in scan_buildsystems(path): + return subpath, buildsystem raise NoBuildToolsFound() From 498aeaa7ef6fd09e95f0d13d389454e13870f39c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 17:29:06 +0000 Subject: [PATCH 076/252] Use SesionAlreadyOpen/NoSessionOpen. --- ognibuild/session/__init__.py | 14 ++++++++++++++ ognibuild/session/plain.py | 12 +++++++++++- ognibuild/session/schroot.py | 12 +++++++++++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 1ef59d5..0908903 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -21,6 +21,20 @@ import sys import subprocess +class NoSessionOpen(Exception): + """There is no session open.""" + + def __init__(self, session): + self.session = session + + +class SessionAlreadyOpen(Exception): + """There is already a session open.""" + + def __init__(self, session): + self.session = session + + class Session(object): def __enter__(self) -> "Session": return self diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 924577a..bec2ae8 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from . import Session +from . import Session, NoSessionOpen, SessionAlreadyOpen import contextlib import os @@ -30,7 +30,12 @@ class PlainSession(Session): location = "/" + def __init__(self): + self.es = None + def _prepend_user(self, user, args): + if self.es is None: + raise NoSessionOpen(self) if user is not None: import getpass if user != getpass.getuser(): @@ -41,12 +46,17 @@ class PlainSession(Session): return "%s()" % (type(self).__name__, ) def __enter__(self) -> "Session": + if self.es is not None: + raise SessionAlreadyOpen(self) self.es = contextlib.ExitStack() self.es.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): + if self.es is None: + raise NoSessionOpen(self) self.es.__exit__(exc_type, exc_val, exc_tb) + self.es = None return False def create_home(self): diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 760c8e6..18dd1f2 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -24,7 +24,7 @@ import tempfile from typing import Optional, List, Dict -from . import Session, SessionSetupFailure +from . import Session, SessionSetupFailure, NoSessionOpen, SessionAlreadyOpen class SchrootSession(Session): @@ -39,8 +39,11 @@ class SchrootSession(Session): self.chroot = chroot self._location = None self._cwd = None + self.session_id = None def _get_location(self) -> str: + if self.session_id is None: + raise NoSessionOpen(self) return ( subprocess.check_output( ["schroot", "--location", "-c", "session:" + self.session_id] @@ -50,9 +53,14 @@ class SchrootSession(Session): ) def _end_session(self) -> None: + if self.session_id is None: + raise NoSessionOpen(self) subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"]) + self.session_id = None def __enter__(self) -> "Session": + if self.es is not None: + raise SessionAlreadyOpen(self) try: self.session_id = ( subprocess.check_output(["schroot", "-c", self.chroot, "-b"]) @@ -87,6 +95,8 @@ class SchrootSession(Session): user: Optional[str] = None, env: Optional[Dict[str, str]] = None, ): + if self.session_id is None: + raise NoSessionOpen(self) base_argv = ["schroot", "-r", "-c", "session:" + self.session_id] if cwd is None: cwd = self._cwd From ca7853464dd8cef05827b66fd6ebbdc4816248e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 17:31:44 +0000 Subject: [PATCH 077/252] Default to empty list. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index e245074..e1a22c1 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -247,7 +247,7 @@ class SetupPy(BuildSystem): return { 'setup_requires': getattr(d, "setup_requires", []), 'install_requires': getattr(d, "install_requires", []), - 'tests_require': getattr(d, "tests_require", []), + 'tests_require': getattr(d, "tests_require", []) or [], 'scripts': getattr(d, "scripts", []), 'entry_points': getattr(d, "entry_points", None) or {}, 'packages': getattr(d, "packages", []), From cfb631896fee171e2504f414afe62b218d599a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 17:39:41 +0000 Subject: [PATCH 078/252] Typo. --- ognibuild/session/schroot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 18dd1f2..c0d0f06 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -59,7 +59,7 @@ class SchrootSession(Session): self.session_id = None def __enter__(self) -> "Session": - if self.es is not None: + if self.session_id is not None: raise SessionAlreadyOpen(self) try: self.session_id = ( From 4ad25dd6ea5d835a75eb7f8958d23eeef9f9c707 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 17:54:13 +0000 Subject: [PATCH 079/252] Don't keep line endings. --- ognibuild/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 56a495b..348d7fe 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -75,7 +75,7 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF else: if retcode == 0: return - lines = ''.join(contents).splitlines(True) + lines = ''.join(contents).splitlines(False) match, error = find_build_failure_description(lines) if error is None: if match: From a29b2d580e809c1553867ab6493ec62347b940bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 18:06:52 +0000 Subject: [PATCH 080/252] Parse out build backend for python stuff. --- ognibuild/buildsystem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index e1a22c1..d32b259 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -219,6 +219,8 @@ class SetupPy(BuildSystem): except FileNotFoundError: self.pyproject = None + self.build_backend = self.pyproject.get("build-system", {}).get('build-backend') + def load_toml(self): import toml @@ -298,6 +300,7 @@ class SetupPy(BuildSystem): raise NotImplementedError def dist(self, session, resolver, fixers, quiet=False): + # TODO(jelmer): Look at self.build_backend if self.has_setup_py: preargs = [] if quiet: From 31200e4ad84c69a38440ce9c1d95d35b12fc84e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 18:08:21 +0000 Subject: [PATCH 081/252] Strip newlines. --- ognibuild/buildsystem.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index d32b259..f6a0f6f 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -709,7 +709,7 @@ class Make(BuildSystem): except UnidentifiedError as e: if ( "Gnulib not yet bootstrapped; " - "run ./bootstrap instead.\n" in e.lines + "run ./bootstrap instead." in e.lines ): run_with_build_fixers(session, ["./bootstrap"], fixers) run_with_build_fixers(session, ["./autogen.sh"], fixers) @@ -754,23 +754,23 @@ class Make(BuildSystem): try: run_with_build_fixers(session, ["make", "dist"], fixers) except UnidentifiedError as e: - if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: + if "make: *** No rule to make target 'dist'. Stop." in e.lines: raise NotImplementedError - elif "make[1]: *** No rule to make target 'dist'. Stop.\n" in e.lines: + elif "make[1]: *** No rule to make target 'dist'. Stop." in e.lines: raise NotImplementedError elif ( "Reconfigure the source tree " - "(via './config' or 'perl Configure'), please.\n" + "(via './config' or 'perl Configure'), please." ) in e.lines: run_with_build_fixers(session, ["./config"], fixers) run_with_build_fixers(session, ["make", "dist"], fixers) elif ( "Please try running 'make manifest' and then run " - "'make dist' again.\n" in e.lines + "'make dist' again." in e.lines ): run_with_build_fixers(session, ["make", "manifest"], fixers) run_with_build_fixers(session, ["make", "dist"], fixers) - elif "Please run ./configure first\n" in e.lines: + elif "Please run ./configure first" in e.lines: run_with_build_fixers(session, ["./configure"], fixers) run_with_build_fixers(session, ["make", "dist"], fixers) elif any( @@ -778,7 +778,7 @@ class Make(BuildSystem): re.match( r"(Makefile|GNUmakefile|makefile):[0-9]+: " r"\*\*\* Missing \'Make.inc\' " - r"Run \'./configure \[options\]\' and retry. Stop.\n", + r"Run \'./configure \[options\]\' and retry. Stop.", line, ) for line in e.lines @@ -948,7 +948,7 @@ class Cabal(BuildSystem): try: run_with_build_fixers(session, ["runhaskell", "Setup.hs"] + args, fixers) except UnidentifiedError as e: - if "Run the 'configure' command first.\n" in e.lines: + if "Run the 'configure' command first." in e.lines: run_with_build_fixers( session, ["runhaskell", "Setup.hs", "configure"], fixers ) From f176b3c596beef58251c735bf5e79e57f2e1c412 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 18:09:42 +0000 Subject: [PATCH 082/252] Default to test_requires being an empty list. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f6a0f6f..a123ad4 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -189,7 +189,7 @@ d = core._setup_distribution r = { 'setup_requires': getattr(d, "setup_requires", []), 'install_requires': getattr(d, "install_requires", []), - 'tests_require': getattr(d, "tests_require", []), + 'tests_require': getattr(d, "tests_require", []) or [], 'scripts': getattr(d, "scripts", []) or [], 'entry_points': getattr(d, "entry_points", None) or {}, 'packages': getattr(d, "packages", []) or [], From a94cf977c0724626b87a317172cc335092ab0a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 18:13:13 +0000 Subject: [PATCH 083/252] only look for build-backend when pyproject is present. --- ognibuild/buildsystem.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index a123ad4..2b6404d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -218,8 +218,9 @@ class SetupPy(BuildSystem): self.pyproject = self.load_toml() except FileNotFoundError: self.pyproject = None - - self.build_backend = self.pyproject.get("build-system", {}).get('build-backend') + self.build_backend = None + else: + self.build_backend = self.pyproject.get("build-system", {}).get('build-backend') def load_toml(self): import toml From c0f150c716b9673cb7797bbb4ebb23460db0c783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 18:24:01 +0000 Subject: [PATCH 084/252] Search for m4 aliases as well. --- ognibuild/resolver/apt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index a1d33e5..323c5d7 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -489,12 +489,14 @@ def resolve_perl_file_req(apt_mgr, req): def _find_aclocal_fun(macro): # TODO(jelmer): Use the API for codesearch.debian.net instead? defun_prefix = b"AC_DEFUN([%s]," % macro.encode("ascii") + au_alias_prefix = b"AU_ALIAS([%s]," % macro.encode("ascii") + prefixes = [defun_prefix, au_alias_prefix] for entry in os.scandir("/usr/share/aclocal"): if not entry.is_file(): continue with open(entry.path, "rb") as f: for line in f: - if line.startswith(defun_prefix): + if any([line.startswith(prefix) for prefix in prefixes]): return entry.path raise KeyError From e5182a4bc9c867d6f17e638cc426abd4bf45e6eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 20:06:28 +0000 Subject: [PATCH 085/252] Install setup requires before invoking setup.py. --- ognibuild/__init__.py | 1 + ognibuild/__main__.py | 19 +++---------------- ognibuild/buildlog.py | 18 ++++++++++++++++++ ognibuild/buildsystem.py | 11 +++++++++++ ognibuild/fix_build.py | 1 - ognibuild/session/schroot.py | 13 +++++++++++-- 6 files changed, 44 insertions(+), 19 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 366f1f8..c7e0bc6 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -69,3 +69,4 @@ class UpstreamOutput(object): def get_declared_dependencies(self): raise NotImplementedError(self.get_declared_dependencies) + diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 47e2f75..ff876b1 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -20,7 +20,7 @@ import os import shlex import sys from . import UnidentifiedError, DetailedFailure -from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall +from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall, install_missing_reqs from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, @@ -60,21 +60,8 @@ def install_necessary_declared_requirements(session, resolver, fixers, buildsyst relevant.extend( get_necessary_declared_requirements(resolver, declared_reqs, stages) ) - missing = [] - for req in relevant: - try: - if not req.met(session): - missing.append(req) - except NotImplementedError: - missing.append(req) - if missing: - if explain: - commands = resolver.explain(missing) - if not commands: - raise UnsatisfiedRequirements(missing) - raise ExplainInstall(commands) - else: - resolver.install(missing) + + install_missing_reqs(session, resolver, relevant, explain=explain) # Types of dependencies: diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 16ed253..a44bef9 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -232,3 +232,21 @@ class ExplainInstallFixer(BuildFixer): if not explanations: return False raise ExplainInstall(explanations) + + +def install_missing_reqs(session, resolver, reqs, explain=False): + missing = [] + for req in reqs: + try: + if not req.met(session): + missing.append(req) + except NotImplementedError: + missing.append(req) + if missing: + if explain: + commands = resolver.explain(missing) + if not commands: + raise UnsatisfiedRequirements(missing) + raise ExplainInstall(commands) + else: + resolver.install(missing) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 2b6404d..f3679b5 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -159,6 +159,7 @@ def run_setup(script_name, script_args=None, stop_after="run"): _setup_wrapper = """\ +import distutils from distutils import core import sys @@ -333,6 +334,16 @@ class SetupPy(BuildSystem): raise NotImplementedError def _run_setup(self, session, resolver, args, fixers): + from .buildlog import install_missing_reqs + distribution = self._extract_setup(session, fixers) + if distribution is not None: + # Install the setup_requires beforehand, since otherwise + # setuptools might fetch eggs instead of our preferred resolver. + install_missing_reqs( + session, + resolver, + [PythonPackageRequirement.from_requirement_str(require) + for require in distribution['setup_requires']]) interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) if interpreter is not None: run_with_build_fixers(session, ["./setup.py"] + args, fixers) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 348d7fe..4489fe3 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -64,7 +64,6 @@ class DependencyContext(object): def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer]): - logging.info("Running %r", args) fixed_errors = [] while True: try: diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index c0d0f06..e44be57 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -52,11 +52,20 @@ class SchrootSession(Session): .decode() ) - def _end_session(self) -> None: + def _end_session(self) -> bool: if self.session_id is None: raise NoSessionOpen(self) - subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"]) + try: + subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"], stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + for line in e.stderr.splitlines(False): + if line.startswith(b'E: '): + logging.error('%s', line[3:].decode(errors='replace')) + logging.warning('Failed to close schroot session %s, leaving stray.', self.session_id) + self.session_id = None + return False self.session_id = None + return True def __enter__(self) -> "Session": if self.session_id is not None: From 514f57435af5cbb2c9cd4de85971736a77da4b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 20:38:08 +0000 Subject: [PATCH 086/252] Some support for setup.cfg files. --- ognibuild/buildsystem.py | 63 +++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f3679b5..92c7402 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -215,6 +215,11 @@ class SetupPy(BuildSystem): else: self.has_setup_py = False + try: + self.config = self.load_setup_cfg() + except FileNotFoundError: + self.config = None + try: self.pyproject = self.load_toml() except FileNotFoundError: @@ -229,6 +234,10 @@ class SetupPy(BuildSystem): with open(os.path.join(self.path, "pyproject.toml"), "r") as pf: return toml.load(pf) + def load_setup_cfg(self): + from setuptools.config import read_configuration + return read_configuration(os.path.join(self.path, 'setup.cfg')) + def _extract_setup(self, session=None, fixers=None): if session is None: return self._extract_setup_direct() @@ -353,34 +362,48 @@ class SetupPy(BuildSystem): def get_declared_dependencies(self, session, fixers=None): distribution = self._extract_setup(session, fixers) - if distribution is None: - raise NotImplementedError - for require in distribution['requires']: - yield "core", PythonPackageRequirement.from_requirement_str(require) - # Not present for distutils-only packages - for require in distribution['setup_requires']: - yield "build", PythonPackageRequirement.from_requirement_str(require) - # Not present for distutils-only packages - for require in distribution['install_requires']: - yield "core", PythonPackageRequirement.from_requirement_str(require) - # Not present for distutils-only packages - for require in distribution['tests_require']: - yield "test", PythonPackageRequirement.from_requirement_str(require) + if distribution is not None: + for require in distribution['requires']: + yield "core", PythonPackageRequirement.from_requirement_str(require) + # Not present for distutils-only packages + for require in distribution['setup_requires']: + yield "build", PythonPackageRequirement.from_requirement_str(require) + # Not present for distutils-only packages + for require in distribution['install_requires']: + yield "core", PythonPackageRequirement.from_requirement_str(require) + # Not present for distutils-only packages + for require in distribution['tests_require']: + yield "test", PythonPackageRequirement.from_requirement_str(require) if self.pyproject: if "build-system" in self.pyproject: for require in self.pyproject['build-system'].get("requires", []): yield "build", PythonPackageRequirement.from_requirement_str(require) + if self.config: + options = self.config.get('options', {}) + for require in options.get('setup_requires', []): + yield "build", PythonPackageRequirement.from_requirement_str(require) + for require in options.get('install_requires', []): + yield "core", PythonPackageRequirement.from_requirement_str(require) def get_declared_outputs(self, session, fixers=None): distribution = self._extract_setup(session, fixers) - if distribution is None: - raise NotImplementedError - for script in distribution['scripts']: - yield BinaryOutput(os.path.basename(script)) - for script in distribution["entry_points"].get("console_scripts", []): - yield BinaryOutput(script.split("=")[0]) + all_packages = set() + if distribution is not None: + for script in distribution['scripts']: + yield BinaryOutput(os.path.basename(script)) + for script in distribution["entry_points"].get("console_scripts", []): + yield BinaryOutput(script.split("=")[0]) + all_packages.update(distribution['packages']) + if self.config: + options = self.config.get('options', {}) + all_packages.update(options.get('packages', [])) + for script in options.get('scripts', []): + yield BinaryOutput(os.path.basename(script)) + for script in options.get("entry_points", {}).get("console_scripts", []): + yield BinaryOutput(script.split("=")[0]) + packages = set() - for package in sorted(distribution['packages']): + for package in sorted(all_packages): pts = package.split('.') b = [] for e in pts: From ba75a235f3429bfdd64bb703b78f772000c2c04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 20:47:03 +0000 Subject: [PATCH 087/252] Handle != in python specs. --- ognibuild/resolver/apt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 323c5d7..7d2350a 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -107,6 +107,9 @@ def python_spec_to_apt_rels(pkg_name, specs): next_maj_deb_version = Version('.'.join(parts)) rels.extend([{"name": pkg_name, "version": ('>=', deb_version)}, {"name": pkg_name, "version": ('<<', next_maj_deb_version)}]) + elif spec[0] == '!=': + rels.extend([{"name": pkg_name, "version": ('>>', deb_version)}, + {"name": pkg_name, "version": ('<<', deb_version)}]) else: c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] rels.append([{"name": pkg_name, "version": (c, deb_version)}]) From 18e258fd17f4bc71822557fe4728b22e5f566d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 20:52:36 +0000 Subject: [PATCH 088/252] Handle missing setup.cfg. --- ognibuild/buildsystem.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 92c7402..943103d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -236,7 +236,10 @@ class SetupPy(BuildSystem): def load_setup_cfg(self): from setuptools.config import read_configuration - return read_configuration(os.path.join(self.path, 'setup.cfg')) + p = os.path.join(self.path, 'setup.cfg') + if os.path.exists(p): + return read_configuration(p) + raise FileNotFoundError(p) def _extract_setup(self, session=None, fixers=None): if session is None: From 8336ac61132ae67d548a17fedcdc3aee4c269544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 20:57:38 +0000 Subject: [PATCH 089/252] Get setup_requires from pyproject and setup.cfg. --- ognibuild/buildlog.py | 2 ++ ognibuild/buildsystem.py | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index a44bef9..d67270b 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -235,6 +235,8 @@ class ExplainInstallFixer(BuildFixer): def install_missing_reqs(session, resolver, reqs, explain=False): + if not reqs: + return missing = [] for req in reqs: try: diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 943103d..5657eed 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -347,15 +347,9 @@ class SetupPy(BuildSystem): def _run_setup(self, session, resolver, args, fixers): from .buildlog import install_missing_reqs - distribution = self._extract_setup(session, fixers) - if distribution is not None: - # Install the setup_requires beforehand, since otherwise - # setuptools might fetch eggs instead of our preferred resolver. - install_missing_reqs( - session, - resolver, - [PythonPackageRequirement.from_requirement_str(require) - for require in distribution['setup_requires']]) + # Install the setup_requires beforehand, since otherwise + # setuptools might fetch eggs instead of our preferred resolver. + install_missing_reqs(session, resolver, list(self._setup_requires())) interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) if interpreter is not None: run_with_build_fixers(session, ["./setup.py"] + args, fixers) @@ -363,6 +357,16 @@ class SetupPy(BuildSystem): # Just assume it's Python 3 run_with_build_fixers(session, [self.DEFAULT_PYTHON, "./setup.py"] + args, fixers) + def _setup_requires(self): + if self.pyproject: + if "build-system" in self.pyproject: + for require in self.pyproject['build-system'].get("requires", []): + yield PythonPackageRequirement.from_requirement_str(require) + if self.config: + options = self.config.get('options', {}) + for require in options.get('setup_requires', []): + yield PythonPackageRequirement.from_requirement_str(require) + def get_declared_dependencies(self, session, fixers=None): distribution = self._extract_setup(session, fixers) if distribution is not None: From a6d333fbd1470f1f6f6b3c174430ad4b5638fdbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 21:40:51 +0000 Subject: [PATCH 090/252] Clarify that home directory creation happens in schroot. --- ognibuild/session/schroot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index e44be57..33e7e3b 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -172,7 +172,7 @@ class SchrootSession(Session): .decode() .rstrip("\n") ) - logging.info("Creating directory %s", home) + logging.info("Creating directory %s in schroot session.", home) self.check_call(["mkdir", "-p", home], cwd="/", user="root") self.check_call(["chown", user, home], cwd="/", user="root") From 4ce2b54c44b32af30e11e40e989e3baae08558cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 20 Mar 2021 00:50:02 +0000 Subject: [PATCH 091/252] Check for go.mod. --- ognibuild/buildsystem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 5657eed..f42b8a2 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -953,6 +953,8 @@ class Golang(BuildSystem): @classmethod def probe(cls, path): + if os.path.exists(os.path.join(path, 'go.mod')): + return Golang(path) for entry in os.scandir(path): if entry.name.endswith(".go"): return Golang(path) From 6bb2b0a847228ba120848842a98aa9d01b43140d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 20 Mar 2021 03:15:24 +0000 Subject: [PATCH 092/252] Add BinaryRequirement.__repr__. --- ognibuild/requirements.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index b789845..fc19f79 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -88,6 +88,9 @@ class BinaryRequirement(Requirement): super(BinaryRequirement, self).__init__("binary") self.binary_name = binary_name + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.binary_name) + def met(self, session): p = session.Popen( ["which", self.binary_name], stdout=subprocess.DEVNULL, From 2aaf74a3845133f4b4c25d39c6d4da5ae3e21aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 19 Mar 2021 21:59:25 +0000 Subject: [PATCH 093/252] If dist is not supported, fall back to exporting. --- ognibuild/dist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 25d2d84..38513db 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -219,7 +219,7 @@ if __name__ == "__main__": packaging_tree=packaging_tree, chroot=args.chroot, ) - except NoBuildToolsFound: + except (NoBuildToolsFound, NotImplementedError): logging.info("No build tools found, falling back to simple export.") export(tree, "dist.tar.gz", "tgz", None) else: From 2269458ee9c49bed041c425ad56f69dd7d60dc41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 20 Mar 2021 12:45:34 +0000 Subject: [PATCH 094/252] Fix handling of MissingPythonDistribution. --- ognibuild/buildlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index d67270b..4b4857e 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -163,7 +163,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 ) elif isinstance(problem, MissingPythonDistribution): return PythonPackageRequirement( - problem.module, + problem.distribution, python_version=problem.python_version, minimum_version=problem.minimum_version, ) From 1edc111bc3bc375aa82bfd5a0d03203cd487c150 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 20 Mar 2021 12:50:23 +0000 Subject: [PATCH 095/252] Strip newlines. --- ognibuild/buildsystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f42b8a2..96326ce 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -681,7 +681,7 @@ class DistInkt(BuildSystem): if not line.startswith(b";;"): continue try: - (key, value) = line[2:].split(b"=", 1) + (key, value) = line[2:].strip().split(b"=", 1) except ValueError: continue if key.strip() == b"class" and value.strip().startswith(b"'Dist::Inkt"): @@ -720,7 +720,7 @@ class DistInkt(BuildSystem): def get_declared_dependencies(self, session, fixers=None): out = session.check_output(["dzil", "authordeps"]) for entry in out.splitlines(): - yield "build", PerlModuleRequirement(entry.decode()) + yield "build", PerlModuleRequirement(entry.decode().strip()) class Make(BuildSystem): From b1aabf40de598232f066da9a74693271e48d4b37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 20 Mar 2021 13:58:59 +0000 Subject: [PATCH 096/252] Pick winner using popcon. --- ognibuild/debian/file_search.py | 16 ++++++++++++-- ognibuild/debian/udd.py | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 ognibuild/debian/udd.py diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index ca9aa00..8f493df 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -280,8 +280,20 @@ def get_package_for_paths( logging.warning( "More than 1 packages found that contain %r: %r", path, candidates ) - # Euhr. Pick the one with the shortest name? - return sorted(candidates, key=len)[0] + try: + from .udd import UDD + except ModuleNotFoundError: + logging.warning('Unable to import UDD, not ranking by popcon') + return sorted(candidates, key=len)[0] + udd = UDD() + udd.connect() + winner = udd.get_most_popular(candidates) + if winner is None: + logging.warning( + 'No relevant popcon information found, not ranking by popcon') + return sorted(candidates, key=len)[0] + logging.info('Picked winner using popcon') + return winner else: return candidates.pop() diff --git a/ognibuild/debian/udd.py b/ognibuild/debian/udd.py new file mode 100644 index 0000000..e45a432 --- /dev/null +++ b/ognibuild/debian/udd.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +# Copyright (C) 2021 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +"""Support for accessing UDD.""" + + +class UDD(object): + + def connect(self): + import psycopg2 + + self._conn = psycopg2.connect( + database="udd", + user="udd-mirror", + password="udd-mirror", + port=5432, + host="udd-mirror.debian.net", + ) + + def get_most_popular(self, packages): + cursor = self._conn.cursor() + cursor.execute( + 'SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1', + (tuple(packages), )) + return cursor.fetchone()[0] From 5030d888c3aa2ea3758fd1ca6523a11fb3496ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 20 Mar 2021 20:23:56 +0000 Subject: [PATCH 097/252] re.escape literals. --- ognibuild/resolver/apt.py | 65 ++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 7d2350a..4392bfc 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -19,6 +19,7 @@ from itertools import chain import logging import os import posixpath +import re from debian.changelog import Version from debian.deb822 import PkgRelation @@ -119,14 +120,14 @@ def python_spec_to_apt_rels(pkg_name, specs): def get_package_for_python_package(apt_mgr, package, python_version, specs=None): if python_version == "pypy": pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % package.replace("-", "_")], + ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_"))], regex=True, ) elif python_version == "cpython2": pkg_name = apt_mgr.get_package_for_paths( [ "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" - % package.replace("-", "_") + % re.escape(package.replace("-", "_")) ], regex=True, ) @@ -134,7 +135,7 @@ def get_package_for_python_package(apt_mgr, package, python_version, specs=None) pkg_name = apt_mgr.get_package_for_paths( [ "/usr/lib/python3/dist-packages/%s-.*.egg-info" - % package.replace("-", "_") + % re.escape(package.replace("-", "_")) ], regex=True, ) @@ -151,50 +152,50 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): paths = [ posixpath.join( "/usr/lib/python3/dist-packages", - module.replace(".", "/"), + re.escape(module.replace(".", "/")), "__init__.py", ), posixpath.join( - "/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py" + "/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")) + ".py" ), posixpath.join( "/usr/lib/python3\\.[0-9]+/lib-dynload", - module.replace(".", "/") + "\\.cpython-.*\\.so", + re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", ), posixpath.join( - "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py" + "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" ), posixpath.join( - "/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py" + "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")), "__init__.py" ), ] elif python_version == "cpython2": paths = [ posixpath.join( "/usr/lib/python2\\.[0-9]/dist-packages", - module.replace(".", "/"), + re.escape(module.replace(".", "/")), "__init__.py", ), posixpath.join( "/usr/lib/python2\\.[0-9]/dist-packages", - module.replace(".", "/") + ".py", + re.escape(module.replace(".", "/")) + ".py", ), posixpath.join( "/usr/lib/python2.\\.[0-9]/lib-dynload", - module.replace(".", "/") + ".so", + re.escape(module.replace(".", "/")) + ".so", ), ] elif python_version == "pypy": paths = [ posixpath.join( - "/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py" + "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")), "__init__.py" ), posixpath.join( - "/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py" + "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" ), posixpath.join( "/usr/lib/pypy/dist-packages", - module.replace(".", "/") + "\\.pypy-.*\\.so", + re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", ), ] else: @@ -225,7 +226,7 @@ def resolve_pkg_config_req(apt_mgr, req): ) if package is None: package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], + [posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], regex=True, ) if package is not None: @@ -246,7 +247,7 @@ def resolve_c_header_req(apt_mgr, req): ) if package is None: package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/include", ".*", req.header)], regex=True + [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True ) if package is None: return None @@ -263,7 +264,7 @@ def resolve_js_runtime_req(apt_mgr, req): def resolve_vala_package_req(apt_mgr, req): - path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package + path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package) package = apt_mgr.get_package_for_paths([path], regex=True) if package is not None: return AptRequirement.simple(package) @@ -274,7 +275,7 @@ def resolve_ruby_gem_req(apt_mgr, req): paths = [ posixpath.join( "/usr/share/rubygems-integration/all/" - "specifications/%s-.*\\.gemspec" % req.gem + "specifications/%s-.*\\.gemspec" % re.escape(req.gem) ) ] package = apt_mgr.get_package_for_paths(paths, regex=True) @@ -285,7 +286,7 @@ def resolve_ruby_gem_req(apt_mgr, req): def resolve_go_package_req(apt_mgr, req): package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/share/gocode/src", req.package, ".*")], regex=True + [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True ) if package is not None: return AptRequirement.simple(package) @@ -309,7 +310,7 @@ def resolve_php_class_req(apt_mgr, req): def resolve_r_package_req(apt_mgr, req): - paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)] + paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] package = apt_mgr.get_package_for_paths(paths, regex=True) if package is not None: return AptRequirement.simple(package) @@ -318,9 +319,9 @@ def resolve_r_package_req(apt_mgr, req): def resolve_node_package_req(apt_mgr, req): paths = [ - "/usr/share/nodejs/.*/node_modules/%s/package.json" % req.package, - "/usr/lib/nodejs/%s/package.json" % req.package, - "/usr/share/nodejs/%s/package.json" % req.package, + "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % req.package, + "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), + "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: @@ -330,10 +331,10 @@ def resolve_node_package_req(apt_mgr, req): def resolve_library_req(apt_mgr, req): paths = [ - posixpath.join("/usr/lib/lib%s.so$" % req.library), - posixpath.join("/usr/lib/.*/lib%s.so$" % req.library), - posixpath.join("/usr/lib/lib%s.a$" % req.library), - posixpath.join("/usr/lib/.*/lib%s.a$" % req.library), + posixpath.join("/usr/lib/lib%s.so$" % re.escape(req.library)), + posixpath.join("/usr/lib/.*/lib%s.so$" % re.escape(req.library)), + posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), + posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is not None: @@ -349,7 +350,7 @@ def resolve_ruby_file_req(apt_mgr, req): paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" - "lib/%s.rb" % req.filename + "lib/%s\\.rb" % re.escape(req.filename) ) ] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) @@ -379,7 +380,7 @@ def resolve_xml_entity_req(apt_mgr, req): def resolve_sprockets_file_req(apt_mgr, req): if req.content_type == "application/javascript": - path = "/usr/share/.*/app/assets/javascripts/%s.js$" % req.name + path = "/usr/share/.*/app/assets/javascripts/%s\\.js$" % re.escape(req.name) else: logging.warning("unable to handle content type %s", req.content_type) return None @@ -409,7 +410,7 @@ def resolve_java_class_req(apt_mgr, req): def resolve_haskell_package_req(apt_mgr, req): - path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0] + path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: return AptRequirement.simple(pkg_name) @@ -453,7 +454,7 @@ def resolve_gnome_common_req(apt_mgr, req): def resolve_jdk_file_req(apt_mgr, req): - path = req.jdk_path + ".*/" + req.filename + path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename) pkg_name = apt_mgr.get_package_for_paths([path], regex=True) if pkg_name is not None: return AptRequirement.simple(pkg_name) @@ -544,7 +545,7 @@ def resolve_python_package_req(apt_mgr, req): def resolve_cargo_crate_req(apt_mgr, req): paths = [ - '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % req.crate] + '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)] pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is None: return None From 49a95cfe4159d92f66d67fce1da87176378a0668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 02:01:09 +0000 Subject: [PATCH 098/252] Invoke pep517 to create source tarball. --- ognibuild/buildsystem.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 96326ce..f22c972 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -322,13 +322,9 @@ class SetupPy(BuildSystem): self._run_setup(session, resolver, preargs + ["sdist"], fixers) return elif self.pyproject: - if "poetry" in self.pyproject.get("tool", []): - logging.debug( - "Found pyproject.toml with poetry section, " "assuming poetry project." - ) - run_with_build_fixers(session, ["poetry", "build", "-f", "sdist"], fixers) - return - raise AssertionError("no supported section in pyproject.toml") + run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source"], fixers) + return + raise AssertionError("no setup.py or pyproject.toml") def clean(self, session, resolver, fixers): if self.has_setup_py: From 88cab07354a4025caaa6ae4b73bc970edfe10ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 02:06:33 +0000 Subject: [PATCH 099/252] Invoke tox if present. --- ognibuild/buildsystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f22c972..6b8dc8d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -302,7 +302,9 @@ class SetupPy(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def test(self, session, resolver, fixers): - if self.has_setup_py: + if os.path.exists(os.path.join(self.path, 'tox.ini')): + run_with_build_fixers(session, ['tox'], fixers) + elif self.has_setup_py: self._run_setup(session, resolver, ["test"], fixers) else: raise NotImplementedError From c657df7b175b649947d919be6a1304b389ebca1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 18:12:43 +0000 Subject: [PATCH 100/252] Split out setup_vcs. --- ognibuild/buildsystem.py | 2 ++ ognibuild/dist.py | 23 ++++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 6b8dc8d..57c294d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -188,6 +188,7 @@ if core._setup_distribution is None: d = core._setup_distribution r = { + 'name': d.name, 'setup_requires': getattr(d, "setup_requires", []), 'install_requires': getattr(d, "install_requires", []), 'tests_require': getattr(d, "tests_require", []) or [], @@ -261,6 +262,7 @@ class SetupPy(BuildSystem): return None return { + 'name': d.name, 'setup_requires': getattr(d, "setup_requires", []), 'install_requires': getattr(d, "install_requires", []), 'tests_require': getattr(d, "tests_require", []) or [], diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 25d2d84..d136ffa 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -35,6 +35,7 @@ from buildlog_consultant.common import ( from . import DetailedFailure from .buildsystem import NoBuildToolsFound +from .session import Session from .session.schroot import SchrootSession @@ -121,10 +122,10 @@ class DistCatcher(object): return False -def create_dist_schroot( +def create_dist( + session: Session, tree: Tree, target_dir: str, - chroot: str, packaging_tree: Optional[Tree] = None, include_controldir: bool = True, subdir: Optional[str] = None, @@ -135,7 +136,7 @@ def create_dist_schroot( if subdir is None: subdir = "package" - with SchrootSession(chroot) as session: + with session: if packaging_tree is not None: from .debian import satisfy_build_deps @@ -165,6 +166,22 @@ def create_dist_schroot( raise DistNoTarball() +def create_dist_schroot( + tree: Tree, + target_dir: str, + chroot: str, + packaging_tree: Optional[Tree] = None, + include_controldir: bool = True, + subdir: Optional[str] = None, +) -> str: + session = SchrootSession(chroot) + return create_dist( + session, tree, target_dir, + packaging_tree=packaging_tree, + include_controldir=include_controldir, + subdir=subdir) + + if __name__ == "__main__": import argparse import breezy.bzr # noqa: F401 From 45e0f797e460944f8e6ba97306a4f016ce61e906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 18:48:24 +0000 Subject: [PATCH 101/252] Factor out create_dist. --- ognibuild/__init__.py | 1 - ognibuild/__main__.py | 1 - ognibuild/dist.py | 36 +++++++++++++++++++++++++++++------- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index c7e0bc6..366f1f8 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -69,4 +69,3 @@ class UpstreamOutput(object): def get_declared_dependencies(self): raise NotImplementedError(self.get_declared_dependencies) - diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index ff876b1..78784f5 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -25,7 +25,6 @@ from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, native_resolvers, - UnsatisfiedRequirements, ) from .resolver.apt import AptResolver diff --git a/ognibuild/dist.py b/ognibuild/dist.py index d136ffa..e71bb4b 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -15,6 +15,13 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +__all__ = [ + 'UnidentifiedError', + 'DetailedFailure', + 'create_dist', + 'create_dist_schroot', + ] + import errno import logging import os @@ -33,7 +40,7 @@ from buildlog_consultant.common import ( ) -from . import DetailedFailure +from . import DetailedFailure, UnidentifiedError from .buildsystem import NoBuildToolsFound from .session import Session from .session.schroot import SchrootSession @@ -129,7 +136,7 @@ def create_dist( packaging_tree: Optional[Tree] = None, include_controldir: bool = True, subdir: Optional[str] = None, -) -> str: +) -> Optional[str]: from .buildsystem import detect_buildsystems from .resolver.apt import AptResolver from .buildlog import InstallFixer @@ -158,9 +165,16 @@ def create_dist( session.chdir(reldir) run_dist(session, buildsystems, resolver, fixers) - for path in dc.files: - shutil.copy(path, target_dir) - return os.path.join(target_dir, os.path.basename(path)) + try: + for path in dc.files: + shutil.copy(path, target_dir) + return os.path.join(target_dir, os.path.basename(path)) + finally: + for path in dc.files: + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.unlink(path) logging.info("No tarball created :(") raise DistNoTarball() @@ -173,7 +187,7 @@ def create_dist_schroot( packaging_tree: Optional[Tree] = None, include_controldir: bool = True, subdir: Optional[str] = None, -) -> str: +) -> Optional[str]: session = SchrootSession(chroot) return create_dist( session, tree, target_dir, @@ -239,6 +253,14 @@ if __name__ == "__main__": except NoBuildToolsFound: logging.info("No build tools found, falling back to simple export.") export(tree, "dist.tar.gz", "tgz", None) + except NotImplementedError: + logging.info("Build system does not support dist tarball creation, " + "falling back to simple export.") + export(tree, "dist.tar.gz", "tgz", None) + except UnidentifiedError as e: + logging.fatal('Unidentified error: %r', e.lines) + except DetailedFailure as e: + logging.fatal('Identified error during dist creation: %s', e.error) else: - print("Created %s" % ret) + logging.info("Created %s", ret) sys.exit(0) From c5a278729e48a18a7750d31678de473c33bf11b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 18:57:24 +0000 Subject: [PATCH 102/252] Use auto_resolver rather than apt resolver. --- ognibuild/dist.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index e71bb4b..b6ef916 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -42,6 +42,7 @@ from buildlog_consultant.common import ( from . import DetailedFailure, UnidentifiedError from .buildsystem import NoBuildToolsFound +from .resolver import auto_resolver from .session import Session from .session.schroot import SchrootSession @@ -138,7 +139,6 @@ def create_dist( subdir: Optional[str] = None, ) -> Optional[str]: from .buildsystem import detect_buildsystems - from .resolver.apt import AptResolver from .buildlog import InstallFixer if subdir is None: @@ -158,7 +158,7 @@ def create_dist( raise buildsystems = list(detect_buildsystems(export_directory)) - resolver = AptResolver.from_session(session) + resolver = auto_resolver(session) fixers = [InstallFixer(resolver)] with DistCatcher(export_directory) as dc: From 04c3a81b9e11f1d6b5a7fdfebbcc244c7a429767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 19:00:08 +0000 Subject: [PATCH 103/252] Make cleanup optional. --- ognibuild/dist.py | 76 +++++++++++++++++++++++++---------------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index b6ef916..c57d5e6 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -129,6 +129,13 @@ class DistCatcher(object): self.find_files() return False + def cleanup(self): + for path in self.files: + if os.path.isdir(path): + shutil.rmtree(path) + else: + os.unlink(path) + def create_dist( session: Session, @@ -137,47 +144,44 @@ def create_dist( packaging_tree: Optional[Tree] = None, include_controldir: bool = True, subdir: Optional[str] = None, + cleanup: bool = False ) -> Optional[str]: from .buildsystem import detect_buildsystems from .buildlog import InstallFixer if subdir is None: subdir = "package" - with session: - if packaging_tree is not None: - from .debian import satisfy_build_deps + if packaging_tree is not None: + from .debian import satisfy_build_deps - satisfy_build_deps(session, packaging_tree) + satisfy_build_deps(session, packaging_tree) - try: - export_directory, reldir = session.setup_from_vcs( - tree, include_controldir=include_controldir, subdir=subdir) - except OSError as e: - if e.errno == errno.ENOSPC: - raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) - raise + try: + export_directory, reldir = session.setup_from_vcs( + tree, include_controldir=include_controldir, subdir=subdir) + except OSError as e: + if e.errno == errno.ENOSPC: + raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) + raise - buildsystems = list(detect_buildsystems(export_directory)) - resolver = auto_resolver(session) - fixers = [InstallFixer(resolver)] + buildsystems = list(detect_buildsystems(export_directory)) + resolver = auto_resolver(session) + fixers = [InstallFixer(resolver)] - with DistCatcher(export_directory) as dc: - session.chdir(reldir) - run_dist(session, buildsystems, resolver, fixers) + with DistCatcher(export_directory) as dc: + session.chdir(reldir) + run_dist(session, buildsystems, resolver, fixers) - try: - for path in dc.files: - shutil.copy(path, target_dir) - return os.path.join(target_dir, os.path.basename(path)) - finally: - for path in dc.files: - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.unlink(path) + try: + for path in dc.files: + shutil.copy(path, target_dir) + return os.path.join(target_dir, os.path.basename(path)) + finally: + if cleanup: + dc.cleanup() - logging.info("No tarball created :(") - raise DistNoTarball() + logging.info("No tarball created :(") + raise DistNoTarball() def create_dist_schroot( @@ -187,13 +191,15 @@ def create_dist_schroot( packaging_tree: Optional[Tree] = None, include_controldir: bool = True, subdir: Optional[str] = None, + cleanup: bool = False ) -> Optional[str]: - session = SchrootSession(chroot) - return create_dist( - session, tree, target_dir, - packaging_tree=packaging_tree, - include_controldir=include_controldir, - subdir=subdir) + with SchrootSession(chroot) as session: + return create_dist( + session, tree, target_dir, + packaging_tree=packaging_tree, + include_controldir=include_controldir, + subdir=subdir, + cleanup=cleanup) if __name__ == "__main__": From 42724a74a88e5e7349e91348acbc397b718acdc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 22:50:19 +0000 Subject: [PATCH 104/252] Add verbose option. --- ognibuild/debian/fix_build.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 71ac72b..0ea8a8b 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -180,7 +180,7 @@ class AutopkgtestDependencyContext(DependencyContext): def add_dependency(self, requirement): return add_test_dependency( self.tree, - self.testname, + self.phase[1], requirement, committer=self.committer, subpath=self.subpath, @@ -510,6 +510,7 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer): def _fix(self, error, context): logging.info("Running 'pg_buildext updatecontrol'") self.session.check_call(["pg_buildext", "updatecontrol"]) + # TODO(jelmer): Copy control file back return commit_debian_changes( context.tree, context.subpath, @@ -655,10 +656,12 @@ def build_incrementally( os.path.join(output_directory, "build.log.%d" % i) ): i += 1 + target_path = os.path.join(output_directory, "build.log.%d" % i) os.rename( os.path.join(output_directory, "build.log"), - os.path.join(output_directory, "build.log.%d" % i), + target_path ) + logging.debug('Storing build log at %s', target_path) def main(argv=None): @@ -701,6 +704,10 @@ def main(argv=None): '--schroot', type=str, help='chroot to use.') + parser.add_argument( + '--verbose', + action='store_true', + help='Be verbose') args = parser.parse_args() from breezy.workingtree import WorkingTree @@ -710,7 +717,10 @@ def main(argv=None): import tempfile import contextlib - logging.basicConfig(level=logging.INFO, format="%(message)s") + if args.verbose: + logging.basicConfig(level=logging.DEBUG, format="%(message)s") + else: + logging.basicConfig(level=logging.INFO, format="%(message)s") with contextlib.ExitStack() as es: if args.output_directory is None: From c38e32516f714fc1708d980de5ad8ec8914d184f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 21 Mar 2021 23:07:46 +0000 Subject: [PATCH 105/252] Add __repr__. --- ognibuild/debian/fix_build.py | 3 +++ ognibuild/resolver/apt.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 0ea8a8b..907a69e 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -507,6 +507,9 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer): def can_fix(self, problem): return isinstance(problem, NeedPgBuildExtUpdateControl) + def __repr__(self): + return "%s()" % (type(self).__name__, ) + def _fix(self, error, context): logging.info("Running 'pg_buildext updatecontrol'") self.session.check_call(["pg_buildext", "updatecontrol"]) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 4392bfc..8cd4f25 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -598,6 +598,9 @@ class AptResolver(Resolver): def __str__(self): return "apt" + def __repr__(self): + return "%s()" % (type(self).__name__, ) + @classmethod def from_session(cls, session): return cls(AptManager.from_session(session)) From b9da3cb2236df05834148bd756bf860e990f056a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 01:51:54 +0000 Subject: [PATCH 106/252] Preserve SETUPTOOLS_SCM_PRETEND_VERSION. --- ognibuild/buildsystem.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 57c294d..e351d0b 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -352,10 +352,16 @@ class SetupPy(BuildSystem): install_missing_reqs(session, resolver, list(self._setup_requires())) interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) if interpreter is not None: - run_with_build_fixers(session, ["./setup.py"] + args, fixers) + argv = ["./setup.py"] + args else: # Just assume it's Python 3 - run_with_build_fixers(session, [self.DEFAULT_PYTHON, "./setup.py"] + args, fixers) + argv = [self.DEFAULT_PYTHON, "./setup.py"] + args + env = {} + # Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment + if 'SETUPTOOLS_SCM_PRETEND_VERSION' in os.environ: + env['SETUPTOOLS_SCM_PRETEND_VERSION'] = ( + os.environ['SETUPTOOLS_SCM_PRETEND_VERSION']) + run_with_build_fixers(session, argv, fixers, env=env) def _setup_requires(self): if self.pyproject: From 41d7d44b57e39b2e1e0439a784c45792ae41457b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 04:59:54 +0000 Subject: [PATCH 107/252] Support packaging subpath in create_dist_schroot. --- ognibuild/__init__.py | 2 ++ ognibuild/debian/__init__.py | 6 +++--- ognibuild/dist.py | 12 +++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index 366f1f8..a74a55b 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -31,6 +31,8 @@ class DetailedFailure(Exception): class UnidentifiedError(Exception): + """An unidentified error.""" + def __init__(self, retcode, argv, lines, secondary=None): self.retcode = retcode self.argv = argv diff --git a/ognibuild/debian/__init__.py b/ognibuild/debian/__init__.py index 8879a4c..23a56a1 100644 --- a/ognibuild/debian/__init__.py +++ b/ognibuild/debian/__init__.py @@ -15,14 +15,14 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import os from debian.deb822 import Deb822 from ..session import Session -# TODO(jelmer): move this to debian/ -def satisfy_build_deps(session: Session, tree): - source = Deb822(tree.get_file("debian/control")) +def satisfy_build_deps(session: Session, tree, debian_path): + source = Deb822(tree.get_file(os.path.join(debian_path, "control"))) deps = [] for name in ["Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"]: try: diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 5277a52..a9bc20f 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -141,7 +141,6 @@ def create_dist( session: Session, tree: Tree, target_dir: str, - packaging_tree: Optional[Tree] = None, include_controldir: bool = True, subdir: Optional[str] = None, cleanup: bool = False @@ -151,11 +150,6 @@ def create_dist( if subdir is None: subdir = "package" - if packaging_tree is not None: - from .debian import satisfy_build_deps - - satisfy_build_deps(session, packaging_tree) - try: export_directory, reldir = session.setup_from_vcs( tree, include_controldir=include_controldir, subdir=subdir) @@ -189,14 +183,18 @@ def create_dist_schroot( target_dir: str, chroot: str, packaging_tree: Optional[Tree] = None, + packaging_subpath: Optional[str] = None, include_controldir: bool = True, subdir: Optional[str] = None, cleanup: bool = False ) -> Optional[str]: with SchrootSession(chroot) as session: + if packaging_tree is not None: + from .debian import satisfy_build_deps + + satisfy_build_deps(session, packaging_tree, packaging_subpath) return create_dist( session, tree, target_dir, - packaging_tree=packaging_tree, include_controldir=include_controldir, subdir=subdir, cleanup=cleanup) From ec126f7ba5cd3862e425eadac2190cb5dde63d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 12:17:09 +0000 Subject: [PATCH 108/252] Support JRE installation. --- ognibuild/buildlog.py | 3 +++ ognibuild/requirements.py | 6 ++++++ ognibuild/resolver/apt.py | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 4b4857e..7d7d4c3 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -35,6 +35,7 @@ from buildlog_consultant.common import ( MissingXmlEntity, MissingJDKFile, MissingJDK, + MissingJRE, MissingNodeModule, MissingPhpClass, MissingRubyGem, @@ -133,6 +134,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return JDKFileRequirement(problem.jdk_path, problem.filename) elif isinstance(problem, MissingJDK): return JDKRequirement() + elif isinstance(problem, MissingJRE): + return JRERequirement() elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": return BinaryRequirement("glib-gettextize") diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index fc19f79..8b6a0b1 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -375,6 +375,12 @@ class JDKRequirement(Requirement): super(JDKRequirement, self).__init__("jdk") +class JRERequirement(Requirement): + + def __init__(self): + super(JRERequirement, self).__init__("jre") + + class PerlFileRequirement(Requirement): filename: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 8cd4f25..cdfa028 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -52,6 +52,7 @@ from ..requirements import ( GnomeCommonRequirement, JDKFileRequirement, JDKRequirement, + JRERequirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, @@ -465,6 +466,10 @@ def resolve_jdk_req(apt_mgr, req): return AptRequirement.simple('default-jdk') +def resolve_jre_req(apt_mgr, req): + return AptRequirement.simple('default-jre') + + def resolve_perl_module_req(apt_mgr, req): DEFAULT_PERL_PATHS = ["/usr/share/perl5"] @@ -575,6 +580,7 @@ APT_REQUIREMENT_RESOLVERS = [ (GnomeCommonRequirement, resolve_gnome_common_req), (JDKFileRequirement, resolve_jdk_file_req), (JDKRequirement, resolve_jdk_req), + (JRERequirement, resolve_jre_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), (AutoconfMacroRequirement, resolve_autoconf_macro_req), From d98fdbc14ee3652e8ab9f3ce1bb58a935bed4c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 12:23:47 +0000 Subject: [PATCH 109/252] Fix missing import. --- ognibuild/buildlog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 7d7d4c3..5609f9c 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -78,6 +78,7 @@ from .requirements import ( GnomeCommonRequirement, JDKFileRequirement, JDKRequirement, + JRERequirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, From fb6d15172af540a8c5303f395315c8f042f95b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 02:10:31 +0000 Subject: [PATCH 110/252] Pass along env to run_with_build_fixers. --- ognibuild/fix_build.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 4489fe3..f32e820 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -16,7 +16,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging -from typing import List, Optional +from typing import List, Optional, Dict from buildlog_consultant.common import ( find_build_failure_description, @@ -63,11 +63,11 @@ class DependencyContext(object): raise NotImplementedError(self.add_dependency) -def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer]): +def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], env: Optional[Dict[str, str]] = None): fixed_errors = [] while True: try: - retcode, contents = run_with_tee(session, args) + retcode, contents = run_with_tee(session, args, env=env) except FileNotFoundError: error = MissingCommand(args[0]) retcode = 1 From 802807f262f0b8a358fdc185d63e67be2ae64804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 13:39:00 +0000 Subject: [PATCH 111/252] Fix typing. --- ognibuild/session/schroot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 33e7e3b..4026669 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -32,6 +32,7 @@ class SchrootSession(Session): _cwd: Optional[str] _location: Optional[str] chroot: str + session_id: Optional[str] def __init__(self, chroot: str): if not isinstance(chroot, str): From a0283a2e4e807014cbc80ecbc4203150b4fdde8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 13:41:15 +0000 Subject: [PATCH 112/252] Require newer buildlog-consultant. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f8d358e..d3e9c72 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup(name="ognibuild", }, install_requires=[ 'breezy', - 'buildlog-consultant', + 'buildlog-consultant>=0.0.4', 'requirements-parser', ], extras_require={ From 7b794a987fde7d1cd7b3c7c6cba76dbbceb1b309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 14:14:15 +0000 Subject: [PATCH 113/252] Better support for user-local. --- ognibuild/__main__.py | 2 +- ognibuild/debian/file_search.py | 2 + ognibuild/resolver/__init__.py | 89 +++++++++++++++++++++++++-------- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 78784f5..1e51a60 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -147,7 +147,7 @@ def main(): # noqa: C901 if args.resolve == "apt": resolver = AptResolver.from_session(session) elif args.resolve == "native": - resolver = native_resolvers(session) + resolver = native_resolvers(session, user_local=args.user) elif args.resolve == "auto": resolver = auto_resolver(session, explain=args.explain) logging.info("Using requirement resolver: %s", resolver) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 8f493df..bd2fcb3 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -280,6 +280,8 @@ def get_package_for_paths( logging.warning( "More than 1 packages found that contain %r: %r", path, candidates ) + # TODO(jelmer): Pick package based on what appears most commonly in + # build-depends{-indep,-arch} try: from .udd import UDD except ModuleNotFoundError: diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index d3ebee6..85b9052 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -39,8 +39,9 @@ class Resolver(object): class CPANResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False): self.session = session + self.user_local = user_local def __str__(self): return "cpan" @@ -62,6 +63,15 @@ class CPANResolver(Resolver): def install(self, requirements): from ..requirements import PerlModuleRequirement + env = { + "PERL_MM_USE_DEFAULT": "1", + "PERL_MM_OPT": "", + "PERL_MB_OPT": "", + } + + if not self.user_local: + user = "root" + missing = [] for requirement in requirements: if not isinstance(requirement, PerlModuleRequirement): @@ -70,16 +80,18 @@ class CPANResolver(Resolver): # TODO(jelmer): Specify -T to skip tests? self.session.check_call( ["cpan", "-i", requirement.module], - env={"PERL_MM_USE_DEFAULT": "1"}, + env=env, + user=user, ) if missing: raise UnsatisfiedRequirements(missing) class RResolver(Resolver): - def __init__(self, session, repos): + def __init__(self, session, repos, user_local=False): self.session = session self.repos = repos + self.user_local = user_local def __str__(self): return "cran" @@ -88,6 +100,7 @@ class RResolver(Resolver): return "%s(%r, %r)" % (type(self).__name__, self.session, self.repos) def _cmd(self, req): + # TODO(jelmer: Handle self.user_local return ["R", "-e", "install.packages('%s', repos=%r)" % (req.package, self.repos)] def explain(self, requirements): @@ -104,32 +117,38 @@ class RResolver(Resolver): def install(self, requirements): from ..requirements import RPackageRequirement + if self.user_local: + user = None + else: + user = "root" + missing = [] for requirement in requirements: if not isinstance(requirement, RPackageRequirement): missing.append(requirement) continue - self.session.check_call(self._cmd(requirement)) + self.session.check_call(self._cmd(requirement), user=user) if missing: raise UnsatisfiedRequirements(missing) class CRANResolver(RResolver): - def __init__(self, session): - super(CRANResolver, self).__init__(session, 'http://cran.r-project.org') + def __init__(self, session, user_local=False): + super(CRANResolver, self).__init__(session, 'http://cran.r-project.org', user_local=user_local) class BioconductorResolver(RResolver): def __init__(self, session): super(BioconductorResolver, self).__init__( - session, 'https://hedgehog.fhcrc.org/bioconductor') + session, 'https://hedgehog.fhcrc.org/bioconductor', user_local=user_local) class HackageResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False): self.session = session + self.user_local = user_local def __str__(self): return "hackage" @@ -137,17 +156,26 @@ class HackageResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def _cmd(self, reqs): + extra_args = [] + if self.user_local: + extra_args.append('--user') + return ["cabal", "install"] + extra_args + [req.package for req in reqs] + def install(self, requirements): from ..requirements import HaskellPackageRequirement + if self.user_local: + user = None + else: + user = "root" + missing = [] for requirement in requirements: if not isinstance(requirement, HaskellPackageRequirement): missing.append(requirement) continue - self.session.check_call( - ["cabal", "install", requirement.package] - ) + self.session.check_call(self._cmd([requirement]), user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -160,13 +188,13 @@ class HackageResolver(Resolver): continue haskellreqs.append(requirement) if haskellreqs: - yield (["cabal", "install"] + [req.package for req in haskellreqs], - haskellreqs) + yield (self._cmd(haskellreqs), haskellreqs) class PypiResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False): self.session = session + self.user_local = user_local def __str__(self): return "pypi" @@ -174,9 +202,20 @@ class PypiResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def _cmd(self, reqs): + extra_args = [] + if self.user_local: + extra_args.append('--user') + return ["pip", "install"] + extra_args + [req.package for req in reqs] + def install(self, requirements): from ..requirements import PythonPackageRequirement + if self.user_local: + user = None + else: + user = "root" + missing = [] for requirement in requirements: if not isinstance(requirement, PythonPackageRequirement): @@ -184,7 +223,7 @@ class PypiResolver(Resolver): continue try: self.session.check_call( - ["pip", "install", requirement.package]) + self._cmd([requirement]), user=user) except subprocess.CalledProcessError: missing.append(requirement) if missing: @@ -199,14 +238,14 @@ class PypiResolver(Resolver): continue pyreqs.append(requirement) if pyreqs: - yield (["pip", "install"] + [req.package for req in pyreqs], - pyreqs) + yield (self._cmd(pyreqs), pyreqs) class GoResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local): self.session = session + # TODO(jelmer): Handle user_local=False def __str__(self): return "go" @@ -245,8 +284,10 @@ NPM_COMMAND_PACKAGES = { class NpmResolver(Resolver): - def __init__(self, session): + def __init__(self, session, user_local=False): self.session = session + self.user_local = user_local + # TODO(jelmer): Handle user_local def __str__(self): return "npm" @@ -324,8 +365,8 @@ NATIVE_RESOLVER_CLS = [ ] -def native_resolvers(session): - return StackedResolver([kls(session) for kls in NATIVE_RESOLVER_CLS]) +def native_resolvers(session, user_local): + return StackedResolver([kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) def auto_resolver(session, explain=False): @@ -339,6 +380,10 @@ def auto_resolver(session, explain=False): # TODO(jelmer): Check VIRTUAL_ENV, and prioritize PypiResolver if # present? if isinstance(session, SchrootSession) or user == "root" or explain: + user_local = False + else: + user_local = True + if not user_local: resolvers.append(AptResolver.from_session(session)) - resolvers.extend([kls(session) for kls in NATIVE_RESOLVER_CLS]) + resolvers.extend([kls(session, user_local) for kls in NATIVE_RESOLVER_CLS]) return StackedResolver(resolvers) From ca95dbeb198ccd9162973289a2c3f7a0c5da0688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 14:14:49 +0000 Subject: [PATCH 114/252] Syntax error. --- ognibuild/resolver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 85b9052..7e03cf3 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -140,7 +140,7 @@ class CRANResolver(RResolver): class BioconductorResolver(RResolver): - def __init__(self, session): + def __init__(self, session, user_local=False): super(BioconductorResolver, self).__init__( session, 'https://hedgehog.fhcrc.org/bioconductor', user_local=user_local) From 4bfd284f2df3ff6c37dc77ccac8080fffdcc1d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 14:23:25 +0000 Subject: [PATCH 115/252] Also look for NAMESPACE when packaging R packages. --- ognibuild/buildsystem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index e351d0b..cd2e010 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -524,7 +524,8 @@ class R(BuildSystem): @classmethod def probe(cls, path): - if os.path.exists(os.path.join(path, 'DESCRIPTION')): + if (os.path.exists(os.path.join(path, 'DESCRIPTION')) and + os.path.exists(os.path.join(path, 'NAMESPACE'))): return cls(path) def _read_description(self): From 08e433b0f801ad7fff05f353b86acd01f3491b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 14:39:48 +0000 Subject: [PATCH 116/252] Add basic Octave support. --- ognibuild/buildsystem.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index cd2e010..20e703a 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -441,6 +441,38 @@ class SetupPy(BuildSystem): return cls(path) +class Octave(BuildSystem): + + name = "octave" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def exists(cls, path): + if not ( + os.path.exists(os.path.join(path, "DESCRIPTION")) and + os.path.exists(os.path.join(path, "COPYING"))): + return False + # Urgh, isn't there a better way to see if this is an octave package? + for entry in os.scandir(path): + if not entry.is_dir(): + continue + for subentry in os.scandir(entry.path): + if subentry.name.endswith('.m'): + return True + return False + + @classmethod + def probe(cls, path): + if cls.exists(path): + logging.debug("Found DESCRIPTION and COPYING, assuming octave package.") + return cls(path) + + class Gradle(BuildSystem): name = "gradle" @@ -1060,7 +1092,7 @@ class PerlBuildTiny(BuildSystem): BUILDSYSTEM_CLSES = [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistInkt, Gem, Make, PerlBuildTiny, Golang, R] + DistInkt, Gem, Make, PerlBuildTiny, Golang, R, Octave] def scan_buildsystems(path): From 24e417650aeefa77c717575a14b3a7757727796e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 14:40:14 +0000 Subject: [PATCH 117/252] Scan in root as well. --- ognibuild/buildsystem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 20e703a..8ffd75b 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -459,6 +459,8 @@ class Octave(BuildSystem): return False # Urgh, isn't there a better way to see if this is an octave package? for entry in os.scandir(path): + if entry.name.endswith('.m'): + return True if not entry.is_dir(): continue for subentry in os.scandir(entry.path): From 7f55169fc6ce95924827ae70ac9ed2815cba22e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 14:49:08 +0000 Subject: [PATCH 118/252] Add basic support for octave forge. --- ognibuild/buildsystem.py | 21 +++++++++++++--- ognibuild/requirements.py | 35 ++++++++++++++++++++++++++ ognibuild/resolver/__init__.py | 45 ++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 8ffd75b..df8320c 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -36,6 +36,7 @@ from .requirements import ( NodePackageRequirement, CargoCrateRequirement, RPackageRequirement, + OctavePackageRequirement, ) from .fix_build import run_with_build_fixers @@ -453,9 +454,7 @@ class Octave(BuildSystem): @classmethod def exists(cls, path): - if not ( - os.path.exists(os.path.join(path, "DESCRIPTION")) and - os.path.exists(os.path.join(path, "COPYING"))): + if not os.path.exists(os.path.join(path, "DESCRIPTION")): return False # Urgh, isn't there a better way to see if this is an octave package? for entry in os.scandir(path): @@ -471,9 +470,23 @@ class Octave(BuildSystem): @classmethod def probe(cls, path): if cls.exists(path): - logging.debug("Found DESCRIPTION and COPYING, assuming octave package.") + logging.debug("Found DESCRIPTION, assuming octave package.") return cls(path) + def _read_description(self): + path = os.path.join(self.path, 'DESCRIPTION') + from email.parser import BytesParser + with open(path, 'rb') as f: + return BytesParser().parse(f) + + def get_declared_dependencies(self, session, fixers=None): + def parse_list(t): + return [s.strip() for s in t.split(',') if s.strip()] + description = self._read_description() + if 'Depends' in description: + for s in parse_list(description['Depends']): + yield "build", OctavePackageRequirement.from_str(s) + class Gradle(BuildSystem): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 8b6a0b1..51fba07 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -278,6 +278,41 @@ class RPackageRequirement(Requirement): raise ValueError(text) +class OctavePackageRequirement(Requirement): + + package: str + minimum_version: Optional[str] + + def __init__(self, package: str, minimum_version: Optional[str] = None): + super(OctavePackageRequirement, self).__init__("octave-package") + self.package = package + self.minimum_version = minimum_version + + def __repr__(self): + return "%s(%r, minimum_version=%r)" % ( + type(self).__name__, + self.package, + self.minimum_version, + ) + + def __str__(self): + if self.minimum_version: + return "Octave package: %s (>= %s)" % (self.package, self.minimum_version) + else: + return "Octave package: %s" % (self.package,) + + @classmethod + def from_str(cls, text): + # TODO(jelmer): More complex parser + m = re.fullmatch(r'(.*) \(>= (.*)\)', text) + if m: + return cls(m.group(1), m.group(2)) + m = re.fullmatch(r'([^ ]+)', text) + if m: + return cls(m.group(1)) + raise ValueError(text) + + class LibraryRequirement(Requirement): library: str diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 7e03cf3..790dd79 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -132,6 +132,50 @@ class RResolver(Resolver): raise UnsatisfiedRequirements(missing) +class OctaveForgeResolver(Resolver): + def __init__(self, session, user_local=False): + self.session = session + self.user_local = user_local + + def __str__(self): + return "octave-forge" + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.session) + + def _cmd(self, req): + # TODO(jelmer: Handle self.user_local + return ["octave-cli", "--eval", "pkg install -forge %s" % req.package] + + def explain(self, requirements): + from ..requirements import OctavePackageRequirement + + rreqs = [] + for requirement in requirements: + if not isinstance(requirement, OctavePackageRequirement): + continue + rreqs.append(requirement) + if rreqs: + yield ([self._cmd(req) for req in rreqs]) + + def install(self, requirements): + from ..requirements import OctavePackageRequirement + + if self.user_local: + user = None + else: + user = "root" + + missing = [] + for requirement in requirements: + if not isinstance(requirement, OctavePackageRequirement): + missing.append(requirement) + continue + self.session.check_call(self._cmd(requirement), user=user) + if missing: + raise UnsatisfiedRequirements(missing) + + class CRANResolver(RResolver): def __init__(self, session, user_local=False): @@ -362,6 +406,7 @@ NATIVE_RESOLVER_CLS = [ HackageResolver, CRANResolver, BioconductorResolver, + OctaveForgeResolver, ] From 02177527c32240a951aece61095db6cd92ca1692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 15:46:09 +0000 Subject: [PATCH 119/252] Drop use of lintian-brush. --- ognibuild/debian/fix_build.py | 48 ++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 907a69e..4abf1c5 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -19,10 +19,11 @@ __all__ = [ "build_incrementally", ] +from datetime import datetime import logging import os import sys -from typing import List, Set, Optional, Type +from typing import List, Set, Optional, Type, Tuple from debian.deb822 import ( Deb822, @@ -32,6 +33,7 @@ from debian.deb822 import ( from breezy.commit import PointlessCommit from breezy.mutabletree import MutableTree from breezy.tree import Tree +from debmutate.changelog import ChangelogEditor from debmutate.control import ( ensure_relation, ControlEditor, @@ -49,12 +51,40 @@ from debmutate.reformatting import ( try: from breezy.workspace import reset_tree -except ImportError: - from lintian_brush import reset_tree +except ImportError: # breezy < 3.2 + def delete_items(deletables, dry_run=False): + """Delete files in the deletables iterable""" + import errno + import shutil + def onerror(function, path, excinfo): + """Show warning for errors seen by rmtree. + """ + # Handle only permission error while removing files. + # Other errors are re-raised. + if function is not os.remove or excinfo[1].errno != errno.EACCES: + raise + logging.warning('unable to remove %s' % path) + for path, subp in deletables: + if os.path.isdir(path): + shutil.rmtree(path, onerror=onerror) + else: + try: + os.unlink(path) + except OSError as e: + # We handle only permission error here + if e.errno != errno.EACCES: + raise e + logging.warning('unable to remove "%s": %s.', path, e.strerror) + + def reset_tree(local_tree, subpath=''): + from breezy.transform import revert + from breezy.clean_tree import iter_deletables + revert(local_tree, local_tree.branch.basis_tree(), + [subpath] if subpath not in ('.', '') else None) + deletables = list(iter_deletables( + local_tree, unknown=True, ignored=False, detritus=False)) + delete_items(deletables) -from lintian_brush.changelog import ( - add_changelog_entry, -) from debmutate._rules import ( dh_invoke_add_with, @@ -283,9 +313,9 @@ def commit_debian_changes( with tree.lock_write(): try: if update_changelog: - add_changelog_entry( - tree, os.path.join(subpath, "debian/changelog"), [summary] - ) + cl_path = tree.abspath(os.path.join(subpath, "debian/changelog")) + with ChangelogEditor(cl_path) as editor: + editor.add_entry([summary]) debcommit(tree, committer=committer, subpath=subpath) else: tree.commit( From dfd18e07aab52db918f71e5f6f79e6307c48a634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 15:53:46 +0000 Subject: [PATCH 120/252] Check for go.sum. --- ognibuild/buildsystem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index df8320c..2ceb9ae 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1009,6 +1009,8 @@ class Golang(BuildSystem): def probe(cls, path): if os.path.exists(os.path.join(path, 'go.mod')): return Golang(path) + if os.path.exists(os.path.join(path, 'go.sum')): + return Golang(path) for entry in os.scandir(path): if entry.name.endswith(".go"): return Golang(path) From 6b9dcb3e54f5dac5bf6b055fd99b2eb90b2b0d08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 15:54:30 +0000 Subject: [PATCH 121/252] Add go test path. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 2ceb9ae..54abbd1 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -994,7 +994,7 @@ class Golang(BuildSystem): return "%s()" % (type(self).__name__) def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["go", "test"], fixers) + run_with_build_fixers(session, ["go", "test", "./..."], fixers) def build(self, session, resolver, fixers): run_with_build_fixers(session, ["go", "build"], fixers) From 816c14b9d82be62251d41a754c90d0791d23796e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 22 Mar 2021 17:07:19 +0000 Subject: [PATCH 122/252] Set user if it is not set. --- README.md | 1 + ognibuild/resolver/__init__.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 61dae9e..2eb483d 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ issues (or lack of support for a particular ecosystem), please file a bug. - ninja, including ninja file generators: - meson - Node +- Octave - Perl - Module::Build::Tiny - PHP Pear diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 790dd79..dc2ee45 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -71,6 +71,8 @@ class CPANResolver(Resolver): if not self.user_local: user = "root" + else: + user = None missing = [] for requirement in requirements: From af3caadad222cfffc077d71b4739b94376bb0cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 02:19:59 +0000 Subject: [PATCH 123/252] Install ca-certificates if they are missing. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 7 +++++++ ognibuild/resolver/apt.py | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 5609f9c..8246d87 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -52,6 +52,7 @@ from buildlog_consultant.common import ( MissingMavenArtifacts, GnomeCommonMissing, MissingGnomeCommonDependency, + UnknownCertificateAuthority, ) from .fix_build import BuildFixer @@ -84,6 +85,7 @@ from .requirements import ( AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, + CertificateAuthorityRequirement, ) from .resolver import UnsatisfiedRequirements @@ -137,6 +139,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return JDKRequirement() elif isinstance(problem, MissingJRE): return JRERequirement() + elif isinstance(problem, UnknownCertificateAuthority): + return CertificateAuthorityRequirement(problem.url) elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": return BinaryRequirement("glib-gettextize") diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 51fba07..3b6fea9 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -416,6 +416,13 @@ class JRERequirement(Requirement): super(JRERequirement, self).__init__("jre") +class CertificateAuthorityRequirement(Requirement): + + def __init__(self, url): + super(CertificateAuthorityRequirement, self).__init__("ca-cert") + self.url = url + + class PerlFileRequirement(Requirement): filename: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index cdfa028..54e3186 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -58,6 +58,7 @@ from ..requirements import ( AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, + CertificateAuthorityRequirement, ) @@ -557,6 +558,10 @@ def resolve_cargo_crate_req(apt_mgr, req): return AptRequirement.simple(pkg_name) +def resolve_ca_req(apt_mgr, req): + return AptRequirement.simple('ca-certificates') + + APT_REQUIREMENT_RESOLVERS = [ (BinaryRequirement, resolve_binary_req), (PkgConfigRequirement, resolve_pkg_config_req), @@ -586,6 +591,7 @@ APT_REQUIREMENT_RESOLVERS = [ (AutoconfMacroRequirement, resolve_autoconf_macro_req), (PythonModuleRequirement, resolve_python_module_req), (PythonPackageRequirement, resolve_python_package_req), + (CertificateAuthorityRequirement, resolve_ca_req), (CargoCrateRequirement, resolve_cargo_crate_req), ] From 2ddc2804a03ba8ebcdb3c644f7cace6d23b1c895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 03:18:48 +0000 Subject: [PATCH 124/252] Detect problems when running resolver. --- ognibuild/fix_build.py | 66 ++++++++++++++++++---------------- ognibuild/resolver/__init__.py | 4 +-- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index f32e820..80460f9 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -63,41 +63,45 @@ class DependencyContext(object): raise NotImplementedError(self.add_dependency) -def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], env: Optional[Dict[str, str]] = None): +def run_detecting_problems(session: Session, args: List[str], **kwargs): + try: + retcode, contents = run_with_tee(session, args, **kwargs) + except FileNotFoundError: + error = MissingCommand(args[0]) + retcode = 1 + else: + if retcode == 0: + return + lines = ''.join(contents).splitlines(False) + match, error = find_build_failure_description(lines) + if error is None: + if match: + logging.warning("Build failed with unidentified error:") + logging.warning("%s", match.line.rstrip("\n")) + else: + logging.warning("Build failed and unable to find cause. Giving up.") + raise UnidentifiedError(retcode, args, lines, secondary=match) + raise DetailedFailure(retcode, args, error) + + +def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): fixed_errors = [] while True: try: - retcode, contents = run_with_tee(session, args, env=env) - except FileNotFoundError: - error = MissingCommand(args[0]) - retcode = 1 + run_detecting_problems(session, args, **kwargs) + except DetailedFailure as e: + logging.info("Identified error: %r", e.error) + if e.error in fixed_errors: + logging.warning( + "Failed to resolve error %r, it persisted. Giving up.", e.error + ) + raise DetailedFailure(e.retcode, args, e.error) + if not resolve_error(e.error, None, fixers=fixers): + logging.warning("Failed to find resolution for error %r. Giving up.", e.error) + raise DetailedFailure(e.retcode, args, e.error) + fixed_errors.append(e.error) else: - if retcode == 0: - return - lines = ''.join(contents).splitlines(False) - match, error = find_build_failure_description(lines) - if error is None: - if match: - logging.warning("Build failed with unidentified error:") - logging.warning("%s", match.line.rstrip("\n")) - else: - logging.warning("Build failed and unable to find cause. Giving up.") - raise UnidentifiedError(retcode, args, lines, secondary=match) - - logging.info("Identified error: %r", error) - if error in fixed_errors: - logging.warning( - "Failed to resolve error %r, it persisted. Giving up.", error - ) - raise DetailedFailure(retcode, args, error) - if not resolve_error( - error, - None, - fixers=fixers, - ): - logging.warning("Failed to find resolution for error %r. Giving up.", error) - raise DetailedFailure(retcode, args, error) - fixed_errors.append(error) + return def resolve_error(error, context, fixers): diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index dc2ee45..4ad0d84 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -17,7 +17,7 @@ import subprocess - +from ..fix_build import run_detecting_problems class UnsatisfiedRequirements(Exception): def __init__(self, reqs): @@ -80,7 +80,7 @@ class CPANResolver(Resolver): missing.append(requirement) continue # TODO(jelmer): Specify -T to skip tests? - self.session.check_call( + run_detecting_problems(self.session, ["cpan", "-i", requirement.module], env=env, user=user, From 3a84dfcdfcd8eb5650cdc82c5a86a38d499aebe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 12:32:40 +0000 Subject: [PATCH 125/252] Require that dist files have a supported extension. --- ognibuild/dist.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index a9bc20f..87c21c0 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -111,12 +111,17 @@ class DistCatcher(object): logging.info("No tarballs found in dist directory.") parent_directory = os.path.dirname(self.export_directory) - diff = set(os.listdir(parent_directory)) - set([self.export_directory]) + diff = (set(os.listdir(parent_directory)) - + set([os.path.basename(self.export_directory)])) if len(diff) == 1: fn = diff.pop() - logging.info("Found tarball %s in parent directory.", fn) - self.files.append(os.path.join(parent_directory, fn)) - return fn + if is_dist_file(fn): + logging.info("Found tarball %s in parent directory.", fn) + self.files.append(os.path.join(parent_directory, fn)) + return fn + logging.warning( + "Found file %s in parent directory, " + "but not in supported dist format", fn) if "dist" in new_files: for entry in os.scandir(os.path.join(self.export_directory, "dist")): From 63d11917a7f520b3519db3207b3ed2b7d376fcaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 13:36:34 +0000 Subject: [PATCH 126/252] Support system-wide go code. --- ognibuild/resolver/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 4ad0d84..51f7204 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -34,8 +34,8 @@ class Resolver(object): def explain(self, requirements): raise NotImplementedError(self.explain) - def met(self, requirement): - raise NotImplementedError(self.met) + def env(self): + return {} class CPANResolver(Resolver): @@ -291,7 +291,7 @@ class GoResolver(Resolver): def __init__(self, session, user_local): self.session = session - # TODO(jelmer): Handle user_local=False + self.user_local = user_local def __str__(self): return "go" @@ -302,12 +302,18 @@ class GoResolver(Resolver): def install(self, requirements): from ..requirements import GoPackageRequirement + if self.user_local: + env = {} + else: + # TODO(jelmer): Isn't this Debian-specific? + env = {'GOPATH': '/usr/share/gocode'} + missing = [] for requirement in requirements: if not isinstance(requirement, GoPackageRequirement): missing.append(requirement) continue - self.session.check_call(["go", "get", requirement.package]) + self.session.check_call(["go", "get", requirement.package], env=env) if missing: raise UnsatisfiedRequirements(missing) @@ -386,6 +392,13 @@ class StackedResolver(Resolver): def __str__(self): return "[" + ", ".join(map(str, self.subs)) + "]" + def env(self): + ret = {} + # Reversed so earlier resolvers override later ones + for sub in reversed(self.subs): + ret.update(sub.env()) + return ret + def explain(self, requirements): for sub in self.subs: yield from sub.explain(requirements) From 96e3e520593365258a3cded304114cc8fc829282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 13:40:12 +0000 Subject: [PATCH 127/252] Add UnidentifiedError.__repr__. --- ognibuild/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index a74a55b..da463df 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -39,6 +39,10 @@ class UnidentifiedError(Exception): self.lines = lines self.secondary = secondary + def __repr__(self): + return "<%s(%r, %r, ..., secondary=%r)>" % ( + type(self).__name__, self.retcode, self.argv, self.secondary) + def shebang_binary(p): if not (os.stat(p).st_mode & stat.S_IEXEC): From 4230c7a6b80e27bab53bba6fc31dcaf31c20f716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 13:50:53 +0000 Subject: [PATCH 128/252] Fix pg_buildext running. --- ognibuild/debian/fix_build.py | 7 ++++++- ognibuild/session/__init__.py | 3 +++ ognibuild/session/plain.py | 3 +++ ognibuild/session/schroot.py | 6 +++--- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 4abf1c5..6dc7eb0 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -22,6 +22,7 @@ __all__ = [ from datetime import datetime import logging import os +import shutil import sys from typing import List, Set, Optional, Type, Tuple @@ -543,7 +544,9 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer): def _fix(self, error, context): logging.info("Running 'pg_buildext updatecontrol'") self.session.check_call(["pg_buildext", "updatecontrol"]) - # TODO(jelmer): Copy control file back + shutil.copy( + self.session.external_path('debian/control'), + context.tree.abspath(os.path.join(context.subpath, 'debian/control'))) return commit_debian_changes( context.tree, context.subpath, @@ -744,6 +747,8 @@ def main(argv=None): args = parser.parse_args() from breezy.workingtree import WorkingTree + import breezy.git + import breezy.bzr from .apt import AptManager from ..session.plain import PlainSession from ..session.schroot import SchrootSession diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 0908903..f9f0225 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -97,6 +97,9 @@ class Session(object): def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]: raise NotImplementedError(self.setup_from_directory) + def external_path(self, path: str) -> str: + raise NotImplementedError + class SessionSetupFailure(Exception): """Session failed to be set up.""" diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index bec2ae8..a27631e 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -92,6 +92,9 @@ class PlainSession(Session): def chdir(self, path): os.chdir(path) + def external_path(self, path): + return os.path.abspath(path) + def setup_from_vcs( self, tree, include_controldir=None, subdir="package"): from ..vcs import dupe_vcs_tree, export_vcs_tree diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 4026669..2faeeb6 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -177,17 +177,17 @@ class SchrootSession(Session): self.check_call(["mkdir", "-p", home], cwd="/", user="root") self.check_call(["chown", user, home], cwd="/", user="root") - def _fullpath(self, path: str) -> str: + def external_path(self, path: str) -> str: if self._cwd is None: raise ValueError("no cwd set") return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/")) def exists(self, path: str) -> bool: - fullpath = self._fullpath(path) + fullpath = self.external_path(path) return os.path.exists(fullpath) def scandir(self, path: str): - fullpath = self._fullpath(path) + fullpath = self.external_path(path) return os.scandir(fullpath) def setup_from_vcs( From 5b8e6718cbd15b7ca5901c72a3ff35d29aae7ff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 14:20:41 +0000 Subject: [PATCH 129/252] Move most of context into fixers. --- ognibuild/buildlog.py | 4 +- ognibuild/debian/fix_build.py | 309 +++++++++-------------- ognibuild/fix_build.py | 34 +-- ognibuild/tests/test_debian_fix_build.py | 17 +- 4 files changed, 133 insertions(+), 231 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 8246d87..b7b5f3e 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -193,7 +193,7 @@ class InstallFixer(BuildFixer): req = problem_to_upstream_requirement(error) return req is not None - def fix(self, error, context): + def fix(self, error, phase): reqs = problem_to_upstream_requirement(error) if reqs is None: return False @@ -228,7 +228,7 @@ class ExplainInstallFixer(BuildFixer): req = problem_to_upstream_requirement(error) return req is not None - def fix(self, error, context): + def fix(self, error, phase): reqs = problem_to_upstream_requirement(error) if reqs is None: return False diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 6dc7eb0..ad71645 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -111,7 +111,7 @@ from buildlog_consultant.sbuild import ( ) from ..buildlog import problem_to_upstream_requirement -from ..fix_build import BuildFixer, resolve_error, DependencyContext +from ..fix_build import BuildFixer, resolve_error from ..resolver.apt import ( AptRequirement, get_package_for_python_module, @@ -129,10 +129,39 @@ class CircularDependency(Exception): self.package = package +class DebianPackagingContext(object): + + def __init__(self, tree, subpath, committer, update_changelog): + self.tree = tree + self.subpath = subpath + self.committer = committer + self.update_changelog = update_changelog + + def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool: + if update_changelog is None: + update_changelog = self.update_changelog + with self.tree.lock_write(): + try: + if update_changelog: + cl_path = self.tree.abspath(os.path.join(self.subpath, "debian/changelog")) + with ChangelogEditor(cl_path) as editor: + editor.add_entry([summary]) + debcommit(self.tree, committer=self.committer, subpath=self.subpath) + else: + self.tree.commit( + message=summary, committer=self.committer, specific_files=[self.subpath] + ) + except PointlessCommit: + return False + else: + return True + + class PackageDependencyFixer(BuildFixer): - def __init__(self, apt_resolver): + def __init__(self, context, apt_resolver): self.apt_resolver = apt_resolver + self.context = context def __repr__(self): return "%s(%r)" % (type(self).__name__, self.apt_resolver) @@ -144,7 +173,7 @@ class PackageDependencyFixer(BuildFixer): req = problem_to_upstream_requirement(error) return req is not None - def fix(self, error, context): + def fix(self, error, phase): reqs = problem_to_upstream_requirement(error) if reqs is None: return False @@ -154,82 +183,33 @@ class PackageDependencyFixer(BuildFixer): changed = False for req in reqs: - package = self.apt_resolver.resolve(req) - if package is None: - return False - if context.phase[0] == "autopkgtest": - return add_test_dependency( - context.tree, - context.phase[1], - package, - committer=context.committer, - subpath=context.subpath, - update_changelog=context.update_changelog, - ) - elif context.phase[0] == "build": - return add_build_dependency( - context.tree, - package, - committer=context.committer, - subpath=context.subpath, - update_changelog=context.update_changelog, - ) - else: - logging.warning('Unknown phase %r', context.phase) + apt_req = self.apt_resolver.resolve(req) + if apt_req is None: return False + if add_dependency(self.context, phase, apt_req): + changed = True return changed -class BuildDependencyContext(DependencyContext): - def __init__( - self, phase, tree, apt, subpath="", committer=None, update_changelog=True - ): - self.phase = phase - super(BuildDependencyContext, self).__init__( - tree, apt, subpath, committer, update_changelog - ) - - def add_dependency(self, requirement: AptRequirement): - return add_build_dependency( - self.tree, - requirement, - committer=self.committer, - subpath=self.subpath, - update_changelog=self.update_changelog, - ) - - -class AutopkgtestDependencyContext(DependencyContext): - def __init__( - self, phase, tree, apt, subpath="", committer=None, update_changelog=True - ): - self.phase = phase - super(AutopkgtestDependencyContext, self).__init__( - tree, apt, subpath, committer, update_changelog - ) - - def add_dependency(self, requirement): +def add_dependency(context, phase, requirement: AptRequirement): + if phase[0] == "autopkgtest": return add_test_dependency( - self.tree, - self.phase[1], + context, + phase[1], requirement, - committer=self.committer, - subpath=self.subpath, - update_changelog=self.update_changelog, ) + elif phase[0] == "build": + return add_build_dependency(context, requirement) + else: + logging.warning('Unknown phase %r', phase) + return False -def add_build_dependency( - tree: Tree, - requirement: AptRequirement, - committer: Optional[str] = None, - subpath: str = "", - update_changelog: bool = True, -): +def add_build_dependency(context, requirement: AptRequirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - control_path = os.path.join(tree.abspath(subpath), "debian/control") + control_path = os.path.join(context.tree.abspath(context.subpath), "debian/control") try: with ControlEditor(path=control_path) as updater: for binary in updater.binaries: @@ -250,27 +230,14 @@ def add_build_dependency( return False logging.info("Adding build dependency: %s", desc) - return commit_debian_changes( - tree, - subpath, - "Add missing build dependency on %s." % desc, - committer=committer, - update_changelog=update_changelog, - ) + return context.commit("Add missing build dependency on %s." % desc) -def add_test_dependency( - tree, - testname, - requirement, - committer=None, - subpath="", - update_changelog=True, -): +def add_test_dependency(context, testname, requirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - tests_control_path = os.path.join(tree.abspath(subpath), "debian/tests/control") + tests_control_path = os.path.join(context.tree.abspath(context.subpath), "debian/tests/control") try: with Deb822Editor(path=tests_control_path) as updater: @@ -296,38 +263,11 @@ def add_test_dependency( desc = requirement.pkg_relation_str() logging.info("Adding dependency to test %s: %s", testname, desc) - return commit_debian_changes( - tree, - subpath, + return context.commit( "Add missing dependency for test %s on %s." % (testname, desc), - update_changelog=update_changelog, ) -def commit_debian_changes( - tree: MutableTree, - subpath: str, - summary: str, - committer: Optional[str] = None, - update_changelog: bool = True, -) -> bool: - with tree.lock_write(): - try: - if update_changelog: - cl_path = tree.abspath(os.path.join(subpath, "debian/changelog")) - with ChangelogEditor(cl_path) as editor: - editor.add_entry([summary]) - debcommit(tree, committer=committer, subpath=subpath) - else: - tree.commit( - message=summary, committer=committer, specific_files=[subpath] - ) - except PointlessCommit: - return False - else: - return True - - def targeted_python_versions(tree: Tree, subpath: str) -> Set[str]: with tree.get_file(os.path.join(subpath, "debian/control")) as f: control = Deb822(f) @@ -345,34 +285,34 @@ def targeted_python_versions(tree: Tree, subpath: str) -> Set[str]: return targeted -def fix_missing_python_distribution(error, context): # noqa: C901 +def fix_missing_python_distribution(error, phase, apt, context): # noqa: C901 targeted = targeted_python_versions(context.tree, context.subpath) default = not targeted - pypy_pkg = context.apt.get_package_for_paths( + pypy_pkg = apt.get_package_for_paths( ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True ) if pypy_pkg is None: pypy_pkg = "pypy-%s" % error.distribution - if not context.apt.package_exists(pypy_pkg): + if not apt.package_exists(pypy_pkg): pypy_pkg = None - py2_pkg = context.apt.get_package_for_paths( + py2_pkg = apt.get_package_for_paths( ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution], regex=True, ) if py2_pkg is None: py2_pkg = "python-%s" % error.distribution - if not context.apt.package_exists(py2_pkg): + if not apt.package_exists(py2_pkg): py2_pkg = None - py3_pkg = context.apt.get_package_for_paths( + py3_pkg = apt.get_package_for_paths( ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution], regex=True, ) if py3_pkg is None: py3_pkg = "python3-%s" % error.distribution - if not context.apt.package_exists(py3_pkg): + if not apt.package_exists(py3_pkg): py3_pkg = None extra_build_deps = [] @@ -405,16 +345,13 @@ def fix_missing_python_distribution(error, context): # noqa: C901 for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency(dep_pkg): + if not add_dependency(context, phase, dep_pkg): return False return True -def fix_missing_python_module(error, context): - if getattr(context, "tree", None) is not None: - targeted = targeted_python_versions(context.tree, context.subpath) - else: - targeted = set() +def fix_missing_python_module(error, phase, apt, context): + targeted = targeted_python_versions(context.tree, context.subpath) default = not targeted if error.minimum_version: @@ -422,9 +359,9 @@ def fix_missing_python_module(error, context): else: specs = [] - pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", specs) - py2_pkg = get_package_for_python_module(context.apt, error.module, "cpython2", specs) - py3_pkg = get_package_for_python_module(context.apt, error.module, "cpython3", specs) + pypy_pkg = get_package_for_python_module(apt, error.module, "pypy", specs) + py2_pkg = get_package_for_python_module(apt, error.module, "cpython2", specs) + py3_pkg = get_package_for_python_module(apt, error.module, "cpython3", specs) extra_build_deps = [] if error.python_version == 2: @@ -456,16 +393,16 @@ def fix_missing_python_module(error, context): for dep_pkg in extra_build_deps: assert dep_pkg is not None - if not context.add_dependency(dep_pkg): + if not add_dependency(context, phase, dep_pkg): return False return True -def retry_apt_failure(error, context): +def retry_apt_failure(error, phase, apt, context): return True -def enable_dh_autoreconf(context): +def enable_dh_autoreconf(context, phase): # Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by # default. debhelper_compat_version = get_debhelper_compat_level(context.tree.abspath(".")) @@ -479,28 +416,28 @@ def enable_dh_autoreconf(context): return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return context.add_dependency(AptRequirement.simple("dh-autoreconf")) + return add_dependency(context, phase, AptRequirement.simple("dh-autoreconf")) return False -def fix_missing_configure(error, context): +def fix_missing_configure(error, phase, context): if not context.tree.has_filename("configure.ac") and not context.tree.has_filename( "configure.in" ): return False - return enable_dh_autoreconf(context) + return enable_dh_autoreconf(context, phase) -def fix_missing_automake_input(error, context): +def fix_missing_automake_input(error, phase, context): # TODO(jelmer): If it's ./NEWS, ./AUTHORS or ./README that's missing, then # try to set 'export AUTOMAKE = automake --foreign' in debian/rules. # https://salsa.debian.org/jelmer/debian-janitor/issues/88 - return enable_dh_autoreconf(context) + return enable_dh_autoreconf(context, phase) -def fix_missing_config_status_input(error, context): +def fix_missing_config_status_input(error, phase, context): autogen_path = "autogen.sh" rules_path = "debian/rules" if context.subpath not in (".", ""): @@ -519,21 +456,13 @@ def fix_missing_config_status_input(error, context): if not update_rules(makefile_cb=add_autogen, path=rules_path): return False - if context.update_changelog: - commit_debian_changes( - context.tree, - context.subpath, - "Run autogen.sh during build.", - committer=context.committer, - update_changelog=context.update_changelog, - ) - - return True + return context.commit("Run autogen.sh during build.") class PgBuildExtOutOfDateControlFixer(BuildFixer): - def __init__(self, session): + def __init__(self, packaging_context, session): self.session = session + self.context = packaging_context def can_fix(self, problem): return isinstance(problem, NeedPgBuildExtUpdateControl) @@ -547,16 +476,11 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer): shutil.copy( self.session.external_path('debian/control'), context.tree.abspath(os.path.join(context.subpath, 'debian/control'))) - return commit_debian_changes( - context.tree, - context.subpath, - "Run 'pgbuildext updatecontrol'.", - committer=context.committer, - update_changelog=False, - ) + return self.context.commit( + "Run 'pgbuildext updatecontrol'.", update_changelog=False) -def fix_missing_makefile_pl(error, context): +def fix_missing_makefile_pl(error, phase, context): if ( error.filename == "Makefile.PL" and not context.tree.has_filename("Makefile.PL") @@ -568,7 +492,8 @@ def fix_missing_makefile_pl(error, context): class SimpleBuildFixer(BuildFixer): - def __init__(self, problem_cls: Type[Problem], fn): + def __init__(self, packaging_context, problem_cls: Type[Problem], fn): + self.context = packaging_context self._problem_cls = problem_cls self._fn = fn @@ -579,28 +504,46 @@ class SimpleBuildFixer(BuildFixer): def can_fix(self, problem: Problem): return isinstance(problem, self._problem_cls) - def _fix(self, problem: Problem, context): - return self._fn(problem, context) + def _fix(self, problem: Problem, phase): + return self._fn(problem, phase, self.context) -def versioned_package_fixers(session): +class DependencyBuildFixer(BuildFixer): + def __init__(self, packaging_context, apt_resolver, problem_cls: Type[Problem], fn): + self.context = packaging_context + self.apt_resolver = apt_resolver + self._problem_cls = problem_cls + self._fn = fn + + def __repr__(self): + return "%s(%s, %r, %s)" % ( + type(self).__name__, self._problem_cls.__name__, self._fn.__name__) + + def can_fix(self, problem: Problem): + return isinstance(problem, self._problem_cls) + + def _fix(self, problem: Problem, phase): + return self._fn(problem, phase, self.apt_resolver, self.context) + + +def versioned_package_fixers(session, packaging_context): return [ - PgBuildExtOutOfDateControlFixer(session), - SimpleBuildFixer(MissingConfigure, fix_missing_configure), - SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input), - SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input), - SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl), + PgBuildExtOutOfDateControlFixer(packaging_context, session), + SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure), + SimpleBuildFixer(packaging_context, MissingAutomakeInput, fix_missing_automake_input), + SimpleBuildFixer(packaging_context, MissingConfigStatusInput, fix_missing_config_status_input), + SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl), ] -def apt_fixers(apt) -> List[BuildFixer]: +def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver resolver = AptResolver(apt) return [ - SimpleBuildFixer(MissingPythonModule, fix_missing_python_module), - SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution), - SimpleBuildFixer(AptFetchFailure, retry_apt_failure), - PackageDependencyFixer(resolver), + DependencyBuildFixer(packaging_context, apt, MissingPythonModule, fix_missing_python_module), + DependencyBuildFixer(packaging_context, apt, MissingPythonDistribution, fix_missing_python_distribution), + DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure), + PackageDependencyFixer(packaging_context, resolver), ] @@ -619,7 +562,10 @@ def build_incrementally( update_changelog=True, ): fixed_errors = [] - fixers = versioned_package_fixers(apt.session) + apt_fixers(apt) + packaging_context = DebianPackagingContext( + local_tree, subpath, committer, update_changelog) + fixers = (versioned_package_fixers(apt.session, packaging_context) + + apt_fixers(apt, packaging_context)) logging.info("Using fixers: %r", fixers) while True: try: @@ -647,29 +593,8 @@ def build_incrementally( logging.warning("Last fix did not address the issue. Giving up.") raise reset_tree(local_tree, subpath=subpath) - if e.phase[0] == "build": - context = BuildDependencyContext( - e.phase, - local_tree, - apt, - subpath=subpath, - committer=committer, - update_changelog=update_changelog, - ) - elif e.phase[0] == "autopkgtest": - context = AutopkgtestDependencyContext( - e.phase, - local_tree, - apt, - subpath=subpath, - committer=committer, - update_changelog=update_changelog, - ) - else: - logging.warning("unable to install for context %r", e.phase) - raise try: - if not resolve_error(e.error, context, fixers): + if not resolve_error(e.error, e.phase, fixers): logging.warning("Failed to resolve error %r. Giving up.", e.error) raise except GeneratedFile: diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 80460f9..6bfc593 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -16,8 +16,9 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import logging -from typing import List, Optional, Dict +from typing import List, Optional, Dict, Tuple +from buildlog_consultant import Problem from buildlog_consultant.common import ( find_build_failure_description, MissingCommand, @@ -32,35 +33,16 @@ from .session import Session, run_with_tee class BuildFixer(object): """Build fixer.""" - def can_fix(self, problem): + def can_fix(self, problem: Problem): raise NotImplementedError(self.can_fix) - def _fix(self, problem, context): + def _fix(self, problem: Problem, phase: Tuple[str, ...]): raise NotImplementedError(self._fix) - def fix(self, problem, context): + def fix(self, problem: Problem, phase: Tuple[str, ...]): if not self.can_fix(problem): return None - return self._fix(problem, context) - - -class DependencyContext(object): - def __init__( - self, - tree: MutableTree, - apt: AptManager, - subpath: str = "", - committer: Optional[str] = None, - update_changelog: bool = True, - ): - self.tree = tree - self.apt = apt - self.subpath = subpath - self.committer = committer - self.update_changelog = update_changelog - - def add_dependency(self, package) -> bool: - raise NotImplementedError(self.add_dependency) + return self._fix(problem, phase) def run_detecting_problems(session: Session, args: List[str], **kwargs): @@ -104,7 +86,7 @@ def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildF return -def resolve_error(error, context, fixers): +def resolve_error(error, phase, fixers): relevant_fixers = [] for fixer in fixers: if fixer.can_fix(error): @@ -114,7 +96,7 @@ def resolve_error(error, context, fixers): return False for fixer in relevant_fixers: logging.info("Attempting to use fixer %s to address %r", fixer, error) - made_changes = fixer.fix(error, context) + made_changes = fixer.fix(error, phase) if made_changes: return True return False diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index a06884a..a947003 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -35,7 +35,7 @@ from ..debian.fix_build import ( resolve_error, versioned_package_fixers, apt_fixers, - BuildDependencyContext, + DebianPackagingContext, ) from breezy.tests import TestCaseWithTransport @@ -97,16 +97,11 @@ blah (0.1) UNRELEASED; urgency=medium session = PlainSession() apt = AptManager(session) apt._searchers = [DummyAptSearcher(self._apt_files)] - context = BuildDependencyContext( - ("build", ), - self.tree, - apt, - subpath="", - committer="ognibuild ", - update_changelog=True, - ) - fixers = versioned_package_fixers(session) + apt_fixers(apt) - return resolve_error(error, context, fixers) + context = DebianPackagingContext( + self.tree, subpath="", committer="ognibuild ", + update_changelog=True) + fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context) + return resolve_error(error, ("build", ), fixers) def get_build_deps(self): with open(self.tree.abspath("debian/control"), "r") as f: From 94e0b4f99de2d8b2e72ecdb878585b564dd9b8ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 14:30:51 +0000 Subject: [PATCH 130/252] Python refactoring. --- ognibuild/resolver/apt.py | 49 ++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 54e3186..0d04cb1 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -119,30 +119,21 @@ def python_spec_to_apt_rels(pkg_name, specs): return rels -def get_package_for_python_package(apt_mgr, package, python_version, specs=None): +def get_package_for_python_package(apt_mgr, package, python_version: Optional[str], specs=None): + pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) + cpython2_regex = "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) + cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) if python_version == "pypy": - pkg_name = apt_mgr.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_"))], - regex=True, - ) + paths = [pypy_regex] elif python_version == "cpython2": - pkg_name = apt_mgr.get_package_for_paths( - [ - "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" - % re.escape(package.replace("-", "_")) - ], - regex=True, - ) + paths = [cpython2_regex] elif python_version == "cpython3": - pkg_name = apt_mgr.get_package_for_paths( - [ - "/usr/lib/python3/dist-packages/%s-.*.egg-info" - % re.escape(package.replace("-", "_")) - ], - regex=True, - ) + paths = [cpython3_regex] + elif python_version is None: + paths = [cpython3_regex, cpython2_regex, pypy_regex] else: - raise NotImplementedError + raise NotImplementedError('unsupported python version %d' % python_version) + pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) if pkg_name is None: return None rels = python_spec_to_apt_rels(pkg_name, specs) @@ -150,8 +141,7 @@ def get_package_for_python_package(apt_mgr, package, python_version, specs=None) def get_package_for_python_module(apt_mgr, module, python_version, specs): - if python_version == "cpython3": - paths = [ + cpython3_regexes = [ posixpath.join( "/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")), @@ -171,8 +161,7 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")), "__init__.py" ), ] - elif python_version == "cpython2": - paths = [ + cpython2_regexes = [ posixpath.join( "/usr/lib/python2\\.[0-9]/dist-packages", re.escape(module.replace(".", "/")), @@ -187,8 +176,7 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): re.escape(module.replace(".", "/")) + ".so", ), ] - elif python_version == "pypy": - paths = [ + pypy_regexes = [ posixpath.join( "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")), "__init__.py" ), @@ -200,6 +188,14 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", ), ] + if python_version == "cpython3": + paths = cpython3_regexes + elif python_version == "cpython2": + paths = cpython2_regexes + elif python_version == "pypy": + paths = pypy_regexes + elif python_version is None: + paths = cpython3_regexes + cpython2_regexes + pypy_regexes else: raise AssertionError("unknown python version %r" % python_version) pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) @@ -395,7 +391,6 @@ def resolve_sprockets_file_req(apt_mgr, req): def resolve_java_class_req(apt_mgr, req): # Unfortunately this only finds classes in jars installed on the host # system :( - # TODO(jelmer): Call in session output = apt_mgr.session.check_output( ["java-propose-classpath", "-c" + req.classname] ) From 0b6cc8d8cc8f65298f4631467d7a05733531990a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 15:29:31 +0000 Subject: [PATCH 131/252] Support passing in tie breaking functions to apt resolver. --- ognibuild/debian/apt.py | 6 +- ognibuild/debian/file_search.py | 41 +---- ognibuild/debian/fix_build.py | 173 ++++++------------ ognibuild/resolver/apt.py | 220 ++++++++++------------- ognibuild/tests/test_debian_fix_build.py | 4 +- 5 files changed, 154 insertions(+), 290 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 4b0bb70..17875ee 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -26,7 +26,7 @@ from buildlog_consultant.apt import ( from .. import DetailedFailure, UnidentifiedError from ..session import Session, run_with_tee, get_user -from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_package_for_paths +from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_packages_for_paths def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None: @@ -81,10 +81,10 @@ class AptManager(object): self._apt_cache = apt.Cache(rootdir=self.session.location) return package in self._apt_cache - def get_package_for_paths(self, paths, regex=False): + def get_packages_for_paths(self, paths, regex=False): logging.debug("Searching for packages containing %r", paths) # TODO(jelmer): Make sure we use whatever is configured in self.session - return get_package_for_paths(paths, self.searchers(), regex=regex) + return get_packages_for_paths(paths, self.searchers(), regex=regex) def missing(self, packages): root = getattr(self.session, "location", "/") diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index bd2fcb3..1dfbb54 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -264,40 +264,14 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( ) -def get_package_for_paths( +def get_packages_for_paths( paths: List[str], searchers: List[FileSearcher], regex: bool = False -) -> Optional[str]: - candidates: Set[str] = set() +) -> List[str]: + candidates: List[str] = list() for path in paths: for searcher in searchers: - candidates.update(searcher.search_files(path, regex=regex)) - if candidates: - break - if len(candidates) == 0: - logging.debug("No packages found that contain %r", paths) - return None - if len(candidates) > 1: - logging.warning( - "More than 1 packages found that contain %r: %r", path, candidates - ) - # TODO(jelmer): Pick package based on what appears most commonly in - # build-depends{-indep,-arch} - try: - from .udd import UDD - except ModuleNotFoundError: - logging.warning('Unable to import UDD, not ranking by popcon') - return sorted(candidates, key=len)[0] - udd = UDD() - udd.connect() - winner = udd.get_most_popular(candidates) - if winner is None: - logging.warning( - 'No relevant popcon information found, not ranking by popcon') - return sorted(candidates, key=len)[0] - logging.info('Picked winner using popcon') - return winner - else: - return candidates.pop() + candidates.extend(searcher.search_files(path, regex=regex)) + return candidates def main(argv): @@ -317,8 +291,9 @@ def main(argv): main_searcher.load_local() searchers = [main_searcher, GENERATED_FILE_SEARCHER] - package = get_package_for_paths(args.path, searchers=searchers, regex=args.regex) - print(package) + packages = get_packages_for_paths(args.path, searchers=searchers, regex=args.regex) + for package in packages: + print(package) if __name__ == '__main__': diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index ad71645..19715ee 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -20,6 +20,7 @@ __all__ = [ ] from datetime import datetime +from functools import partial import logging import os import shutil @@ -131,11 +132,12 @@ class CircularDependency(Exception): class DebianPackagingContext(object): - def __init__(self, tree, subpath, committer, update_changelog): + def __init__(self, tree, subpath, committer, update_changelog, commit_reporter=None): self.tree = tree self.subpath = subpath self.committer = committer self.update_changelog = update_changelog + self.commit_reporter = commit_reporter def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool: if update_changelog is None: @@ -149,7 +151,8 @@ class DebianPackagingContext(object): debcommit(self.tree, committer=self.committer, subpath=self.subpath) else: self.tree.commit( - message=summary, committer=self.committer, specific_files=[self.subpath] + message=summary, committer=self.committer, specific_files=[self.subpath], + reporter=self.commit_reporter ) except PointlessCommit: return False @@ -268,134 +271,36 @@ def add_test_dependency(context, testname, requirement): ) -def targeted_python_versions(tree: Tree, subpath: str) -> Set[str]: +def targeted_python_versions(tree: Tree, subpath: str) -> List[str]: with tree.get_file(os.path.join(subpath, "debian/control")) as f: control = Deb822(f) build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) all_build_deps: Set[str] = set() for or_deps in build_depends: all_build_deps.update(or_dep["name"] for or_dep in or_deps) - targeted = set() - if any(x.startswith("pypy") for x in all_build_deps): - targeted.add("pypy") - if any(x.startswith("python-") for x in all_build_deps): - targeted.add("cpython2") + targeted = [] if any(x.startswith("python3-") for x in all_build_deps): - targeted.add("cpython3") + targeted.append("python3") + if any(x.startswith("pypy") for x in all_build_deps): + targeted.append("pypy") + if any(x.startswith("python-") for x in all_build_deps): + targeted.append("python") return targeted -def fix_missing_python_distribution(error, phase, apt, context): # noqa: C901 - targeted = targeted_python_versions(context.tree, context.subpath) - default = not targeted - - pypy_pkg = apt.get_package_for_paths( - ["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True - ) - if pypy_pkg is None: - pypy_pkg = "pypy-%s" % error.distribution - if not apt.package_exists(pypy_pkg): - pypy_pkg = None - - py2_pkg = apt.get_package_for_paths( - ["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution], - regex=True, - ) - if py2_pkg is None: - py2_pkg = "python-%s" % error.distribution - if not apt.package_exists(py2_pkg): - py2_pkg = None - - py3_pkg = apt.get_package_for_paths( - ["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution], - regex=True, - ) - if py3_pkg is None: - py3_pkg = "python3-%s" % error.distribution - if not apt.package_exists(py3_pkg): - py3_pkg = None - - extra_build_deps = [] - if error.python_version == 2: - if "pypy" in targeted: - if not pypy_pkg: - logging.warning("no pypy package found for %s", error.module) - else: - extra_build_deps.append(pypy_pkg) - if "cpython2" in targeted or default: - if not py2_pkg: - logging.warning("no python 2 package found for %s", error.module) - return False - extra_build_deps.append(py2_pkg) - elif error.python_version == 3: - if not py3_pkg: - logging.warning("no python 3 package found for %s", error.module) - return False - extra_build_deps.append(py3_pkg) - else: - if py3_pkg and ("cpython3" in targeted or default): - extra_build_deps.append(py3_pkg) - if py2_pkg and ("cpython2" in targeted or default): - extra_build_deps.append(py2_pkg) - if pypy_pkg and "pypy" in targeted: - extra_build_deps.append(pypy_pkg) - - if not extra_build_deps: - return False - - for dep_pkg in extra_build_deps: - assert dep_pkg is not None - if not add_dependency(context, phase, dep_pkg): - return False - return True - - -def fix_missing_python_module(error, phase, apt, context): - targeted = targeted_python_versions(context.tree, context.subpath) - default = not targeted - - if error.minimum_version: - specs = [(">=", error.minimum_version)] - else: - specs = [] - - pypy_pkg = get_package_for_python_module(apt, error.module, "pypy", specs) - py2_pkg = get_package_for_python_module(apt, error.module, "cpython2", specs) - py3_pkg = get_package_for_python_module(apt, error.module, "cpython3", specs) - - extra_build_deps = [] - if error.python_version == 2: - if "pypy" in targeted: - if not pypy_pkg: - logging.warning("no pypy package found for %s", error.module) - else: - extra_build_deps.append(pypy_pkg) - if "cpython2" in targeted or default: - if not py2_pkg: - logging.warning("no python 2 package found for %s", error.module) - return False - extra_build_deps.append(py2_pkg) - elif error.python_version == 3: - if not py3_pkg: - logging.warning("no python 3 package found for %s", error.module) - return False - extra_build_deps.append(py3_pkg) - else: - if py3_pkg and ("cpython3" in targeted or default): - extra_build_deps.append(py3_pkg) - if py2_pkg and ("cpython2" in targeted or default): - extra_build_deps.append(py2_pkg) - if pypy_pkg and "pypy" in targeted: - extra_build_deps.append(pypy_pkg) - - if not extra_build_deps: - return False - - for dep_pkg in extra_build_deps: - assert dep_pkg is not None - if not add_dependency(context, phase, dep_pkg): - return False - return True +def python_tie_breaker(tree, subpath, reqs): + targeted = targeted_python_versions(tree, subpath) + if not targeted: + return None + for prefix in targeted: + for req in reqs: + if any(name.startswith(prefix + '-') for name in req.package_names()): + logging.info( + 'Breaking tie between %r to %r, since package already ' + 'has %r build-dependencies', [str(req) for req in reqs], + str(req), prefix) + return req + return None def retry_apt_failure(error, phase, apt, context): @@ -536,12 +441,34 @@ def versioned_package_fixers(session, packaging_context): ] +def udd_tie_breaker(candidates): + # TODO(jelmer): Pick package based on what appears most commonly in + # build-depends{-indep,-arch} + try: + from .udd import UDD + except ModuleNotFoundError: + logging.warning('Unable to import UDD, not ranking by popcon') + return sorted(candidates, key=len)[0] + udd = UDD() + udd.connect() + names = {list(c.package_names())[0]: c for c in candidates} + winner = udd.get_most_popular(list(names.keys())) + if winner is None: + logging.warning( + 'No relevant popcon information found, not ranking by popcon') + return None + logging.info('Picked winner using popcon') + return names[winner] + + def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver - resolver = AptResolver(apt) + apt_tie_breakers = [ + partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), + udd_tie_breaker, + ] + resolver = AptResolver(apt, apt_tie_breakers) return [ - DependencyBuildFixer(packaging_context, apt, MissingPythonModule, fix_missing_python_module), - DependencyBuildFixer(packaging_context, apt, MissingPythonDistribution, fix_missing_python_distribution), DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure), PackageDependencyFixer(packaging_context, resolver), ] diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 0d04cb1..5d22160 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -20,6 +20,7 @@ import logging import os import posixpath import re +from typing import Optional, List from debian.changelog import Version from debian.deb822 import PkgRelation @@ -84,14 +85,30 @@ class AptRequirement(Requirement): def __str__(self): return "apt requirement: %s" % self.pkg_relation_str() - def touches_package(self, package): + def package_names(self): for rel in self.relations: for entry in rel: - if entry["name"] == package: - return True + yield entry["name"] + + def touches_package(self, package): + for name in self.package_names(): + if name == package: + return True return False +def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False) -> List[str]: + if not isinstance(paths, list): + raise TypeError(paths) + return apt_mgr.get_packages_for_paths(paths, regex) + + +def find_reqs_simple(apt_mgr: AptManager, paths: List[str], regex: bool = False, minimum_version=None) -> List[str]: + if not isinstance(paths, list): + raise TypeError(paths) + return [AptRequirement.simple(package, minimum_version=minimum_version) for package in find_package_names(apt_mgr, paths, regex)] + + def python_spec_to_apt_rels(pkg_name, specs): # TODO(jelmer): Dealing with epoch, etc? if not specs: @@ -133,11 +150,8 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st paths = [cpython3_regex, cpython2_regex, pypy_regex] else: raise NotImplementedError('unsupported python version %d' % python_version) - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is None: - return None - rels = python_spec_to_apt_rels(pkg_name, specs) - return AptRequirement(rels) + names = find_package_names(apt_mgr, paths, regex=True) + return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] def get_package_for_python_module(apt_mgr, module, python_version, specs): @@ -198,11 +212,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): paths = cpython3_regexes + cpython2_regexes + pypy_regexes else: raise AssertionError("unknown python version %r" % python_version) - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is None: - return None - rels = python_spec_to_apt_rels(pkg_name, specs) - return AptRequirement(rels) + names = find_package_names(apt_mgr, paths, regex=True) + return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] def resolve_binary_req(apt_mgr, req): @@ -212,61 +223,42 @@ def resolve_binary_req(apt_mgr, req): paths = [ posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] ] - pkg_name = apt_mgr.get_package_for_paths(paths) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths) def resolve_pkg_config_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")], - ) - if package is None: - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], - regex=True, - ) - if package is not None: - return AptRequirement.simple(package, minimum_version=req.minimum_version) - return None + names = find_package_names(apt_mgr, + [posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], + regex=True) + if not names: + names = find_package_names( + apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")]) + return [AptRequirement.simple(name, minimum_version=req.minimum_version) for name in names] def resolve_path_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths([req.path]) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, [req.path]) def resolve_c_header_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - [posixpath.join("/usr/include", req.header)], regex=False - ) - if package is None: - package = apt_mgr.get_package_for_paths( + reqs = find_reqs_simple( + apt_mgr, + [posixpath.join("/usr/include", req.header)], regex=False) + if not reqs: + reqs = find_package_names( + apt_mgr, [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True ) - if package is None: - return None - return AptRequirement.simple(package) + return reqs def resolve_js_runtime_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( - ["/usr/bin/node", "/usr/bin/duk"], regex=False - ) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, ["/usr/bin/node", "/usr/bin/duk"]) def resolve_vala_package_req(apt_mgr, req): path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package) - package = apt_mgr.get_package_for_paths([path], regex=True) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_ruby_gem_req(apt_mgr, req): @@ -276,43 +268,29 @@ def resolve_ruby_gem_req(apt_mgr, req): "specifications/%s-.*\\.gemspec" % re.escape(req.gem) ) ] - package = apt_mgr.get_package_for_paths(paths, regex=True) - if package is not None: - return AptRequirement.simple(package, minimum_version=req.minimum_version) - return None + return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version) def resolve_go_package_req(apt_mgr, req): - package = apt_mgr.get_package_for_paths( + return find_reqs_simple( + apt_mgr, [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True ) - if package is not None: - return AptRequirement.simple(package) - return None def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] - package = apt_mgr.get_package_for_paths(paths) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, paths) def resolve_php_class_req(apt_mgr, req): path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") - package = apt_mgr.get_package_for_paths([path]) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, [path]) def resolve_r_package_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] - package = apt_mgr.get_package_for_paths(paths, regex=True) - if package is not None: - return AptRequirement.simple(package) - return None + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_node_package_req(apt_mgr, req): @@ -321,10 +299,7 @@ def resolve_node_package_req(apt_mgr, req): "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_library_req(apt_mgr, req): @@ -334,27 +309,21 @@ def resolve_library_req(apt_mgr, req): posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths) def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] - package = apt_mgr.get_package_for_paths(paths) - if package is not None: - return AptRequirement.simple(package) + reqs = find_reqs_simple(apt_mgr, paths, regex=False) + if reqs: + return reqs paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" "lib/%s\\.rb" % re.escape(req.filename) ) ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_xml_entity_req(apt_mgr, req): @@ -370,10 +339,7 @@ def resolve_xml_entity_req(apt_mgr, req): else: return None - pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [search_path], regex=False) def resolve_sprockets_file_req(apt_mgr, req): @@ -382,10 +348,7 @@ def resolve_sprockets_file_req(apt_mgr, req): else: logging.warning("unable to handle content type %s", req.content_type) return None - pkg_name = apt_mgr.get_package_for_paths([path], regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_java_class_req(apt_mgr, req): @@ -399,19 +362,12 @@ def resolve_java_class_req(apt_mgr, req): logging.warning("unable to find classpath for %s", req.classname) return False logging.info("Classpath for %s: %r", req.classname, classpath) - package = apt_mgr.get_package_for_paths(classpath) - if package is None: - logging.warning("no package for files in %r", classpath) - return None - return AptRequirement.simple(package) + return find_reqs_simple(apt_mgr, [classpath]) def resolve_haskell_package_req(apt_mgr, req): path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) - pkg_name = apt_mgr.get_package_for_paths([path], regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_maven_artifact_req(apt_mgr, req): @@ -440,10 +396,7 @@ def resolve_maven_artifact_req(apt_mgr, req): "%s-%s.%s" % (artifact_id, version, kind), ) ] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=regex) def resolve_gnome_common_req(apt_mgr, req): @@ -452,10 +405,7 @@ def resolve_gnome_common_req(apt_mgr, req): def resolve_jdk_file_req(apt_mgr, req): path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename) - pkg_name = apt_mgr.get_package_for_paths([path], regex=True) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_jdk_req(apt_mgr, req): @@ -478,17 +428,11 @@ def resolve_perl_module_req(apt_mgr, req): paths = [req.filename] else: paths = [posixpath.join(inc, req.filename) for inc in req.inc] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=False) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, paths, regex=False) def resolve_perl_file_req(apt_mgr, req): - pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [req.filename], regex=False) def _find_aclocal_fun(macro): @@ -512,10 +456,7 @@ def resolve_autoconf_macro_req(apt_mgr, req): except KeyError: logging.info("No local m4 file found defining %s", req.macro) return None - pkg_name = apt_mgr.get_package_for_paths([path]) - if pkg_name is not None: - return AptRequirement.simple(pkg_name) - return None + return find_reqs_simple(apt_mgr, [path]) def resolve_python_module_req(apt_mgr, req): @@ -547,10 +488,7 @@ def resolve_python_package_req(apt_mgr, req): def resolve_cargo_crate_req(apt_mgr, req): paths = [ '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)] - pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) - if pkg_name is None: - return None - return AptRequirement.simple(pkg_name) + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_ca_req(apt_mgr, req): @@ -591,16 +529,24 @@ APT_REQUIREMENT_RESOLVERS = [ ] -def resolve_requirement_apt(apt_mgr, req: Requirement) -> AptRequirement: +def resolve_requirement_apt(apt_mgr, req: Requirement) -> List[AptRequirement]: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): - return rr_fn(apt_mgr, req) + ret = rr_fn(apt_mgr, req) + if not ret: + return [] + if not isinstance(ret, list): + raise TypeError(ret) + return ret raise NotImplementedError(type(req)) class AptResolver(Resolver): - def __init__(self, apt): + def __init__(self, apt, tie_breakers=None): self.apt = apt + if tie_breakers is None: + tie_breakers = [] + self.tie_breakers = tie_breakers def __str__(self): return "apt" @@ -647,4 +593,18 @@ class AptResolver(Resolver): yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements]) def resolve(self, req: Requirement): - return resolve_requirement_apt(self.apt, req) + ret = resolve_requirement_apt(self.apt, req) + if not ret: + return None + if len(ret) == 1: + return ret[0] + for tie_breaker in self.tie_breakers: + winner = tie_breaker(ret) + if winner is not None: + if not isinstance(winner, AptRequirement): + raise TypeError(winner) + return winner + logging.info( + 'Unable to break tie over %r, picking first: %r', + ret, ret[0]) + return ret[0] diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index a947003..4434a5f 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -37,6 +37,7 @@ from ..debian.fix_build import ( apt_fixers, DebianPackagingContext, ) +from breezy.commit import NullCommitReporter from breezy.tests import TestCaseWithTransport @@ -99,7 +100,8 @@ blah (0.1) UNRELEASED; urgency=medium apt._searchers = [DummyAptSearcher(self._apt_files)] context = DebianPackagingContext( self.tree, subpath="", committer="ognibuild ", - update_changelog=True) + update_changelog=True, + commit_reporter=NullCommitReporter()) fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context) return resolve_error(error, ("build", ), fixers) From aa8dd6bb7068ded2ec902999e386e9c238ebec87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 15:33:51 +0000 Subject: [PATCH 132/252] Add missing escape. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 5d22160..e78d182 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -295,7 +295,7 @@ def resolve_r_package_req(apt_mgr, req): def resolve_node_package_req(apt_mgr, req): paths = [ - "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % req.package, + "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % re.escape(req.package), "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), ] From ddea6f957a269771d04816dffb65121bc80c06ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 15:37:12 +0000 Subject: [PATCH 133/252] Distinguish node module and package. --- ognibuild/buildlog.py | 6 +++++- ognibuild/requirements.py | 12 ++++++++++++ ognibuild/resolver/apt.py | 11 +++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index b7b5f3e..5d4d7d3 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -37,6 +37,7 @@ from buildlog_consultant.common import ( MissingJDK, MissingJRE, MissingNodeModule, + MissingNodePackage, MissingPhpClass, MissingRubyGem, MissingLibrary, @@ -86,6 +87,7 @@ from .requirements import ( PythonModuleRequirement, PythonPackageRequirement, CertificateAuthorityRequirement, + NodeModuleRequirement, ) from .resolver import UnsatisfiedRequirements @@ -114,7 +116,9 @@ def problem_to_upstream_requirement(problem): # noqa: C901 elif isinstance(problem, MissingRPackage): return RPackageRequirement(problem.package, problem.minimum_version) elif isinstance(problem, MissingNodeModule): - return NodePackageRequirement(problem.module) + return NodeModuleRequirement(problem.module) + elif isinstance(problem, MissingNodePackage): + return NodePackageRequirement(problem.package) elif isinstance(problem, MissingLibrary): return LibraryRequirement(problem.library) elif isinstance(problem, MissingRubyFile): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 3b6fea9..7f89204 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -131,6 +131,18 @@ class NodePackageRequirement(Requirement): return "%s(%r)" % (type(self).__name__, self.package) +class NodeModuleRequirement(Requirement): + + module: str + + def __init__(self, module): + super(NodeModuleRequirement, self).__init__("npm-module") + self.module = module + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.module) + + class CargoCrateRequirement(Requirement): crate: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index e78d182..1463fb3 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -42,6 +42,7 @@ from ..requirements import ( DhAddonRequirement, PhpClassRequirement, RPackageRequirement, + NodeModuleRequirement, NodePackageRequirement, LibraryRequirement, RubyFileRequirement, @@ -293,6 +294,15 @@ def resolve_r_package_req(apt_mgr, req): return find_reqs_simple(apt_mgr, paths, regex=True) +def resolve_node_module_req(apt_mgr, req): + paths = [ + "/usr/share/nodejs/.*/node_modules/%s/index.js" % re.escape(req.module), + "/usr/lib/nodejs/%s/index.js" % re.escape(req.module), + "/usr/share/nodejs/%s/index.js" % re.escape(req.module), + ] + return find_reqs_simple(apt_mgr, paths, regex=True) + + def resolve_node_package_req(apt_mgr, req): paths = [ "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % re.escape(req.package), @@ -507,6 +517,7 @@ APT_REQUIREMENT_RESOLVERS = [ (DhAddonRequirement, resolve_dh_addon_req), (PhpClassRequirement, resolve_php_class_req), (RPackageRequirement, resolve_r_package_req), + (NodeModuleRequirement, resolve_node_module_req), (NodePackageRequirement, resolve_node_package_req), (LibraryRequirement, resolve_library_req), (RubyFileRequirement, resolve_ruby_file_req), From 6cb75e75b0f74b489e4ba9d17ef069db84848454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 15:41:53 +0000 Subject: [PATCH 134/252] Try harder to install node packages. --- ognibuild/resolver/__init__.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 51f7204..1ff1a79 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -348,18 +348,27 @@ class NpmResolver(Resolver): return "%s(%r)" % (type(self).__name__, self.session) def install(self, requirements): - from ..requirements import NodePackageRequirement + from ..requirements import ( + NodePackageRequirement, + NodeModuleRequirement, + BinaryRequirement, + ) missing = [] for requirement in requirements: + if isinstance(requirement, BinaryRequirement): + try: + package = NPM_COMMAND_PACKAGES[requirement.command] + except KeyError: + pass + else: + requirement = NodePackageRequirement(package) + if isinstance(requirement, NodeModuleRequirement): + # TODO: Is this legit? + requirement = NodePackageRequirement(requirement.module.split('/')[0]) if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - try: - package = NPM_COMMAND_PACKAGES[requirement.command] - except KeyError: - missing.append(requirement) - continue self.session.check_call(["npm", "-g", "install", package]) if missing: raise UnsatisfiedRequirements(missing) From a5a615d9e8be761f1f08869b3535c177929abe9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 15:49:08 +0000 Subject: [PATCH 135/252] Add husky. --- ognibuild/resolver/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 1ff1a79..c4e52e1 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -332,6 +332,7 @@ class GoResolver(Resolver): NPM_COMMAND_PACKAGES = { "del-cli": "del-cli", + "husky": "husky", } From 713a6dc2a7e2f24ec06dbedc18f26b70c0cd58b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 16:04:58 +0000 Subject: [PATCH 136/252] Pass in build directory. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 54abbd1..47af8db 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -327,7 +327,7 @@ class SetupPy(BuildSystem): self._run_setup(session, resolver, preargs + ["sdist"], fixers) return elif self.pyproject: - run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source"], fixers) + run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], fixers) return raise AssertionError("no setup.py or pyproject.toml") From b961a0d275e0b7482fab1178b32205d422bb9639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 16:18:38 +0000 Subject: [PATCH 137/252] Handle gradlew not being executable. --- ognibuild/buildsystem.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 47af8db..9e3e267 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,6 +20,7 @@ import logging import os import re +import stat from typing import Optional, Tuple import warnings @@ -508,7 +509,7 @@ class Gradle(BuildSystem): @classmethod def from_path(cls, path): if os.path.exists(os.path.join(path, "gradlew")): - return cls(path, "./gradlew") + return cls(path, './gradlew') return cls(path) @classmethod @@ -521,28 +522,31 @@ class Gradle(BuildSystem): if not self.executable.startswith('./'): resolver.install([BinaryRequirement(self.executable)]) - def clean(self, session, resolver, fixers): + def _run(self, session, resolver, args, fixers): self.setup(resolver) - run_with_build_fixers(session, [self.executable, "clean"], fixers) + if self.executable.startswith('./') and ( + not os.access(os.path.join(self.path, self.executable), os.X_OK)): + argv = ['sh', self.executable] + else: + argv = [self.executable] + run_with_build_fixers(session, argv + args, fixers) + + def clean(self, session, resolver, fixers): + self._run(session, resolver, ['clean'], fixers) def build(self, session, resolver, fixers): - self.setup(resolver) - run_with_build_fixers(session, [self.executable, "build"], fixers) + self._run(session, resolver, ['build'], fixers) def test(self, session, resolver, fixers): - self.setup(resolver) - run_with_build_fixers(session, [self.executable, "test"], fixers) + self._run(session, resolver, ['test'], fixers) def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) - run_with_build_fixers(session, [self.executable, "distTar"], fixers) + self._run(session, resolver, ['distTar'], fixers) def install(self, session, resolver, fixers, install_target): raise NotImplementedError - self.setup(resolver) # TODO(jelmer): installDist just creates files under build/install/... - run_with_build_fixers( - session, [self.executable, "installDist"], fixers) + self._run(session, resolver, ['installDist'], fixers) class R(BuildSystem): From 5030341d932915f44090d6f6a0f606dcd73590f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 16:27:53 +0000 Subject: [PATCH 138/252] Handle simple cases. --- ognibuild/resolver/apt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 1463fb3..b4e932e 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -150,7 +150,7 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st elif python_version is None: paths = [cpython3_regex, cpython2_regex, pypy_regex] else: - raise NotImplementedError('unsupported python version %d' % python_version) + raise NotImplementedError('unsupported python version %s' % python_version) names = find_package_names(apt_mgr, paths, regex=True) return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] @@ -410,7 +410,7 @@ def resolve_maven_artifact_req(apt_mgr, req): def resolve_gnome_common_req(apt_mgr, req): - return AptRequirement.simple("gnome-common") + return [AptRequirement.simple("gnome-common")] def resolve_jdk_file_req(apt_mgr, req): @@ -419,11 +419,11 @@ def resolve_jdk_file_req(apt_mgr, req): def resolve_jdk_req(apt_mgr, req): - return AptRequirement.simple('default-jdk') + return [AptRequirement.simple('default-jdk')] def resolve_jre_req(apt_mgr, req): - return AptRequirement.simple('default-jre') + return [AptRequirement.simple('default-jre')] def resolve_perl_module_req(apt_mgr, req): @@ -502,7 +502,7 @@ def resolve_cargo_crate_req(apt_mgr, req): def resolve_ca_req(apt_mgr, req): - return AptRequirement.simple('ca-certificates') + return [AptRequirement.simple('ca-certificates')] APT_REQUIREMENT_RESOLVERS = [ From 980e8b72020ab5b96249d972c5679f32c1beedb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 16:33:49 +0000 Subject: [PATCH 139/252] Fix repr. --- ognibuild/debian/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 19715ee..bf563ab 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -421,7 +421,7 @@ class DependencyBuildFixer(BuildFixer): self._fn = fn def __repr__(self): - return "%s(%s, %r, %s)" % ( + return "%s(%s, %s)" % ( type(self).__name__, self._problem_cls.__name__, self._fn.__name__) def can_fix(self, problem: Problem): From 00e4c5e14adca457ef17a84e9924bbdc23d34d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 16:37:20 +0000 Subject: [PATCH 140/252] Install setuptools preemptively. --- ognibuild/buildsystem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9e3e267..042d563 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -309,6 +309,10 @@ class SetupPy(BuildSystem): if os.path.exists(os.path.join(self.path, 'tox.ini')): run_with_build_fixers(session, ['tox'], fixers) elif self.has_setup_py: + # Pre-emptively insall setuptools, since distutils doesn't provide + # a 'test' subcommand and some packages fall back to distutils + # if setuptools is not available. + resolver.install([PythonPackageRequirement('setuptools')]) self._run_setup(session, resolver, ["test"], fixers) else: raise NotImplementedError From c6bcbd71fbac5f75a4a2b129eea206363082245c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 17:25:55 +0000 Subject: [PATCH 141/252] Fix attribute name. --- ognibuild/resolver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index c4e52e1..3383d8a 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -359,7 +359,7 @@ class NpmResolver(Resolver): for requirement in requirements: if isinstance(requirement, BinaryRequirement): try: - package = NPM_COMMAND_PACKAGES[requirement.command] + package = NPM_COMMAND_PACKAGES[requirement.binary_name] except KeyError: pass else: From ab585ea7992a85236734becd615fa5e936685e4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 17:41:47 +0000 Subject: [PATCH 142/252] Fix meson build if dir already exists. --- ognibuild/buildsystem.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 042d563..017fa76 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -620,9 +620,8 @@ class Meson(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def _setup(self, session, fixers): - if session.exists("build"): - return - session.check_call(['mkdir', 'build']) + if not session.exists('build'): + session.check_call(['mkdir', 'build']) run_with_build_fixers(session, ["meson", "setup", "build"], fixers) def clean(self, session, resolver, fixers): From 3b3e9bf0b2849f43cdfb9a7f9f323f2ebb840fe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 18:10:43 +0000 Subject: [PATCH 143/252] Add AptRequirement.__repr__. --- ognibuild/resolver/apt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index b4e932e..7f276c4 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -86,6 +86,9 @@ class AptRequirement(Requirement): def __str__(self): return "apt requirement: %s" % self.pkg_relation_str() + def __repr__(self): + return "%s.from_str(%r)" % (type(self).__name__, self.pkg_relation_str()) + def package_names(self): for rel in self.relations: for entry in rel: From b1d2012b18135a2ccfde8377aae62e7e1c49bb0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 18:15:09 +0000 Subject: [PATCH 144/252] Add TODO. --- ognibuild/dist.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 87c21c0..abdfacf 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -163,6 +163,7 @@ def create_dist( raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) raise + # TODO(jelmer): use scan_buildsystems to also look in subdirectories buildsystems = list(detect_buildsystems(export_directory)) resolver = auto_resolver(session) fixers = [InstallFixer(resolver)] From 834d858adbc1a555b048039fd084398181bb0ab3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 18:41:37 +0000 Subject: [PATCH 145/252] Call correct fn. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 7f276c4..ccf4af3 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -249,7 +249,7 @@ def resolve_c_header_req(apt_mgr, req): apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False) if not reqs: - reqs = find_package_names( + reqs = find_reqs_simple( apt_mgr, [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True ) From 82e680f67f587abf9a6638d53778df56bad503b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 18:45:32 +0000 Subject: [PATCH 146/252] Cope with missing setup.py. --- ognibuild/buildsystem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 017fa76..ec5fd16 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -245,6 +245,8 @@ class SetupPy(BuildSystem): raise FileNotFoundError(p) def _extract_setup(self, session=None, fixers=None): + if not self.has_setup_py: + return {} if session is None: return self._extract_setup_direct() else: @@ -308,6 +310,8 @@ class SetupPy(BuildSystem): def test(self, session, resolver, fixers): if os.path.exists(os.path.join(self.path, 'tox.ini')): run_with_build_fixers(session, ['tox'], fixers) + elif self.pyproject: + run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers) elif self.has_setup_py: # Pre-emptively insall setuptools, since distutils doesn't provide # a 'test' subcommand and some packages fall back to distutils From 375dd6fa0178b7f798b5a7c89f76bcaa8448894f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 20:29:12 +0000 Subject: [PATCH 147/252] Move udd tie breaker. --- ognibuild/debian/fix_build.py | 21 +-------------------- ognibuild/debian/udd.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index bf563ab..c0507c6 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -441,28 +441,9 @@ def versioned_package_fixers(session, packaging_context): ] -def udd_tie_breaker(candidates): - # TODO(jelmer): Pick package based on what appears most commonly in - # build-depends{-indep,-arch} - try: - from .udd import UDD - except ModuleNotFoundError: - logging.warning('Unable to import UDD, not ranking by popcon') - return sorted(candidates, key=len)[0] - udd = UDD() - udd.connect() - names = {list(c.package_names())[0]: c for c in candidates} - winner = udd.get_most_popular(list(names.keys())) - if winner is None: - logging.warning( - 'No relevant popcon information found, not ranking by popcon') - return None - logging.info('Picked winner using popcon') - return names[winner] - - def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver + from .udd import udd_tie_breaker apt_tie_breakers = [ partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), udd_tie_breaker, diff --git a/ognibuild/debian/udd.py b/ognibuild/debian/udd.py index e45a432..bd2f1ce 100644 --- a/ognibuild/debian/udd.py +++ b/ognibuild/debian/udd.py @@ -17,6 +17,8 @@ """Support for accessing UDD.""" +import logging + class UDD(object): @@ -37,3 +39,23 @@ class UDD(object): 'SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1', (tuple(packages), )) return cursor.fetchone()[0] + + +def udd_tie_breaker(candidates): + # TODO(jelmer): Pick package based on what appears most commonly in + # build-depends{-indep,-arch} + try: + from .udd import UDD + except ModuleNotFoundError: + logging.warning('Unable to import UDD, not ranking by popcon') + return sorted(candidates, key=len)[0] + udd = UDD() + udd.connect() + names = {list(c.package_names())[0]: c for c in candidates} + winner = udd.get_most_popular(list(names.keys())) + if winner is None: + logging.warning( + 'No relevant popcon information found, not ranking by popcon') + return None + logging.info('Picked winner using popcon') + return names[winner] From 6faffb5d65b7ef7306b8b2951cf7097b87c3f88d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 20:29:59 +0000 Subject: [PATCH 148/252] Pass along tie_breakers argument. --- ognibuild/resolver/apt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index ccf4af3..ade60b8 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -569,8 +569,8 @@ class AptResolver(Resolver): return "%s()" % (type(self).__name__, ) @classmethod - def from_session(cls, session): - return cls(AptManager.from_session(session)) + def from_session(cls, session, tie_breakers=None): + return cls(AptManager.from_session(session), tie_breakers=tie_breakers) def install(self, requirements): missing = [] From 219a2dae0a128eae1e09c76058c1d483723b7f52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 20:40:14 +0000 Subject: [PATCH 149/252] Fix NPM installs. --- ognibuild/resolver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 3383d8a..0b142c4 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -370,7 +370,7 @@ class NpmResolver(Resolver): if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - self.session.check_call(["npm", "-g", "install", package]) + self.session.check_call(["npm", "-g", "install", requirement.package]) if missing: raise UnsatisfiedRequirements(missing) From 33aa977763efe4a502025b85d1b33e21d912427b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 20:56:26 +0000 Subject: [PATCH 150/252] Always specify interpreter. --- ognibuild/buildsystem.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index ec5fd16..4a78f90 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -361,11 +361,9 @@ class SetupPy(BuildSystem): # setuptools might fetch eggs instead of our preferred resolver. install_missing_reqs(session, resolver, list(self._setup_requires())) interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) - if interpreter is not None: - argv = ["./setup.py"] + args - else: - # Just assume it's Python 3 - argv = [self.DEFAULT_PYTHON, "./setup.py"] + args + if interpreter is None: + interpreter = self.DEFAULT_PYTHON + argv = [interpreter, "./setup.py"] + args env = {} # Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment if 'SETUPTOOLS_SCM_PRETEND_VERSION' in os.environ: From 415a183d79a3d6b397d4f5aee9f22cf4562c9281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:03:44 +0000 Subject: [PATCH 151/252] Find R path. --- ognibuild/buildsystem.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 4a78f90..7277897 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -40,6 +40,14 @@ from .requirements import ( OctavePackageRequirement, ) from .fix_build import run_with_build_fixers +from .session import which + + +def guaranteed_which(session, resolver, name): + path = which(session, name) + if not path: + resolver.install([BinaryRequirement(name)]) + return which(session, name) class NoBuildToolsFound(Exception): @@ -571,13 +579,16 @@ class R(BuildSystem): pass def dist(self, session, resolver, fixers, quiet=False): - run_with_build_fixers(session, ["R", "CMD", "build", "."], fixers) + r_path = guaranteed_which(session, resolver, "R") + run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) def install(self, session, resolver, fixers, install_target): - run_with_build_fixers(session, ["R", "CMD", "INSTALL", "."], fixers) + r_path = guaranteed_which(session, resolver, "R") + run_with_build_fixers(session, [r_path, "CMD", "INSTALL", "."], fixers) def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["R", "CMD", "check", "."], fixers) + r_path = guaranteed_which(session, resolver, "R") + run_with_build_fixers(session, [r_path, "CMD", "check", "."], fixers) @classmethod def probe(cls, path): From 4d8571556ae79095fbe0a292b04642fbd81aa87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:11:26 +0000 Subject: [PATCH 152/252] Add which. --- ognibuild/session/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index f9f0225..4693341 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -118,3 +118,10 @@ def run_with_tee(session: Session, args: List[str], **kwargs): def get_user(session): return session.check_output(["echo", "$USER"], cwd="/").decode().strip() + + +def which(session, name): + ret = session.check_output(["which", name], cwd="/").decode().strip() + if not ret: + return None + return ret From c651b061efb29f75eb6ab6ccf7eabdf83a38c633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:23:06 +0000 Subject: [PATCH 153/252] Always check for make last. --- ognibuild/buildsystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 7277897..86ad00e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1129,7 +1129,9 @@ class PerlBuildTiny(BuildSystem): BUILDSYSTEM_CLSES = [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistInkt, Gem, Make, PerlBuildTiny, Golang, R, Octave] + DistInkt, Gem, PerlBuildTiny, Golang, R, Octave, + # Make is intentionally at the end of the list. + Make] def scan_buildsystems(path): From 7547b28dfdbb5c06ee8974a05a7934e75948fe64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:37:08 +0000 Subject: [PATCH 154/252] Handle command being absent in which. --- ognibuild/session/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 4693341..320cd64 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -121,7 +121,12 @@ def get_user(session): def which(session, name): - ret = session.check_output(["which", name], cwd="/").decode().strip() + try: + ret = session.check_output(["which", name], cwd="/").decode().strip() + except subprocess.CalledProcessError as e: + if e.returncode == 1: + return None + raise if not ret: return None return ret From c52ddccd176d1ed369141fcbfc25786596fe583d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:44:49 +0000 Subject: [PATCH 155/252] Only return packages once. --- ognibuild/debian/file_search.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 1dfbb54..dac6a27 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -270,7 +270,9 @@ def get_packages_for_paths( candidates: List[str] = list() for path in paths: for searcher in searchers: - candidates.extend(searcher.search_files(path, regex=regex)) + for pkg in searcher.search_files(path, regex=regex): + if pkg not in candidates: + candidates.append(pkg) return candidates From 663f67574a6ca046e125f9c51d79a2a34cb9feb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:46:04 +0000 Subject: [PATCH 156/252] Implement AptRequirement.__eq__. --- ognibuild/resolver/apt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index ade60b8..126c82f 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -83,6 +83,9 @@ class AptRequirement(Requirement): def pkg_relation_str(self): return PkgRelation.str(self.relations) + def __eq__(self, other): + return isinstance(self, type(other)) and self.relations == other.relations + def __str__(self): return "apt requirement: %s" % self.pkg_relation_str() From db36bbf06c8b6b1c97bff8c25c79835ffab6b248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:49:43 +0000 Subject: [PATCH 157/252] Fix setup.py-less build. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 86ad00e..1a34419 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -254,7 +254,7 @@ class SetupPy(BuildSystem): def _extract_setup(self, session=None, fixers=None): if not self.has_setup_py: - return {} + return None if session is None: return self._extract_setup_direct() else: From 40ad64b000b175fef79db9f4b939ebcb67233687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 21:57:04 +0000 Subject: [PATCH 158/252] Whitespace. --- ognibuild/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 7f89204..a9aa927 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -281,7 +281,7 @@ class RPackageRequirement(Requirement): @classmethod def from_str(cls, text): # TODO(jelmer): More complex parser - m = re.fullmatch(r'(.*) \(>= (.*)\)', text) + m = re.fullmatch(r'(.*)\s+\(>= (.*)\)', text) if m: return cls(m.group(1), m.group(2)) m = re.fullmatch(r'([^ ]+)', text) @@ -316,7 +316,7 @@ class OctavePackageRequirement(Requirement): @classmethod def from_str(cls, text): # TODO(jelmer): More complex parser - m = re.fullmatch(r'(.*) \(>= (.*)\)', text) + m = re.fullmatch(r'(.*)\s+\(>= (.*)\)', text) if m: return cls(m.group(1), m.group(2)) m = re.fullmatch(r'([^ ]+)', text) From 177411725d9af7ff6d00b7e9fe42d60b8d87a767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:19:38 +0000 Subject: [PATCH 159/252] Add new tie breaker. --- ognibuild/debian/build_deps.py | 65 ++++++++++++++++++++++++++++++++++ ognibuild/resolver/apt.py | 3 ++ 2 files changed, 68 insertions(+) create mode 100644 ognibuild/debian/build_deps.py diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py new file mode 100644 index 0000000..a8ef301 --- /dev/null +++ b/ognibuild/debian/build_deps.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 +# Copyright (C) 2021 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +"""Tie breaking by build deps.""" + + +class BuildDependencyTieBreaker(object): + + def __init__(self, rootdir): + import apt_pkg + apt_pkg.init() + apt_pkg.config.set('Dir', rootdir) + self._apt_cache = apt_pkg.SourceRecords() + self._counts = None + + @classmethod + def from_session(cls, session): + return cls(session.location) + + def _count(self): + counts = {} + self._apt_cache.restart() + while self._apt_cache.step(): + try: + for d in self._apt_cache.build_depends.values(): + for o in d: + for p in o: + counts.setdefault(p[0], 0) + counts[p[0]] += 1 + except AttributeError: + pass + return counts + + def __call__(self, reqs): + if self._counts is None: + self._counts = self._count() + by_count = {} + for req in reqs: + by_count[req] = self._counts.get(list(req.package_names())[0]) + return max(by_count.items(), key=lambda k: k[1] or 0)[0] + + +if __name__ == '__main__': + import argparse + from ..resolver.apt import AptRequirement + parser = argparse.ArgumentParser() + parser.add_argument('req', nargs='+') + args = parser.parse_args() + reqs = [AptRequirement.from_str(req) for req in args.req] + tie_breaker = BuildDependencyTieBreaker('/') + print(tie_breaker(reqs)) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 126c82f..d79a1f7 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -83,6 +83,9 @@ class AptRequirement(Requirement): def pkg_relation_str(self): return PkgRelation.str(self.relations) + def __hash__(self): + return hash((type(self), self.pkg_relation_str())) + def __eq__(self, other): return isinstance(self, type(other)) and self.relations == other.relations From 23cfa120ef33700bd020ee721333f43e9ddc3251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:21:24 +0000 Subject: [PATCH 160/252] use new tie breaker. --- ognibuild/debian/build_deps.py | 9 +++++++-- ognibuild/debian/fix_build.py | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index a8ef301..a39ef50 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -50,8 +50,13 @@ class BuildDependencyTieBreaker(object): self._counts = self._count() by_count = {} for req in reqs: - by_count[req] = self._counts.get(list(req.package_names())[0]) - return max(by_count.items(), key=lambda k: k[1] or 0)[0] + try: + by_count[req] = self._counts[list(req.package_names())[0]] + except KeyError: + pass + if not by_count: + return None + return max(by_count.items(), key=lambda k: k[1])[0] if __name__ == '__main__': diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index c0507c6..4530c25 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -444,8 +444,10 @@ def versioned_package_fixers(session, packaging_context): def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver from .udd import udd_tie_breaker + from .build_deps import BuildDependencyTieBreaker apt_tie_breakers = [ partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), + BuildDependencyTieBreaker.from_session(apt.sesion), udd_tie_breaker, ] resolver = AptResolver(apt, apt_tie_breakers) From 0a1f8216601fa9668ea2ca44e50fbb7d69c175a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:22:39 +0000 Subject: [PATCH 161/252] Can't create apt object :( --- ognibuild/debian/build_deps.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index a39ef50..e4ef97c 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -21,10 +21,7 @@ class BuildDependencyTieBreaker(object): def __init__(self, rootdir): - import apt_pkg - apt_pkg.init() - apt_pkg.config.set('Dir', rootdir) - self._apt_cache = apt_pkg.SourceRecords() + self.rootdir = rootdir self._counts = None @classmethod @@ -33,10 +30,14 @@ class BuildDependencyTieBreaker(object): def _count(self): counts = {} - self._apt_cache.restart() - while self._apt_cache.step(): + import apt_pkg + apt_pkg.init() + apt_pkg.config.set('Dir', self.rootdir) + apt_cache = apt_pkg.SourceRecords() + apt_cache.restart() + while apt_cache.step(): try: - for d in self._apt_cache.build_depends.values(): + for d in apt_cache.build_depends.values(): for o in d: for p in o: counts.setdefault(p[0], 0) From ce02dd4a85366896e45633ac83be7ff78dba0997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:31:42 +0000 Subject: [PATCH 162/252] Fix typo. --- ognibuild/debian/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 4530c25..cfab2e5 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -447,7 +447,7 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from .build_deps import BuildDependencyTieBreaker apt_tie_breakers = [ partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), - BuildDependencyTieBreaker.from_session(apt.sesion), + BuildDependencyTieBreaker.from_session(apt.session), udd_tie_breaker, ] resolver = AptResolver(apt, apt_tie_breakers) From bfc2141626b767a764646367216c7926622f80f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:48:34 +0000 Subject: [PATCH 163/252] Rename udd_tie_breaker -> popcon_tie_breaker. --- ognibuild/debian/fix_build.py | 4 ++-- ognibuild/debian/udd.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index cfab2e5..9db884c 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -443,12 +443,12 @@ def versioned_package_fixers(session, packaging_context): def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver - from .udd import udd_tie_breaker + from .udd import popcon_tie_breaker from .build_deps import BuildDependencyTieBreaker apt_tie_breakers = [ partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), BuildDependencyTieBreaker.from_session(apt.session), - udd_tie_breaker, + popcon_tie_breaker, ] resolver = AptResolver(apt, apt_tie_breakers) return [ diff --git a/ognibuild/debian/udd.py b/ognibuild/debian/udd.py index bd2f1ce..223eb3c 100644 --- a/ognibuild/debian/udd.py +++ b/ognibuild/debian/udd.py @@ -41,7 +41,7 @@ class UDD(object): return cursor.fetchone()[0] -def udd_tie_breaker(candidates): +def popcon_tie_breaker(candidates): # TODO(jelmer): Pick package based on what appears most commonly in # build-depends{-indep,-arch} try: From 89417574eee37321b2c2c0f91e6ca5ab41be9a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:50:44 +0000 Subject: [PATCH 164/252] Install system-wide with root. --- ognibuild/resolver/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 0b142c4..cd6614c 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -355,6 +355,11 @@ class NpmResolver(Resolver): BinaryRequirement, ) + if self.user_local: + user = None + else: + user = "root" + missing = [] for requirement in requirements: if isinstance(requirement, BinaryRequirement): @@ -370,7 +375,9 @@ class NpmResolver(Resolver): if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - self.session.check_call(["npm", "-g", "install", requirement.package]) + self.session.check_call( + ["npm", "-g", "install", requirement.package], + user=user) if missing: raise UnsatisfiedRequirements(missing) From 362ed8df9a150079d22a50c161819de9ff010629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 23 Mar 2021 22:56:03 +0000 Subject: [PATCH 165/252] Handle aclocal. --- ognibuild/debian/file_search.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index dac6a27..cba1c26 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -259,7 +259,9 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( { "/etc/locale.gen": "locales", # Alternative - "/usr/bin/rst2html": "/usr/share/docutils/scripts/python3/rst2html", + "/usr/bin/rst2html": "python3-docutils", + # aclocal is a symlink to aclocal-1.XY + "/usr/bin/aclocal": "automake", } ) From 855041aee928b50001c7efdc1a3953357eea245e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 00:21:11 +0000 Subject: [PATCH 166/252] Support loading from file. --- ognibuild/debian/file_search.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index cba1c26..dda9dc6 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -244,6 +244,18 @@ class GeneratedFileSearcher(FileSearcher): def __init__(self, db): self._db = db + @classmethod + def from_path(cls, path): + self = cls({}) + self.load_from_path(path) + return self + + def load_from_path(self, path): + with open(path, 'r') as f: + for line in f: + (path, pkg) = line.strip().split(None, 1) + self._db[path] = pkg + def search_files(self, path: str, regex: bool = False) -> Iterator[str]: for p, pkg in sorted(self._db.items()): if regex: From d4021d8220240cb149ce334792f438ab34aaa9d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 02:42:57 +0000 Subject: [PATCH 167/252] Build in 'dist' since dzil will remove the directory. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 1a34419..e3c0cf5 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -782,7 +782,7 @@ class DistInkt(BuildSystem): else: # Default to invoking Dist::Zilla resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers(session, ["dzil", "build", "--in", ".."], fixers) + run_with_build_fixers(session, ["dzil", "build", "--in", "dist"], fixers) @classmethod def probe(cls, path): From 68242735990d00902893fdc945abb40b52621f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 02:56:34 +0000 Subject: [PATCH 168/252] Install qt. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 6 ++++++ ognibuild/resolver/apt.py | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 5d4d7d3..5cdbc25 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -54,6 +54,7 @@ from buildlog_consultant.common import ( GnomeCommonMissing, MissingGnomeCommonDependency, UnknownCertificateAuthority, + MissingQt, ) from .fix_build import BuildFixer @@ -88,6 +89,7 @@ from .requirements import ( PythonPackageRequirement, CertificateAuthorityRequirement, NodeModuleRequirement, + QTRequirement, ) from .resolver import UnsatisfiedRequirements @@ -143,6 +145,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return JDKRequirement() elif isinstance(problem, MissingJRE): return JRERequirement() + elif isinstance(problem, MissingQt): + return QTRequirement() elif isinstance(problem, UnknownCertificateAuthority): return CertificateAuthorityRequirement(problem.url) elif isinstance(problem, MissingGnomeCommonDependency): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index a9aa927..d513982 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -428,6 +428,12 @@ class JRERequirement(Requirement): super(JRERequirement, self).__init__("jre") +class QTRequirement(Requirement): + + def __init__(self): + super(QTRequirement, self).__init__("qt") + + class CertificateAuthorityRequirement(Requirement): def __init__(self, url): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index d79a1f7..3b5b631 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -55,6 +55,7 @@ from ..requirements import ( JDKFileRequirement, JDKRequirement, JRERequirement, + QTRequirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, @@ -435,6 +436,10 @@ def resolve_jre_req(apt_mgr, req): return [AptRequirement.simple('default-jre')] +def resolve_qt_req(apt_mgr, req): + return find_reqs_simple(apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) + + def resolve_perl_module_req(apt_mgr, req): DEFAULT_PERL_PATHS = ["/usr/share/perl5"] @@ -539,6 +544,7 @@ APT_REQUIREMENT_RESOLVERS = [ (JDKFileRequirement, resolve_jdk_file_req), (JDKRequirement, resolve_jdk_req), (JRERequirement, resolve_jre_req), + (QTRequirement, resolve_qt_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), (AutoconfMacroRequirement, resolve_autoconf_macro_req), From 2262b299cc3c168f6b0a6e16f01923cf85fd5ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 03:10:56 +0000 Subject: [PATCH 169/252] Add git identity fixer. --- ognibuild/dist.py | 3 ++- ognibuild/fixers.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 ognibuild/fixers.py diff --git a/ognibuild/dist.py b/ognibuild/dist.py index abdfacf..fc70ccb 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -152,6 +152,7 @@ def create_dist( ) -> Optional[str]: from .buildsystem import detect_buildsystems from .buildlog import InstallFixer + from .fixers import GitIdentityFixer if subdir is None: subdir = "package" @@ -166,7 +167,7 @@ def create_dist( # TODO(jelmer): use scan_buildsystems to also look in subdirectories buildsystems = list(detect_buildsystems(export_directory)) resolver = auto_resolver(session) - fixers = [InstallFixer(resolver)] + fixers = [InstallFixer(resolver), GitIdentityFixer(session)] with DistCatcher(export_directory) as dc: session.chdir(reldir) diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py new file mode 100644 index 0000000..2a37773 --- /dev/null +++ b/ognibuild/fixers.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import subprocess +from typing import Tuple + +from buildlog_consultant import Problem +from buildlog_consultant.common import ( + MissingGitIdentity, + ) + + +class GitIdentityFixer(object): + + def __init__(self, session): + self.session = session + + def can_fix(self, problem: Problem): + return isinstance(problem, MissingGitIdentity) + + def _fix(self, problem: Problem, phase: Tuple[str, ...]): + for name in ['user.email', 'user.name']: + value = subprocess.check_output( + ['git', 'config', '--global', name]).decode().strip() + self.session.check_call( + ['git', 'config', '--global', 'user.email', value]) + return True From 815a6796fd61fede05c63afc75afd5faaa902594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 03:30:38 +0000 Subject: [PATCH 170/252] Handle distTar not existing. --- ognibuild/buildsystem.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index e3c0cf5..f216602 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -536,31 +536,39 @@ class Gradle(BuildSystem): if not self.executable.startswith('./'): resolver.install([BinaryRequirement(self.executable)]) - def _run(self, session, resolver, args, fixers): + def _run(self, session, resolver, task, args, fixers): self.setup(resolver) + argv = [] if self.executable.startswith('./') and ( not os.access(os.path.join(self.path, self.executable), os.X_OK)): - argv = ['sh', self.executable] - else: - argv = [self.executable] - run_with_build_fixers(session, argv + args, fixers) + argv.append('sh') + argv.extend([self.executable, task]) + argv.extend(args) + try: + run_with_build_fixers(session, argv, fixers) + except UnidentifiedError as e: + if any([re.match( + r"Task '" + task + "' not found in root project '.*'\.", + line) for line in e.lines]): + raise NotImplementedError + raise def clean(self, session, resolver, fixers): - self._run(session, resolver, ['clean'], fixers) + self._run(session, resolver, 'clean', [], fixers) def build(self, session, resolver, fixers): - self._run(session, resolver, ['build'], fixers) + self._run(session, resolver, 'build', [], fixers) def test(self, session, resolver, fixers): - self._run(session, resolver, ['test'], fixers) + self._run(session, resolver, 'test', [], fixers) def dist(self, session, resolver, fixers, quiet=False): - self._run(session, resolver, ['distTar'], fixers) + self._run(session, resolver, 'distTar', [], fixers) def install(self, session, resolver, fixers, install_target): raise NotImplementedError # TODO(jelmer): installDist just creates files under build/install/... - self._run(session, resolver, ['installDist'], fixers) + self._run(session, resolver, 'installDist', [], fixers) class R(BuildSystem): From 4d84971dc4a314dfc9b242ea5eb3c025501a5780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 03:32:32 +0000 Subject: [PATCH 171/252] Fix inheritance. --- ognibuild/fixers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index 2a37773..94c8c72 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -19,12 +19,13 @@ import subprocess from typing import Tuple from buildlog_consultant import Problem +from buildlog_consultant.fix_build import BuildFixer from buildlog_consultant.common import ( MissingGitIdentity, ) -class GitIdentityFixer(object): +class GitIdentityFixer(BuildFixer): def __init__(self, session): self.session = session From 1c859b2fd352c4b81d7d4900e24e6603e4d895cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 03:35:19 +0000 Subject: [PATCH 172/252] Allow different whitespace. --- ognibuild/requirements.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index d513982..bdd3901 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -281,7 +281,7 @@ class RPackageRequirement(Requirement): @classmethod def from_str(cls, text): # TODO(jelmer): More complex parser - m = re.fullmatch(r'(.*)\s+\(>= (.*)\)', text) + m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text) if m: return cls(m.group(1), m.group(2)) m = re.fullmatch(r'([^ ]+)', text) @@ -316,7 +316,7 @@ class OctavePackageRequirement(Requirement): @classmethod def from_str(cls, text): # TODO(jelmer): More complex parser - m = re.fullmatch(r'(.*)\s+\(>= (.*)\)', text) + m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text) if m: return cls(m.group(1), m.group(2)) m = re.fullmatch(r'([^ ]+)', text) From 664ffd7b0a1b86ff5d1916496a4debc952f65692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 03:41:38 +0000 Subject: [PATCH 173/252] Fix import. --- ognibuild/fixers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index 94c8c72..9eeb4b4 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -19,11 +19,12 @@ import subprocess from typing import Tuple from buildlog_consultant import Problem -from buildlog_consultant.fix_build import BuildFixer from buildlog_consultant.common import ( MissingGitIdentity, ) +from .fix_build import BuildFixer + class GitIdentityFixer(BuildFixer): From f29d45dbb257275aa2ce81f00c9ada4fce99c4c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 03:55:42 +0000 Subject: [PATCH 174/252] Fix git identity? --- ognibuild/fixers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index 9eeb4b4..68764f5 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -39,5 +39,5 @@ class GitIdentityFixer(BuildFixer): value = subprocess.check_output( ['git', 'config', '--global', name]).decode().strip() self.session.check_call( - ['git', 'config', '--global', 'user.email', value]) + ['git', 'config', '--global', name, value]) return True From 95b9ce3990abd508c2f347cc3cea620782b44e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 04:45:38 +0000 Subject: [PATCH 175/252] Add env argument to PlainSession.Popen. --- ognibuild/session/plain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index a27631e..c7507f1 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -79,9 +79,9 @@ class PlainSession(Session): argv = self._prepend_user(user, argv) return subprocess.check_output(argv, cwd=cwd, env=env) - def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): + def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None, env=None): args = self._prepend_user(user, args) - return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) + return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd, env=env) def exists(self, path): return os.path.exists(path) From 47dda67229b8eb203ad918e436a48738c7bca99e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 04:55:40 +0000 Subject: [PATCH 176/252] Handle DistNoTarball. --- ognibuild/dist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index fc70ccb..01a5b5d 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -272,6 +272,8 @@ if __name__ == "__main__": logging.fatal('Unidentified error: %r', e.lines) except DetailedFailure as e: logging.fatal('Identified error during dist creation: %s', e.error) + except DistNoTarball: + logging.fatal('dist operation did not create a tarball') else: logging.info("Created %s", ret) sys.exit(0) From b79dcf0bfa64304e7aa74b0fd76542cba804ab3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 04:56:05 +0000 Subject: [PATCH 177/252] Fix tgz for dist-zilla. --- ognibuild/buildsystem.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f216602..759ec01 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -753,10 +753,12 @@ class Gem(BuildSystem): return cls(gemfiles[0]) -class DistInkt(BuildSystem): +class DistZilla(BuildSystem): + + name = "dist-zilla" + def __init__(self, path): self.path = path - self.name = "dist-zilla" self.dist_inkt_class = None with open(self.path, "rb") as f: for line in f: @@ -790,7 +792,17 @@ class DistInkt(BuildSystem): else: # Default to invoking Dist::Zilla resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers(session, ["dzil", "build", "--in", "dist"], fixers) + run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers) + + def test(self, session, resolver, fixers): + self.setup(resolver) + resolver.install([PerlModuleRequirement("Dist::Zilla")]) + run_with_build_fixers(session, ["dzil", "test"], fixers) + + def build(self, session, resolver, fixers): + self.setup(resolver) + resolver.install([PerlModuleRequirement("Dist::Zilla")]) + run_with_build_fixers(session, ["dzil", "build"], fixers) @classmethod def probe(cls, path): @@ -1137,7 +1149,7 @@ class PerlBuildTiny(BuildSystem): BUILDSYSTEM_CLSES = [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistInkt, Gem, PerlBuildTiny, Golang, R, Octave, + DistZilla, Gem, PerlBuildTiny, Golang, R, Octave, # Make is intentionally at the end of the list. Make] From b80bc0b83e94a6556d07449c066e3ace3720171d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 05:07:30 +0000 Subject: [PATCH 178/252] Fix package.json. --- ognibuild/buildsystem.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 759ec01..9e627fb 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -20,6 +20,7 @@ import logging import os import re +import shlex import stat from typing import Optional, Tuple import warnings @@ -679,9 +680,14 @@ class Npm(BuildSystem): def __init__(self, path): import json + self.path = path + with open(path, "r") as f: self.package = json.load(f) + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + def get_declared_dependencies(self, session, fixers=None): if "devDependencies" in self.package: for name, unused_version in self.package["devDependencies"].items(): @@ -695,6 +701,30 @@ class Npm(BuildSystem): self.setup(resolver) run_with_build_fixers(session, ["npm", "pack"], fixers) + def test(self, session, resolver, fixers): + self.setup(resolver) + test_script = self.package['scripts'].get('test') + if test_script: + run_with_build_fixers(session, shlex.split(test_script), fixers) + else: + raise NotImplementedError + + def build(self, session, resolver, fixers): + self.setup(resolver) + build_script = self.package['scripts'].get('build') + if build_script: + run_with_build_fixers(session, shlex.split(build_script), fixers) + else: + raise NotImplementedError + + def clean(self, session, resolver, fixers): + self.setup(resolver) + clean_script = self.package['scripts'].get('clean') + if clean_script: + run_with_build_fixers(session, shlex.split(clean_script), fixers) + else: + raise NotImplementedError + @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, "package.json")): From 2fb54e9ac9d029af9f012c3fca3282260a564fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 14:12:59 +0000 Subject: [PATCH 179/252] Allow control files in root. --- ognibuild/debian/build.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 227a38b..ea5a40a 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -36,6 +36,7 @@ from debmutate.changelog import get_maintainer, format_datetime from breezy import osutils from breezy.mutabletree import MutableTree from breezy.plugins.debian.builder import BuildFailedError +from breezy.tree import Tree from buildlog_consultant.sbuild import ( worker_failure_from_sbuild_log, @@ -71,6 +72,18 @@ def get_build_architecture(): raise Exception("Could not find the build architecture: %s" % e) +def control_files_in_root(tree: Tree, subpath: str) -> bool: + debian_path = os.path.join(subpath, "debian") + if tree.has_filename(debian_path): + return False + control_path = os.path.join(subpath, "control") + if tree.has_filename(control_path): + return True + if tree.has_filename(control_path + ".in"): + return True + return False + + def add_dummy_changelog_entry( tree: MutableTree, subpath: str, @@ -99,7 +112,10 @@ def add_dummy_changelog_entry( else: return v + suffix + "1" - path = os.path.join(subpath, "debian", "changelog") + if control_files_in_root(tree, subpath): + path = os.path.join(subpath, "changelog") + else: + path = os.path.join(subpath, "debian", "changelog") if maintainer is None: maintainer = get_maintainer() if timestamp is None: @@ -126,7 +142,10 @@ def add_dummy_changelog_entry( def get_latest_changelog_version(local_tree, subpath=""): - path = osutils.pathjoin(subpath, "debian/changelog") + if control_files_in_root(local_tree, subpath): + path = os.path.join(subpath, "changelog") + else: + path = os.path.join(subpath, "debian", "changelog") with local_tree.get_file(path) as f: cl = Changelog(f, max_blocks=1) return cl.package, cl.version From a7d3f5d2f3cdb5e3cde8f789c1d9a2fdee01c8d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:11:22 +0000 Subject: [PATCH 180/252] Include apt and tie breakers in repr. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 3b5b631..14535d1 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -578,7 +578,7 @@ class AptResolver(Resolver): return "apt" def __repr__(self): - return "%s()" % (type(self).__name__, ) + return "%s(%r, %r)" % (type(self).__name__, self.apt, self.tie_breakers) @classmethod def from_session(cls, session, tie_breakers=None): From 8efbae883d5f4417a3fa0d47ea811cb0437e5519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:13:32 +0000 Subject: [PATCH 181/252] Log winners. --- ognibuild/debian/build_deps.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index e4ef97c..761b27f 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -18,6 +18,9 @@ """Tie breaking by build deps.""" +import logging + + class BuildDependencyTieBreaker(object): def __init__(self, rootdir): @@ -57,7 +60,11 @@ class BuildDependencyTieBreaker(object): pass if not by_count: return None - return max(by_count.items(), key=lambda k: k[1])[0] + top = max(by_count.items(), key=lambda k: k[1]) + logging.info( + 'Breaking tie between %r to %r based on build-depends count', + [repr(r) for r in reqs], top[0]) + return top[0] if __name__ == '__main__': From b31ac7b50e86bb452397a0fc4e21efc87e8208a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:20:31 +0000 Subject: [PATCH 182/252] Also tie break libpythonXX. --- ognibuild/debian/fix_build.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 9db884c..9635a8f 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -115,7 +115,6 @@ from ..buildlog import problem_to_upstream_requirement from ..fix_build import BuildFixer, resolve_error from ..resolver.apt import ( AptRequirement, - get_package_for_python_module, ) from .build import attempt_build, DEFAULT_BUILDER @@ -292,13 +291,20 @@ def python_tie_breaker(tree, subpath, reqs): targeted = targeted_python_versions(tree, subpath) if not targeted: return None - for prefix in targeted: + + def same(pkg, python_version): + if pkg.startswith(python_version + '-'): + return True + if pkg.startswith('lib%s-' % python_version): + return True + return False + for python_version in targeted: for req in reqs: - if any(name.startswith(prefix + '-') for name in req.package_names()): + if any(same(name, python_version) for name in req.package_names()): logging.info( 'Breaking tie between %r to %r, since package already ' 'has %r build-dependencies', [str(req) for req in reqs], - str(req), prefix) + str(req), python_version) return req return None From 4421546c0d6337b3fdfa28c61fc90e130e180ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:27:52 +0000 Subject: [PATCH 183/252] Add some maven support. --- ognibuild/buildsystem.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9e627fb..98278b5 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1103,6 +1103,23 @@ class Maven(BuildSystem): logging.debug("Found pom.xml, assuming maven package.") return cls(os.path.join(path, "pom.xml")) + def test(self, session, resolver, fixers): + run_with_build_fixers(session, ["maven", "test"], fixers) + + def clean(self, session, resolver, fixers): + run_with_build_fixers(session, ["maven", "clean"], fixers) + + def install(self, session, resolver, fixers, install_target): + run_with_build_fixers(session, ["maven", "install"], fixers) + + def build(self, session, resolver, fixers): + run_with_build_fixers(session, ["maven", "compile"], fixers) + + def dist(self, session, resolver, fixers): + # TODO(jelmer): 'mvn generate-sources' creates a jar in target/. + # is that what we need? + raise NotImplementedError + class Cabal(BuildSystem): From b34a19ca2389ce2f6d30724943caa709448e6399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:32:17 +0000 Subject: [PATCH 184/252] Fix command. --- ognibuild/buildsystem.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 98278b5..26a696a 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1104,16 +1104,16 @@ class Maven(BuildSystem): return cls(os.path.join(path, "pom.xml")) def test(self, session, resolver, fixers): - run_with_build_fixers(session, ["maven", "test"], fixers) + run_with_build_fixers(session, ["mvn", "test"], fixers) def clean(self, session, resolver, fixers): - run_with_build_fixers(session, ["maven", "clean"], fixers) + run_with_build_fixers(session, ["mvn", "clean"], fixers) def install(self, session, resolver, fixers, install_target): - run_with_build_fixers(session, ["maven", "install"], fixers) + run_with_build_fixers(session, ["mvn", "install"], fixers) def build(self, session, resolver, fixers): - run_with_build_fixers(session, ["maven", "compile"], fixers) + run_with_build_fixers(session, ["mvn", "compile"], fixers) def dist(self, session, resolver, fixers): # TODO(jelmer): 'mvn generate-sources' creates a jar in target/. From 669b84ed8aba0b8e4e48f1edccdbbc52eb812a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:34:28 +0000 Subject: [PATCH 185/252] Handle mvn. --- ognibuild/debian/file_search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index dda9dc6..ead9909 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -274,6 +274,8 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( "/usr/bin/rst2html": "python3-docutils", # aclocal is a symlink to aclocal-1.XY "/usr/bin/aclocal": "automake", + # maven lives in /usr/share + "/usr/bin/mvn": "maven", } ) From 22baa23f5270b2fa90cd39a21f8552d94555a603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 17:38:00 +0000 Subject: [PATCH 186/252] Support quiet. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 26a696a..f47b83c 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1115,7 +1115,7 @@ class Maven(BuildSystem): def build(self, session, resolver, fixers): run_with_build_fixers(session, ["mvn", "compile"], fixers) - def dist(self, session, resolver, fixers): + def dist(self, session, resolver, fixers, quiet=False): # TODO(jelmer): 'mvn generate-sources' creates a jar in target/. # is that what we need? raise NotImplementedError From 4a08e54d15588f226e77b362c39ec1f4df0049f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 19:06:06 +0000 Subject: [PATCH 187/252] Add repr. --- ognibuild/debian/build_deps.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index 761b27f..5a5f811 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -27,6 +27,9 @@ class BuildDependencyTieBreaker(object): self.rootdir = rootdir self._counts = None + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.rootdir) + @classmethod def from_session(cls, session): return cls(session.location) From ef1d684b653a5d2cb88e075ba2b42da81e30f520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 20:33:07 +0000 Subject: [PATCH 188/252] Add SecretGpgKeyFixer. --- ognibuild/fixers.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index 68764f5..c5ed6db 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -21,6 +21,7 @@ from typing import Tuple from buildlog_consultant import Problem from buildlog_consultant.common import ( MissingGitIdentity, + MissingSecretGpgKey, ) from .fix_build import BuildFixer @@ -41,3 +42,31 @@ class GitIdentityFixer(BuildFixer): self.session.check_call( ['git', 'config', '--global', name, value]) return True + + +class SecretGpgKeyFixer(BuildFixer): + + def __init__(self, session): + self.session = session + + def can_fix(self, problem: Problem): + return isinstance(problem, MissingSecretGpgKey) + + def _fix(self, problem: Problem, phase: Tuple[str, ...]): + SCRIPT = b"""\ +Key-Type: 1 +Key-Length: 4096 +Subkey-Type: 1 +Subkey-Length: 4096 +Name-Real: Dummy Key for ognibuild +Name-Email: dummy@example.com +Expire-Date: 0 +Passphrase: "" +""" + p = self.session.Popen( + ['gpg', '--gen-key', '--batch', '/dev/stdin'], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + p.communicate(SCRIPT) + if p.returncode == 0: + return True + return False From 3e41558f91c8a9725fe481241e06a124a395a7ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 20:36:17 +0000 Subject: [PATCH 189/252] Set is_temporary. --- ognibuild/session/__init__.py | 2 ++ ognibuild/session/plain.py | 2 ++ ognibuild/session/schroot.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 320cd64..801c125 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -100,6 +100,8 @@ class Session(object): def external_path(self, path: str) -> str: raise NotImplementedError + is_temporary: bool + class SessionSetupFailure(Exception): """Session failed to be set up.""" diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index c7507f1..8d9de34 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -112,3 +112,5 @@ class PlainSession(Session): def setup_from_directory(self, path): return path, path + + is_temporary = False diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 2faeeb6..10fc9ca 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -215,3 +215,5 @@ class SchrootSession(Session): export_directory = os.path.join(directory, subdir) shutil.copytree(path, export_directory, dirs_exist_ok=True) return export_directory, os.path.join(reldir, subdir) + + is_temporary = True From 5decae132cbaee5fae16758e54a8f4216fbb2f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 20:36:34 +0000 Subject: [PATCH 190/252] Support running runtests.sh. --- ognibuild/buildsystem.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f47b83c..c6d7696 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -847,6 +847,28 @@ class DistZilla(BuildSystem): yield "build", PerlModuleRequirement(entry.decode().strip()) +class RunTests(BuildSystem): + + name = "runtests" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "runtests.sh")): + return cls(path) + + def test(self, session, resolver, fixers): + if shebang_binary(os.path.join(self.path, "runtests.sh")) is not None: + run_with_build_fixers(session, ["./runtests.sh"], fixers) + else: + run_with_build_fixers(session, ["/bin/bash", "./runtests.sh"], fixers) + + class Make(BuildSystem): name = "make" @@ -1198,7 +1220,7 @@ BUILDSYSTEM_CLSES = [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, DistZilla, Gem, PerlBuildTiny, Golang, R, Octave, # Make is intentionally at the end of the list. - Make] + Make, RunTests] def scan_buildsystems(path): From 65036278c0e98430b6505a0c99af6366bfdecb61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 20:36:49 +0000 Subject: [PATCH 191/252] Add secretgpgkeyfixer. --- ognibuild/dist.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 01a5b5d..308eb30 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -28,7 +28,7 @@ import os import shutil import sys import time -from typing import Optional +from typing import Optional, List from debian.deb822 import Deb822 @@ -152,7 +152,8 @@ def create_dist( ) -> Optional[str]: from .buildsystem import detect_buildsystems from .buildlog import InstallFixer - from .fixers import GitIdentityFixer + from .fix_build import BuildFixer + from .fixers import GitIdentityFixer, SecretGpgKeyFixer if subdir is None: subdir = "package" @@ -167,7 +168,12 @@ def create_dist( # TODO(jelmer): use scan_buildsystems to also look in subdirectories buildsystems = list(detect_buildsystems(export_directory)) resolver = auto_resolver(session) - fixers = [InstallFixer(resolver), GitIdentityFixer(session)] + fixers: List[BuildFixer] = [InstallFixer(resolver)] + + if session.is_temporary: + # Only muck about with temporary sessions + fixers.extend([ + GitIdentityFixer(session), SecretGpgKeyFixer(session)]) with DistCatcher(export_directory) as dc: session.chdir(reldir) From 072b845a5e217fe487d5cf82332e5e387361e7a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 24 Mar 2021 21:27:22 +0000 Subject: [PATCH 192/252] Support libtool. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 6 ++++++ ognibuild/resolver/apt.py | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 5cdbc25..0cd358c 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -54,6 +54,7 @@ from buildlog_consultant.common import ( GnomeCommonMissing, MissingGnomeCommonDependency, UnknownCertificateAuthority, + MissingLibtool, MissingQt, ) @@ -90,6 +91,7 @@ from .requirements import ( CertificateAuthorityRequirement, NodeModuleRequirement, QTRequirement, + LibtoolRequirement, ) from .resolver import UnsatisfiedRequirements @@ -147,6 +149,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return JRERequirement() elif isinstance(problem, MissingQt): return QTRequirement() + elif isinstance(problem, MissingLibtool): + return LibtoolRequirement() elif isinstance(problem, UnknownCertificateAuthority): return CertificateAuthorityRequirement(problem.url) elif isinstance(problem, MissingGnomeCommonDependency): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index bdd3901..ddebd95 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -459,6 +459,12 @@ class AutoconfMacroRequirement(Requirement): self.macro = macro +class LibtoolRequirement(Requirement): + + def __init__(self): + super(LibtoolRequirement, self).__init__("libtool") + + class PythonModuleRequirement(Requirement): module: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 14535d1..76a534a 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -62,6 +62,7 @@ from ..requirements import ( PythonModuleRequirement, PythonPackageRequirement, CertificateAuthorityRequirement, + LibtoolRequirement, ) @@ -440,6 +441,10 @@ def resolve_qt_req(apt_mgr, req): return find_reqs_simple(apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) +def resolve_libtool_req(apt_mgr, req): + return [AptRequirement.simple("libtool")] + + def resolve_perl_module_req(apt_mgr, req): DEFAULT_PERL_PATHS = ["/usr/share/perl5"] @@ -545,6 +550,7 @@ APT_REQUIREMENT_RESOLVERS = [ (JDKRequirement, resolve_jdk_req), (JRERequirement, resolve_jre_req), (QTRequirement, resolve_qt_req), + (LibtoolRequirement, resolve_libtool_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), (AutoconfMacroRequirement, resolve_autoconf_macro_req), From 2ce79cc89308f1c8a5180dc51fa9429204058e37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 12:21:03 +0000 Subject: [PATCH 193/252] Handle vague dependencies. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 0cd358c..af412f4 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -49,6 +49,7 @@ from buildlog_consultant.common import ( MissingValaPackage, MissingXfceDependency, MissingHaskellDependencies, + MissingVagueDependency, DhAddonLoadFailure, MissingMavenArtifacts, GnomeCommonMissing, @@ -92,6 +93,7 @@ from .requirements import ( NodeModuleRequirement, QTRequirement, LibtoolRequirement, + VagueDependencyRequirement, ) from .resolver import UnsatisfiedRequirements @@ -123,6 +125,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return NodeModuleRequirement(problem.module) elif isinstance(problem, MissingNodePackage): return NodePackageRequirement(problem.package) + elif isinstance(problem, MissingVagueDependency): + return VagueDependencyRequirement(problem.name) elif isinstance(problem, MissingLibrary): return LibraryRequirement(problem.library) elif isinstance(problem, MissingRubyFile): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index ddebd95..055c3c8 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -119,6 +119,18 @@ class PerlModuleRequirement(Requirement): return "%s(%r)" % (type(self).__name__, self.module) +class VagueDependencyRequirement(Requirement): + + name: str + + def __init__(self, name): + super(VagueDependencyRequirement, self).__init__("vague") + self.name = name + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.name) + + class NodePackageRequirement(Requirement): package: str From 3bb27504769e39eb075b1903f22fd6b7d8ccac2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 12:23:56 +0000 Subject: [PATCH 194/252] Handle vague dependencies. --- ognibuild/resolver/apt.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 76a534a..aee5b6e 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -63,6 +63,7 @@ from ..requirements import ( PythonPackageRequirement, CertificateAuthorityRequirement, LibtoolRequirement, + VagueDependencyRequirement, ) @@ -228,6 +229,16 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] +def resolve_vague_dep_req(apt_mgr, req): + name = req.name + if name.startswith('gnu '): + name = name[4:] + options = [] + options.extend(resolve_binary_req(apt_mgr, [BinaryRequirement(name)])) + options.extend(resolve_binary_req(apt_mgr, [LibraryRequirement(name)])) + return options + + def resolve_binary_req(apt_mgr, req): if posixpath.isabs(req.binary_name): paths = [req.binary_name] From e062075cb78955139d7b662a1fa498f13fb0bfe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 14:34:29 +0000 Subject: [PATCH 195/252] Fix handling of vague requirements. --- ognibuild/requirements.py | 6 ++++++ ognibuild/resolver/apt.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 055c3c8..313bdfd 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -127,6 +127,12 @@ class VagueDependencyRequirement(Requirement): super(VagueDependencyRequirement, self).__init__("vague") self.name = name + def met(self, session): + for cls in [BinaryRequirement, LibraryRequirement]: + if cls(self.name).met(session): + return True + return False + def __repr__(self): return "%s(%r)" % (type(self).__name__, self.name) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index aee5b6e..7667347 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -235,7 +235,7 @@ def resolve_vague_dep_req(apt_mgr, req): name = name[4:] options = [] options.extend(resolve_binary_req(apt_mgr, [BinaryRequirement(name)])) - options.extend(resolve_binary_req(apt_mgr, [LibraryRequirement(name)])) + options.extend(resolve_library_req(apt_mgr, [LibraryRequirement(name)])) return options @@ -537,6 +537,7 @@ def resolve_ca_req(apt_mgr, req): APT_REQUIREMENT_RESOLVERS = [ (BinaryRequirement, resolve_binary_req), + (VagueDependencyRequirement, resolve_vague_dep_req), (PkgConfigRequirement, resolve_pkg_config_req), (PathRequirement, resolve_path_req), (CHeaderRequirement, resolve_c_header_req), From 3ce2df8f6e489b3eaeb5f172d8c54db23608910e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 16:02:47 +0000 Subject: [PATCH 196/252] Always do dist with setuptools. --- ognibuild/buildsystem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index c6d7696..7db32a4 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -342,6 +342,9 @@ class SetupPy(BuildSystem): preargs = [] if quiet: preargs.append("--quiet") + # Preemptively install setuptools since some packages fail in + # some way without it. + resolver.install([PythonPackageRequirement('setuptools')]) self._run_setup(session, resolver, preargs + ["sdist"], fixers) return elif self.pyproject: From 609b8b219603678fe21081070cabdcd82764dc0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 16:03:00 +0000 Subject: [PATCH 197/252] Fix vague requirement handling/ --- ognibuild/requirements.py | 10 ++++++++-- ognibuild/resolver/apt.py | 9 +++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 313bdfd..92c30f4 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -127,9 +127,15 @@ class VagueDependencyRequirement(Requirement): super(VagueDependencyRequirement, self).__init__("vague") self.name = name + def expand(self): + yield BinaryRequirement(self.name) + yield LibraryRequirement(self.name) + from resolver.apt import AptRequirement + yield AptRequirement(self.name) + def met(self, session): - for cls in [BinaryRequirement, LibraryRequirement]: - if cls(self.name).met(session): + for x in self.expand(): + if x.met(session): return True return False diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 7667347..0b80e44 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -234,8 +234,8 @@ def resolve_vague_dep_req(apt_mgr, req): if name.startswith('gnu '): name = name[4:] options = [] - options.extend(resolve_binary_req(apt_mgr, [BinaryRequirement(name)])) - options.extend(resolve_library_req(apt_mgr, [LibraryRequirement(name)])) + for x in req.expand(): + options.extend(resolve_requirement_apt(apt_mgr, x)) return options @@ -535,7 +535,12 @@ def resolve_ca_req(apt_mgr, req): return [AptRequirement.simple('ca-certificates')] +def resolve_apt_req(apt_mgr, req): + return [req] + + APT_REQUIREMENT_RESOLVERS = [ + (AptRequirement, resolve_apt_req), (BinaryRequirement, resolve_binary_req), (VagueDependencyRequirement, resolve_vague_dep_req), (PkgConfigRequirement, resolve_pkg_config_req), From ac1f25d0be46b63fd83da976f022edf37b2e985b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 16:09:45 +0000 Subject: [PATCH 198/252] Handle case sensitivity. --- ognibuild/debian/apt.py | 4 ++-- ognibuild/debian/file_search.py | 27 ++++++++++++++++++++------- ognibuild/resolver/apt.py | 13 ++++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 17875ee..c435b56 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -81,10 +81,10 @@ class AptManager(object): self._apt_cache = apt.Cache(rootdir=self.session.location) return package in self._apt_cache - def get_packages_for_paths(self, paths, regex=False): + def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): logging.debug("Searching for packages containing %r", paths) # TODO(jelmer): Make sure we use whatever is configured in self.session - return get_packages_for_paths(paths, self.searchers(), regex=regex) + return get_packages_for_paths(paths, self.searchers(), regex=regex, case_insensitive=case_insensitive) def missing(self, packages): root = getattr(self.session, "location", "/") diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index ead9909..21c570f 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -29,7 +29,7 @@ from .. import USER_AGENT class FileSearcher(object): - def search_files(self, path: str, regex: bool = False) -> Iterator[str]: + def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]: raise NotImplementedError(self.search_files) @@ -216,10 +216,16 @@ class AptCachedContentsFileSearcher(FileSearcher): def __setitem__(self, path, package): self._db[path] = package - def search_files(self, path, regex=False): + def search_files(self, path, regex=False, case_insensitive=False): path = path.lstrip('/').encode('utf-8', 'surrogateescape') + if case_insensitive and not regex: + regex = True + path = re.escape(path) if regex: - c = re.compile(path) + flags = 0 + if case_insensitive: + flags |= re.I + c = re.compile(path, flags=flags) ret = [] for p, rest in self._db.items(): if c.match(p): @@ -256,10 +262,16 @@ class GeneratedFileSearcher(FileSearcher): (path, pkg) = line.strip().split(None, 1) self._db[path] = pkg - def search_files(self, path: str, regex: bool = False) -> Iterator[str]: + def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]: for p, pkg in sorted(self._db.items()): if regex: - if re.match(path, p): + flags = 0 + if case_insensitive: + flags |= re.I + if re.match(path, p, flags=flags): + yield pkg + elif case_insensitive: + if path.lower() == p.lower(): yield pkg else: if path == p: @@ -281,12 +293,13 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( def get_packages_for_paths( - paths: List[str], searchers: List[FileSearcher], regex: bool = False + paths: List[str], searchers: List[FileSearcher], regex: bool = False, + case_insensitive: bool = False ) -> List[str]: candidates: List[str] = list() for path in paths: for searcher in searchers: - for pkg in searcher.search_files(path, regex=regex): + for pkg in searcher.search_files(path, regex=regex, case_insensitive=case_insensitive): if pkg not in candidates: candidates.append(pkg) return candidates diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 0b80e44..da332af 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -110,16 +110,19 @@ class AptRequirement(Requirement): return False -def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False) -> List[str]: +def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) - return apt_mgr.get_packages_for_paths(paths, regex) + return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) -def find_reqs_simple(apt_mgr: AptManager, paths: List[str], regex: bool = False, minimum_version=None) -> List[str]: +def find_reqs_simple( + apt_mgr: AptManager, paths: List[str], regex: bool = False, + minimum_version=None, case_insensitive=False) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) - return [AptRequirement.simple(package, minimum_version=minimum_version) for package in find_package_names(apt_mgr, paths, regex)] + return [AptRequirement.simple(package, minimum_version=minimum_version) + for package in find_package_names(apt_mgr, paths, regex, case_insensitive)] def python_spec_to_apt_rels(pkg_name, specs): @@ -163,7 +166,7 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st paths = [cpython3_regex, cpython2_regex, pypy_regex] else: raise NotImplementedError('unsupported python version %s' % python_version) - names = find_package_names(apt_mgr, paths, regex=True) + names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True) return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] From a4bcbeef406d3cdfcbe93b23d3b1f4f78b79e660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 16:37:45 +0000 Subject: [PATCH 199/252] Handle x11. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 6 ++++++ ognibuild/resolver/apt.py | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index af412f4..e8fcc7a 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -57,6 +57,7 @@ from buildlog_consultant.common import ( UnknownCertificateAuthority, MissingLibtool, MissingQt, + MissingX11, ) from .fix_build import BuildFixer @@ -92,6 +93,7 @@ from .requirements import ( CertificateAuthorityRequirement, NodeModuleRequirement, QTRequirement, + X11Requirement, LibtoolRequirement, VagueDependencyRequirement, ) @@ -153,6 +155,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return JRERequirement() elif isinstance(problem, MissingQt): return QTRequirement() + elif isinstance(problem, MissingX11): + return X11Requirement() elif isinstance(problem, MissingLibtool): return LibtoolRequirement() elif isinstance(problem, UnknownCertificateAuthority): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 92c30f4..1e37a4d 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -458,6 +458,12 @@ class QTRequirement(Requirement): super(QTRequirement, self).__init__("qt") +class X11Requirement(Requirement): + + def __init__(self): + super(X11Requirement, self).__init__("x11") + + class CertificateAuthorityRequirement(Requirement): def __init__(self, url): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index da332af..6b6f1d2 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -56,6 +56,7 @@ from ..requirements import ( JDKRequirement, JRERequirement, QTRequirement, + X11Requirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, @@ -451,6 +452,10 @@ def resolve_jre_req(apt_mgr, req): return [AptRequirement.simple('default-jre')] +def resolve_x11_req(apt_mgr, req): + return [AptRequirement.simple('libx11-dev')] + + def resolve_qt_req(apt_mgr, req): return find_reqs_simple(apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) @@ -570,6 +575,7 @@ APT_REQUIREMENT_RESOLVERS = [ (JDKRequirement, resolve_jdk_req), (JRERequirement, resolve_jre_req), (QTRequirement, resolve_qt_req), + (X11Requirement, resolve_x11_req), (LibtoolRequirement, resolve_libtool_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), From f2c3f252a9980045ceb69ffa2fd4044321b98788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 17:15:38 +0000 Subject: [PATCH 200/252] Add support for package.json dependencies. --- ognibuild/buildsystem.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 7db32a4..0d6114c 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -692,10 +692,14 @@ class Npm(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def get_declared_dependencies(self, session, fixers=None): + for 'dependencies' in self.package: + for name, unused_version in self.package["dependencies"].items(): + # TODO(jelmer): Look at version + yield "core", NodePackageRequirement(name) if "devDependencies" in self.package: for name, unused_version in self.package["devDependencies"].items(): # TODO(jelmer): Look at version - yield "dev", NodePackageRequirement(name) + yield "build", NodePackageRequirement(name) def setup(self, resolver): resolver.install([BinaryRequirement("npm")]) From a3978aea48349ce46390172c3d9a466ca0a83b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 17:35:36 +0000 Subject: [PATCH 201/252] Refactor DistCatcher. --- ognibuild/dist.py | 83 +++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 38 deletions(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 308eb30..f42ff0d 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -83,52 +83,59 @@ def run_dist(session, buildsystems, resolver, fixers, quiet=False): class DistCatcher(object): - def __init__(self, directory): - self.export_directory = directory + def __init__(self, directories): + self.directories = directories self.files = [] self.existing_files = None self.start_time = time.time() + @classmethod + def default(cls, directory): + return cls([ + os.path.join(directory, 'dist'), + directory, + os.path.join(directory, '..')]) + def __enter__(self): - self.existing_files = os.listdir(self.export_directory) + self.existing_files = {} + for directory in self.directories: + try: + self.existing_files[directory] = { + entry.name: entry for entry in os.scandir(directory)} + except FileNotFoundError: + self.existing_files[directory] = {} return self def find_files(self): - new_files = os.listdir(self.export_directory) - diff_files = set(new_files) - set(self.existing_files) - diff = set([n for n in diff_files if is_dist_file(n)]) - if len(diff) == 1: - fn = diff.pop() - logging.info("Found tarball %s in package directory.", fn) - self.files.append(os.path.join(self.export_directory, fn)) - return fn - if "dist" in diff_files: - for entry in os.scandir(os.path.join(self.export_directory, "dist")): - if is_dist_file(entry.name): - logging.info("Found tarball %s in dist directory.", entry.name) - self.files.append(entry.path) - return entry.name - logging.info("No tarballs found in dist directory.") + for directory in self.directories: + old_files = self.existing_files[directory] + possible_new = [] + possible_updated = [] + for entry in os.scandir(directory): + if not entry.is_file() or not is_dist_file(entry.name): + continue + old_entry = old_files.get(entry.name) + if not old_entry: + possible_new.append(entry) + continue + if entry.stat().st_mtime < self.start_time: + possible_updated.append(entry) + continue + if len(possible_new) == 1: + fn = possible_new[0] + logging.info("Found new tarball %s in %s.", fn, directory) + self.files.append(os.path.join(directory, fn)) + return entry.name + elif len(possible_new) > 1: + logging.warning( + "Found multiple tarballs %r in %s.", possible_new, directory) + return - parent_directory = os.path.dirname(self.export_directory) - diff = (set(os.listdir(parent_directory)) - - set([os.path.basename(self.export_directory)])) - if len(diff) == 1: - fn = diff.pop() - if is_dist_file(fn): - logging.info("Found tarball %s in parent directory.", fn) - self.files.append(os.path.join(parent_directory, fn)) - return fn - logging.warning( - "Found file %s in parent directory, " - "but not in supported dist format", fn) - - if "dist" in new_files: - for entry in os.scandir(os.path.join(self.export_directory, "dist")): - if is_dist_file(entry.name) and entry.stat().st_mtime > self.start_time: - logging.info("Found tarball %s in dist directory.", entry.name) - self.files.append(entry.path) - return entry.name + if len(possible_updated) == 1: + fn = possible_updated[0] + logging.info("Found updated tarball %s in %s.", fn, directory) + self.files.append(os.path.join(directory, fn)) + return entry.name def __exit__(self, exc_type, exc_val, exc_tb): self.find_files() @@ -175,7 +182,7 @@ def create_dist( fixers.extend([ GitIdentityFixer(session), SecretGpgKeyFixer(session)]) - with DistCatcher(export_directory) as dc: + with DistCatcher.default(export_directory) as dc: session.chdir(reldir) run_dist(session, buildsystems, resolver, fixers) From fbeb1f80ce90f9ae3b426bfe1115d707e503805c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 17:55:27 +0000 Subject: [PATCH 202/252] Fix syntax. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 0d6114c..279b1c6 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -692,7 +692,7 @@ class Npm(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def get_declared_dependencies(self, session, fixers=None): - for 'dependencies' in self.package: + if 'dependencies' in self.package: for name, unused_version in self.package["dependencies"].items(): # TODO(jelmer): Look at version yield "core", NodePackageRequirement(name) From 16c8718817377f6d619c08950901148e06a70914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 18:14:01 +0000 Subject: [PATCH 203/252] Support more complex dist catching. --- ognibuild/__main__.py | 3 +- ognibuild/buildsystem.py | 177 ++++++++++++++++++++++---------------- ognibuild/dist.py | 115 ++----------------------- ognibuild/dist_catcher.py | 116 +++++++++++++++++++++++++ 4 files changed, 227 insertions(+), 184 deletions(-) create mode 100644 ognibuild/dist_catcher.py diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 1e51a60..6d6d430 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -170,7 +170,8 @@ def main(): # noqa: C901 from .dist import run_dist run_dist( - session=session, buildsystems=bss, resolver=resolver, fixers=fixers + session=session, buildsystems=bss, resolver=resolver, fixers=fixers, + target_directory='.' ) if args.subcommand == "build": from .build import run_build diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 279b1c6..71e44c5 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -26,6 +26,7 @@ from typing import Optional, Tuple import warnings from . import shebang_binary, UnidentifiedError +from .dist_catcher import DistCatcher from .outputs import ( BinaryOutput, PythonPackageOutput, @@ -71,7 +72,7 @@ class BuildSystem(object): def __str__(self): return self.name - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory: str, quiet=False) -> str: raise NotImplementedError(self.dist) def test(self, session, resolver, fixers): @@ -107,9 +108,11 @@ class Pear(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("pear")]) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory: str, quiet=False): self.setup(resolver) - run_with_build_fixers(session, ["pear", "package"], fixers) + with DistCatcher([session.external_path('.')]) as dc: + run_with_build_fixers(session, ["pear", "package"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(resolver) @@ -336,7 +339,7 @@ class SetupPy(BuildSystem): else: raise NotImplementedError - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): # TODO(jelmer): Look at self.build_backend if self.has_setup_py: preargs = [] @@ -345,11 +348,13 @@ class SetupPy(BuildSystem): # Preemptively install setuptools since some packages fail in # some way without it. resolver.install([PythonPackageRequirement('setuptools')]) - self._run_setup(session, resolver, preargs + ["sdist"], fixers) - return + with DistCatcher([session.external_path('dist')]) as dc: + self._run_setup(session, resolver, preargs + ["sdist"], fixers) + return dc.copy_single(target_directory) elif self.pyproject: - run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], fixers) - return + with DistCatcher([session.external_path('dist')]) as dc: + run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], fixers) + return dc.copy_single(target_directory) raise AssertionError("no setup.py or pyproject.toml") def clean(self, session, resolver, fixers): @@ -566,8 +571,10 @@ class Gradle(BuildSystem): def test(self, session, resolver, fixers): self._run(session, resolver, 'test', [], fixers) - def dist(self, session, resolver, fixers, quiet=False): - self._run(session, resolver, 'distTar', [], fixers) + def dist(self, session, resolver, fixers, target_directory, quiet=False): + with DistCatcher([session.external_path('.')]) as dc: + self._run(session, resolver, 'distTar', [], fixers) + return dc.copy_single(target_directory) def install(self, session, resolver, fixers, install_target): raise NotImplementedError @@ -590,9 +597,11 @@ class R(BuildSystem): def build(self, session, resolver, fixers): pass - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): r_path = guaranteed_which(session, resolver, "R") - run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) + with DistCatcher([session.external_path('.')]) as dc: + run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) + return dc.copy_single(target_directory) def install(self, session, resolver, fixers, install_target): r_path = guaranteed_which(session, resolver, "R") @@ -657,9 +666,11 @@ class Meson(BuildSystem): self._setup(session, fixers) run_with_build_fixers(session, ["ninja", "-C", "build"], fixers) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self._setup(session, fixers) - run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) + with DistCatcher([session.external_path('build/meson-dist')]) as dc: + run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self._setup(session, fixers) @@ -704,9 +715,11 @@ class Npm(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("npm")]) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(resolver) - run_with_build_fixers(session, ["npm", "pack"], fixers) + with DistCatcher([session.external_path('.')]) as dc: + run_with_build_fixers(session, ["npm", "pack"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(resolver) @@ -749,9 +762,11 @@ class Waf(BuildSystem): def setup(self, session, resolver, fixers): resolver.install([BinaryRequirement("python3")]) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) - run_with_build_fixers(session, ["./waf", "dist"], fixers) + with DistCatcher.default(session.external_path('.')) as dc: + run_with_build_fixers(session, ["./waf", "dist"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(session, resolver, fixers) @@ -774,14 +789,16 @@ class Gem(BuildSystem): def setup(self, resolver): resolver.install([BinaryRequirement("gem2deb")]) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(resolver) gemfiles = [ entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") ] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") - run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) + with DistCatcher.default(session.external_path('.')) as dc: + run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) + return dc.copy_single(target_directory) @classmethod def probe(cls, path): @@ -821,15 +838,19 @@ class DistZilla(BuildSystem): ] ) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(resolver) if self.name == "dist-inkt": resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) - run_with_build_fixers(session, ["distinkt-dist"], fixers) + with DistCatcher.default(session.external_path('.')) as dc: + run_with_build_fixers(session, ["distinkt-dist"], fixers) + return dc.copy_single(target_directory) else: # Default to invoking Dist::Zilla resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers) + with DistCatcher.default(session.external_path('.')) as dc: + run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers) + return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(resolver) @@ -944,57 +965,59 @@ class Make(BuildSystem): self.setup(session, resolver, fixers) run_with_build_fixers(session, ["make", "install"], fixers) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) - try: - run_with_build_fixers(session, ["make", "dist"], fixers) - except UnidentifiedError as e: - if "make: *** No rule to make target 'dist'. Stop." in e.lines: - raise NotImplementedError - elif "make[1]: *** No rule to make target 'dist'. Stop." in e.lines: - raise NotImplementedError - elif ( - "Reconfigure the source tree " - "(via './config' or 'perl Configure'), please." - ) in e.lines: - run_with_build_fixers(session, ["./config"], fixers) + with DistCatcher.default(session.external_path('.')) as dc: + try: run_with_build_fixers(session, ["make", "dist"], fixers) - elif ( - "Please try running 'make manifest' and then run " - "'make dist' again." in e.lines - ): - run_with_build_fixers(session, ["make", "manifest"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - elif "Please run ./configure first" in e.lines: - run_with_build_fixers(session, ["./configure"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - elif any( - [ - re.match( - r"(Makefile|GNUmakefile|makefile):[0-9]+: " - r"\*\*\* Missing \'Make.inc\' " - r"Run \'./configure \[options\]\' and retry. Stop.", - line, - ) - for line in e.lines - ] - ): - run_with_build_fixers(session, ["./configure"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - elif any( - [ - re.match( - r"Problem opening MANIFEST: No such file or directory " - r"at .* line [0-9]+\.", - line, - ) - for line in e.lines - ] - ): - run_with_build_fixers(session, ["make", "manifest"], fixers) - run_with_build_fixers(session, ["make", "dist"], fixers) - else: - raise + except UnidentifiedError as e: + if "make: *** No rule to make target 'dist'. Stop." in e.lines: + raise NotImplementedError + elif "make[1]: *** No rule to make target 'dist'. Stop." in e.lines: + raise NotImplementedError + elif ( + "Reconfigure the source tree " + "(via './config' or 'perl Configure'), please." + ) in e.lines: + run_with_build_fixers(session, ["./config"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif ( + "Please try running 'make manifest' and then run " + "'make dist' again." in e.lines + ): + run_with_build_fixers(session, ["make", "manifest"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif "Please run ./configure first" in e.lines: + run_with_build_fixers(session, ["./configure"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif any( + [ + re.match( + r"(Makefile|GNUmakefile|makefile):[0-9]+: " + r"\*\*\* Missing \'Make.inc\' " + r"Run \'./configure \[options\]\' and retry. Stop.", + line, + ) + for line in e.lines + ] + ): + run_with_build_fixers(session, ["./configure"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + elif any( + [ + re.match( + r"Problem opening MANIFEST: No such file or directory " + r"at .* line [0-9]+\.", + line, + ) + for line in e.lines + ] + ): + run_with_build_fixers(session, ["make", "manifest"], fixers) + run_with_build_fixers(session, ["make", "dist"], fixers) + else: + raise + return dc.copy_single(target_directory) def get_declared_dependencies(self, session, fixers=None): # TODO(jelmer): Split out the perl-specific stuff? @@ -1144,7 +1167,7 @@ class Maven(BuildSystem): def build(self, session, resolver, fixers): run_with_build_fixers(session, ["mvn", "compile"], fixers) - def dist(self, session, resolver, fixers, quiet=False): + def dist(self, session, resolver, fixers, target_directory, quiet=False): # TODO(jelmer): 'mvn generate-sources' creates a jar in target/. # is that what we need? raise NotImplementedError @@ -1177,8 +1200,12 @@ class Cabal(BuildSystem): def test(self, session, resolver, fixers): self._run(session, ["test"], fixers) - def dist(self, session, resolver, fixers, quiet=False): - self._run(session, ["sdist"], fixers) + def dist(self, session, resolver, fixers, target_directory, quiet=False): + with DistCatcher([ + session.external_path('dist-newstyle/sdist'), + session.external_path('dist')]) as dc: + self._run(session, ["sdist"], fixers) + return dc.copy_single(target_directory) @classmethod def probe(cls, path): diff --git a/ognibuild/dist.py b/ognibuild/dist.py index f42ff0d..085a3a2 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -27,7 +27,6 @@ import logging import os import shutil import sys -import time from typing import Optional, List from debian.deb822 import Deb822 @@ -47,108 +46,19 @@ from .session import Session from .session.schroot import SchrootSession -SUPPORTED_DIST_EXTENSIONS = [ - ".tar.gz", - ".tgz", - ".tar.bz2", - ".tar.xz", - ".tar.lzma", - ".tbz2", - ".tar", - ".zip", -] - - -def is_dist_file(fn): - for ext in SUPPORTED_DIST_EXTENSIONS: - if fn.endswith(ext): - return True - return False - - -class DistNoTarball(Exception): - """Dist operation did not create a tarball.""" - - -def run_dist(session, buildsystems, resolver, fixers, quiet=False): +def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=False): # Some things want to write to the user's home directory, # e.g. pip caches in ~/.cache session.create_home() for buildsystem in buildsystems: - buildsystem.dist(session, resolver, fixers, quiet=quiet) - return + filename = buildsystem.dist( + session, resolver, fixers, target_directory, quiet=quiet) + return filename raise NoBuildToolsFound() -class DistCatcher(object): - def __init__(self, directories): - self.directories = directories - self.files = [] - self.existing_files = None - self.start_time = time.time() - - @classmethod - def default(cls, directory): - return cls([ - os.path.join(directory, 'dist'), - directory, - os.path.join(directory, '..')]) - - def __enter__(self): - self.existing_files = {} - for directory in self.directories: - try: - self.existing_files[directory] = { - entry.name: entry for entry in os.scandir(directory)} - except FileNotFoundError: - self.existing_files[directory] = {} - return self - - def find_files(self): - for directory in self.directories: - old_files = self.existing_files[directory] - possible_new = [] - possible_updated = [] - for entry in os.scandir(directory): - if not entry.is_file() or not is_dist_file(entry.name): - continue - old_entry = old_files.get(entry.name) - if not old_entry: - possible_new.append(entry) - continue - if entry.stat().st_mtime < self.start_time: - possible_updated.append(entry) - continue - if len(possible_new) == 1: - fn = possible_new[0] - logging.info("Found new tarball %s in %s.", fn, directory) - self.files.append(os.path.join(directory, fn)) - return entry.name - elif len(possible_new) > 1: - logging.warning( - "Found multiple tarballs %r in %s.", possible_new, directory) - return - - if len(possible_updated) == 1: - fn = possible_updated[0] - logging.info("Found updated tarball %s in %s.", fn, directory) - self.files.append(os.path.join(directory, fn)) - return entry.name - - def __exit__(self, exc_type, exc_val, exc_tb): - self.find_files() - return False - - def cleanup(self): - for path in self.files: - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.unlink(path) - - def create_dist( session: Session, tree: Tree, @@ -182,20 +92,8 @@ def create_dist( fixers.extend([ GitIdentityFixer(session), SecretGpgKeyFixer(session)]) - with DistCatcher.default(export_directory) as dc: - session.chdir(reldir) - run_dist(session, buildsystems, resolver, fixers) - - try: - for path in dc.files: - shutil.copy(path, target_dir) - return os.path.join(target_dir, os.path.basename(path)) - finally: - if cleanup: - dc.cleanup() - - logging.info("No tarball created :(") - raise DistNoTarball() + session.chdir(reldir) + return run_dist(session, buildsystems, resolver, fixers, target_dir) def create_dist_schroot( @@ -225,6 +123,7 @@ if __name__ == "__main__": import breezy.bzr # noqa: F401 import breezy.git # noqa: F401 from breezy.export import export + from .dist_catcher import DistNoTarball parser = argparse.ArgumentParser() parser.add_argument( diff --git a/ognibuild/dist_catcher.py b/ognibuild/dist_catcher.py new file mode 100644 index 0000000..d9d9bca --- /dev/null +++ b/ognibuild/dist_catcher.py @@ -0,0 +1,116 @@ +#!/usr/bin/python3 +# Copyright (C) 2020 Jelmer Vernooij +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import os +import logging +import shutil +import time + + +class DistNoTarball(Exception): + """Dist operation did not create a tarball.""" + + +SUPPORTED_DIST_EXTENSIONS = [ + ".tar.gz", + ".tgz", + ".tar.bz2", + ".tar.xz", + ".tar.lzma", + ".tbz2", + ".tar", + ".zip", +] + + +def is_dist_file(fn): + for ext in SUPPORTED_DIST_EXTENSIONS: + if fn.endswith(ext): + return True + return False + + +class DistCatcher(object): + def __init__(self, directories): + self.directories = [os.path.abspath(d) for d in directories] + self.files = [] + self.existing_files = None + self.start_time = time.time() + + @classmethod + def default(cls, directory): + return cls([ + os.path.join(directory, 'dist'), + directory, + os.path.join(directory, '..')]) + + def __enter__(self): + self.existing_files = {} + for directory in self.directories: + try: + self.existing_files[directory] = { + entry.name: entry for entry in os.scandir(directory)} + except FileNotFoundError: + self.existing_files[directory] = {} + return self + + def find_files(self): + for directory in self.directories: + old_files = self.existing_files[directory] + possible_new = [] + possible_updated = [] + if not os.path.isdir(directory): + continue + for entry in os.scandir(directory): + if not entry.is_file() or not is_dist_file(entry.name): + continue + old_entry = old_files.get(entry.name) + if not old_entry: + possible_new.append(entry) + continue + if entry.stat().st_mtime > self.start_time: + possible_updated.append(entry) + continue + if len(possible_new) == 1: + entry = possible_new[0] + logging.info("Found new tarball %s in %s.", entry.name, directory) + self.files.append(entry.path) + return entry.name + elif len(possible_new) > 1: + logging.warning( + "Found multiple tarballs %r in %s.", possible_new, directory) + return + + if len(possible_updated) == 1: + entry = possible_updated[0] + logging.info("Found updated tarball %s in %s.", entry.name, directory) + self.files.append(entry.path) + return entry.name + + def __exit__(self, exc_type, exc_val, exc_tb): + self.find_files() + return False + + def copy_single(self, target_dir): + for path in self.files: + try: + shutil.copy(path, target_dir) + except shutil.SameFileError: + pass + return os.path.basename(path) + logging.info("No tarball created :(") + raise DistNoTarball() From f75d4b6cb3e24efaa5a29a86ad744f20d5bd732e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 18:43:03 +0000 Subject: [PATCH 204/252] Move import to top-level. --- ognibuild/dist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index 085a3a2..b7c56c9 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -40,6 +40,7 @@ from buildlog_consultant.common import ( from . import DetailedFailure, UnidentifiedError +from .dist_catcher import DistNoTarball from .buildsystem import NoBuildToolsFound from .resolver import auto_resolver from .session import Session @@ -123,7 +124,6 @@ if __name__ == "__main__": import breezy.bzr # noqa: F401 import breezy.git # noqa: F401 from breezy.export import export - from .dist_catcher import DistNoTarball parser = argparse.ArgumentParser() parser.add_argument( From 0688a0cbbafe5beacdf8ba92289d79015419311a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 19:06:00 +0000 Subject: [PATCH 205/252] Add support for composer. --- ognibuild/buildsystem.py | 56 ++++++++++++++++++++++++++++++++++++++- ognibuild/fix_build.py | 27 ++++++++++++++----- ognibuild/requirements.py | 16 +++++++++++ ognibuild/resolver/apt.py | 6 +++++ 4 files changed, 97 insertions(+), 8 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 71e44c5..82d4992 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -40,6 +40,7 @@ from .requirements import ( CargoCrateRequirement, RPackageRequirement, OctavePackageRequirement, + PhpPackageRequirement, ) from .fix_build import run_with_build_fixers from .session import which @@ -98,6 +99,16 @@ class BuildSystem(object): return None +def xmlparse_simplify_namespaces(path, namespaces): + import xml.etree.ElementTree as ET + namespaces = ['{%s}' % ns for ns in namespaces] + tree = ET.iterparse(path) + for _, el in tree: + for namespace in namespaces: + el.tag = el.tag.replace(namespace, '') + return tree.root + + class Pear(BuildSystem): name = "pear" @@ -130,6 +141,32 @@ class Pear(BuildSystem): self.setup(resolver) run_with_build_fixers(session, ["pear", "install", self.path], fixers) + def get_declared_dependencies(self, session, fixers=None): + path = os.path.join(self.path, "package.xml") + import xml.etree.ElementTree as ET + try: + root = xmlparse_simplify_namespaces(path, [ + 'http://pear.php.net/dtd/package-2.0', + 'http://pear.php.net/dtd/package-2.1']) + except ET.ParseError as e: + logging.warning('Unable to parse package.xml: %s', e) + return + assert root.tag == 'package', 'root tag is %r' % root.tag + dependencies_tag = root.find('dependencies') + if dependencies_tag is not None: + required_tag = root.find('dependencies') + if required_tag is not None: + for package_tag in root.findall('package'): + name = package_tag.find('name').text + min_tag = package_tag.find('min') + max_tag = package_tag.find('max') + channel_tag = package_tag.find('channel') + yield "core", PhpPackageRequirement( + name, + channel=(channel_tag.text if channel_tag else None), + min_version=(min_tag.text if min_tag else None), + max_version=(max_tag.text if max_tag else None)) + @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, "package.xml")): @@ -1214,6 +1251,23 @@ class Cabal(BuildSystem): return cls(os.path.join(path, "Setup.hs")) +class Composer(BuildSystem): + + name = "composer" + + def __init__(self, path): + self.path = path + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + + @classmethod + def probe(cls, path): + if os.path.exists(os.path.join(path, "composer.json")): + logging.debug("Found composer.json, assuming composer package.") + return cls(path) + + class PerlBuildTiny(BuildSystem): name = "perl-build-tiny" @@ -1254,7 +1308,7 @@ BUILDSYSTEM_CLSES = [ Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, DistZilla, Gem, PerlBuildTiny, Golang, R, Octave, # Make is intentionally at the end of the list. - Make, RunTests] + Make, Composer, RunTests] def scan_buildsystems(path): diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 6bfc593..3853b26 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -69,21 +69,34 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs): def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): fixed_errors = [] while True: + to_resolve = [] try: run_detecting_problems(session, args, **kwargs) except DetailedFailure as e: + to_resolve.append(e) + else: + return + while to_resolve: + e = to_resolve.pop(-1) logging.info("Identified error: %r", e.error) if e.error in fixed_errors: logging.warning( "Failed to resolve error %r, it persisted. Giving up.", e.error ) - raise DetailedFailure(e.retcode, args, e.error) - if not resolve_error(e.error, None, fixers=fixers): - logging.warning("Failed to find resolution for error %r. Giving up.", e.error) - raise DetailedFailure(e.retcode, args, e.error) - fixed_errors.append(e.error) - else: - return + raise e + try: + if not resolve_error(e.error, None, fixers=fixers): + logging.warning("Failed to find resolution for error %r. Giving up.", e.error) + raise e + except DetailedFailure as n: + logging.info('New error %r while resolving %r', n, e) + if n in to_resolve: + raise + to_resolve.append(e) + to_resolve.append(n) + else: + fixed_errors.append(e.error) + def resolve_error(error, phase, fixers): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 1e37a4d..cb83fbd 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -80,6 +80,22 @@ class PythonPackageRequirement(Requirement): return p.returncode == 0 +class PhpPackageRequirement(Requirement): + + def __init__(self, package: str, channel: Optional[str] = None, + min_version: Optional[str] = None, + max_version: Optional[str] = None): + self.package = package + self.channel = channel + self.min_version = min_version + self.max_version = max_version + + def __repr__(self): + return "%s(%r, %r, %r, %r)" % ( + type(self).__name__, self.package, self.channel, + self.min_version, self.max_version) + + class BinaryRequirement(Requirement): binary_name: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 6b6f1d2..39ceb7c 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -41,6 +41,7 @@ from ..requirements import ( GoPackageRequirement, DhAddonRequirement, PhpClassRequirement, + PhpPackageRequirement, RPackageRequirement, NodeModuleRequirement, NodePackageRequirement, @@ -315,6 +316,10 @@ def resolve_php_class_req(apt_mgr, req): return find_reqs_simple(apt_mgr, [path]) +def resolve_php_package_req(apt_mgr, req): + return [AptRequirement.simple('php-%s' % req.package, minimum_version=req.min_version)] + + def resolve_r_package_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] return find_reqs_simple(apt_mgr, paths, regex=True) @@ -560,6 +565,7 @@ APT_REQUIREMENT_RESOLVERS = [ (GoPackageRequirement, resolve_go_package_req), (DhAddonRequirement, resolve_dh_addon_req), (PhpClassRequirement, resolve_php_class_req), + (PhpPackageRequirement, resolve_php_package_req), (RPackageRequirement, resolve_r_package_req), (NodeModuleRequirement, resolve_node_module_req), (NodePackageRequirement, resolve_node_package_req), From 16d37dc0458d524cca3b4e53f731db7c35ebaacb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 19:07:31 +0000 Subject: [PATCH 206/252] handle missing setup.py test command. --- ognibuild/buildlog.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index e8fcc7a..bdae2db 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -41,6 +41,7 @@ from buildlog_consultant.common import ( MissingPhpClass, MissingRubyGem, MissingLibrary, + MissingSetupPyCommand, MissingJavaClass, MissingCSharpCompiler, MissingRPackage, @@ -161,6 +162,10 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return LibtoolRequirement() elif isinstance(problem, UnknownCertificateAuthority): return CertificateAuthorityRequirement(problem.url) + elif isinstance(problem, MissingSetupPyCommand): + if problem.command == "test": + return MissingPythonDistribution("setuptools") + return None elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": return BinaryRequirement("glib-gettextize") From 85575ad526fe2df9584679546f9ee44348acf3ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 19:22:08 +0000 Subject: [PATCH 207/252] Undo fix build changes. --- ognibuild/fix_build.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 3853b26..6bfc593 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -69,34 +69,21 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs): def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): fixed_errors = [] while True: - to_resolve = [] try: run_detecting_problems(session, args, **kwargs) except DetailedFailure as e: - to_resolve.append(e) - else: - return - while to_resolve: - e = to_resolve.pop(-1) logging.info("Identified error: %r", e.error) if e.error in fixed_errors: logging.warning( "Failed to resolve error %r, it persisted. Giving up.", e.error ) - raise e - try: - if not resolve_error(e.error, None, fixers=fixers): - logging.warning("Failed to find resolution for error %r. Giving up.", e.error) - raise e - except DetailedFailure as n: - logging.info('New error %r while resolving %r', n, e) - if n in to_resolve: - raise - to_resolve.append(e) - to_resolve.append(n) - else: - fixed_errors.append(e.error) - + raise DetailedFailure(e.retcode, args, e.error) + if not resolve_error(e.error, None, fixers=fixers): + logging.warning("Failed to find resolution for error %r. Giving up.", e.error) + raise DetailedFailure(e.retcode, args, e.error) + fixed_errors.append(e.error) + else: + return def resolve_error(error, phase, fixers): From eec4b0a7d58915e76ff661942ffec310c8788009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 19:52:23 +0000 Subject: [PATCH 208/252] Fix type name. --- ognibuild/buildlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index bdae2db..35f3d23 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -164,7 +164,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return CertificateAuthorityRequirement(problem.url) elif isinstance(problem, MissingSetupPyCommand): if problem.command == "test": - return MissingPythonDistribution("setuptools") + return PythonPackageRequirement("setuptools") return None elif isinstance(problem, MissingGnomeCommonDependency): if problem.package == "glib-gettext": From 057599575a65cdc01c04b41d9fc9c0b1550cd029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 21:28:51 +0000 Subject: [PATCH 209/252] Try harder. --- ognibuild/fix_build.py | 48 ++++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 6bfc593..ca3f375 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -15,8 +15,9 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +from functools import partial import logging -from typing import List, Optional, Dict, Tuple +from typing import List, Optional, Tuple, Callable, Any from buildlog_consultant import Problem from buildlog_consultant.common import ( @@ -66,24 +67,45 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs): raise DetailedFailure(retcode, args, error) -def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): +def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): + """Call cb() until there are no more DetailedFailures we can fix. + + Args: + fixers: List of fixers to use to resolve issues + """ fixed_errors = [] while True: + to_resolve = [] try: - run_detecting_problems(session, args, **kwargs) + return cb() except DetailedFailure as e: - logging.info("Identified error: %r", e.error) - if e.error in fixed_errors: + to_resolve.append(e) + while to_resolve: + f = to_resolve.pop(-1) + logging.info("Identified error: %r", f.error) + if f.error in fixed_errors: logging.warning( - "Failed to resolve error %r, it persisted. Giving up.", e.error + "Failed to resolve error %r, it persisted. Giving up.", f.error ) - raise DetailedFailure(e.retcode, args, e.error) - if not resolve_error(e.error, None, fixers=fixers): - logging.warning("Failed to find resolution for error %r. Giving up.", e.error) - raise DetailedFailure(e.retcode, args, e.error) - fixed_errors.append(e.error) - else: - return + raise f + try: + if not resolve_error(f.error, None, fixers=fixers): + logging.warning("Failed to find resolution for error %r. Giving up.", f.error) + raise f + except DetailedFailure as n: + logging.info('New error %r while resolving %r', n, f) + if n in to_resolve: + raise + to_resolve.append(f) + to_resolve.append(n) + else: + fixed_errors.append(f.error) + + +def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): + return iterate_with_build_fixers( + fixers, + partial(run_detecting_problems, session, args, **kwargs)) def resolve_error(error, phase, fixers): From ae9e30468a8725d95771a63973942b473f8b3299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 22:17:33 +0000 Subject: [PATCH 210/252] Improve maven support. --- ognibuild/buildlog.py | 6 ++++-- ognibuild/buildsystem.py | 18 ++++++++++++++++++ ognibuild/requirements.py | 36 +++++++++++++++++++++++++++++++++--- ognibuild/resolver/apt.py | 35 ++++++++++++++--------------------- 4 files changed, 69 insertions(+), 26 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 35f3d23..115b3f2 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -141,9 +141,11 @@ def problem_to_upstream_requirement(problem): # noqa: C901 elif isinstance(problem, MissingJavaClass): return JavaClassRequirement(problem.classname) elif isinstance(problem, MissingHaskellDependencies): - return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps] + return [HaskellPackageRequirement.from_string(dep) + for dep in problem.deps] elif isinstance(problem, MissingMavenArtifacts): - return [MavenArtifactRequirement(artifact) for artifact in problem.artifacts] + return [MavenArtifactRequirement.from_str(artifact) + for artifact in problem.artifacts] elif isinstance(problem, MissingCSharpCompiler): return BinaryRequirement("msc") elif isinstance(problem, GnomeCommonMissing): diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 82d4992..17dd76f 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -41,6 +41,7 @@ from .requirements import ( RPackageRequirement, OctavePackageRequirement, PhpPackageRequirement, + MavenArtifactRequirement, ) from .fix_build import run_with_build_fixers from .session import which @@ -1209,6 +1210,23 @@ class Maven(BuildSystem): # is that what we need? raise NotImplementedError + def get_declared_dependencies(self, session, fixers=None): + import xml.etree.ElementTree as ET + try: + root = xmlparse_simplify_namespaces(self.path, + ['http://maven.apache.org/POM/4.0.0']) + except ET.ParseError as e: + logging.warning('Unable to parse package.xml: %s', e) + return + assert root.tag == 'project', 'root tag is %r' % root.tag + deps_tag = root.find('dependencies') + if deps_tag: + for dep in deps_tag.findall('dependency'): + yield "core", MavenArtifactRequirement( + dep.find('groupId').text, + dep.find('artifactId').text, + dep.find('version').text) + class Cabal(BuildSystem): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index cb83fbd..729e77b 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -429,11 +429,41 @@ class HaskellPackageRequirement(Requirement): class MavenArtifactRequirement(Requirement): - artifacts: List[Tuple[str, str, str]] + group_id: str + artifact_id: str + version: Optional[str] + kind: Optional[str] - def __init__(self, artifacts): + def __init__(self, group_id, artifact_id, version=None, kind=None): super(MavenArtifactRequirement, self).__init__("maven-artifact") - self.artifacts = artifacts + self.group_id = group_id + self.artifact_id = artifact_id + self.version = version + self.kind = kind + + def __str__(self): + return "maven requirement: %s:%s:%s" % ( + self.group_id, self.artifact_id, self.version) + + @classmethod + def from_str(cls, text): + return cls.from_tuple(text.split(':')) + + @classmethod + def from_tuple(cls, parts): + if len(parts) == 4: + (group_id, artifact_id, kind, version) = parts + elif len(parts) == 3: + (group_id, artifact_id, version) = parts + kind = "jar" + elif len(parts) == 2: + version = None + (group_id, artifact_id) = parts + kind = "jar" + else: + raise ValueError( + "invalid number of parts to artifact %r" % parts) + return cls(group_id, artifact_id, version, kind) class GnomeCommonRequirement(Requirement): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 39ceb7c..ce160f3 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -412,32 +412,25 @@ def resolve_haskell_package_req(apt_mgr, req): def resolve_maven_artifact_req(apt_mgr, req): - artifact = req.artifacts[0] - parts = artifact.split(":") - if len(parts) == 4: - (group_id, artifact_id, kind, version) = parts - regex = False - elif len(parts) == 3: - (group_id, artifact_id, version) = parts - kind = "jar" - regex = False - elif len(parts) == 2: + if req.version is None: version = ".*" - (group_id, artifact_id) = parts - kind = "jar" regex = True + escape = re.escape else: - raise AssertionError("invalid number of parts to artifact %s" % artifact) - paths = [ - posixpath.join( - "/usr/share/maven-repo", - group_id.replace(".", "/"), - artifact_id, + version = req.version + regex = False + def escape(x): + return x + kind = req.kind or 'jar' + path = posixpath.join( + escape("/usr/share/maven-repo"), + escape(req.group_id.replace(".", "/")), + escape(req.artifact_id), version, - "%s-%s.%s" % (artifact_id, version, kind), + escape("%s-") + version + escape("." + kind) ) - ] - return find_reqs_simple(apt_mgr, paths, regex=regex) + + return find_reqs_simple(apt_mgr, [path], regex=regex) def resolve_gnome_common_req(apt_mgr, req): From 5b7cf25d14c693f2132e25209e79a28ede22f0c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 22:54:22 +0000 Subject: [PATCH 211/252] More go support. --- ognibuild/buildsystem.py | 22 ++++++++++++++++++++++ ognibuild/requirements.py | 15 +++++++++++++-- ognibuild/resolver/apt.py | 6 ++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 17dd76f..e5718f8 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -42,6 +42,8 @@ from .requirements import ( OctavePackageRequirement, PhpPackageRequirement, MavenArtifactRequirement, + GoRequirement, + GoPackageRequirement, ) from .fix_build import run_with_build_fixers from .session import which @@ -1165,6 +1167,26 @@ class Golang(BuildSystem): def clean(self, session, resolver, fixers): session.check_call(["go", "clean"]) + def get_declared_dependencies(self, session, fixers=None): + go_mod_path = os.path.join(self.path, 'go.mod') + if not os.path.exists(go_mod_path): + with open(go_mod_path, 'r') as f: + for line in f: + parts = line.strip().split(' ') + if not parts: + continue + if parts[0] == 'go': + yield "build", GoRequirement(parts[1]) + elif parts[0] == 'require': + yield "build", GoPackageRequirement( + parts[1], parts[2] if len(parts) > 2 else None) + elif parts[0] in ('module', 'exclude', 'replace'): + pass + else: + logging.warning( + 'Unknown directive %s in go.mod', + parts[0]) + @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, 'go.mod')): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 729e77b..2b0efaa 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -271,10 +271,21 @@ class RubyGemRequirement(Requirement): class GoPackageRequirement(Requirement): package: str + version: Optional[str] - def __init__(self, package: str): - super(GoPackageRequirement, self).__init__("go") + def __init__(self, package: str, version: Optional[str] = None): + super(GoPackageRequirement, self).__init__("go-package") self.package = package + self.version = version + + +class GoRequirement(Requirement): + + version: Optional[str] + + def __init__(self, version: Optional[str] = None): + super(GoPackageRequirement, self).__init__("go") + self.version = version class DhAddonRequirement(Requirement): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index ce160f3..1bc4f78 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -39,6 +39,7 @@ from ..requirements import ( ValaPackageRequirement, RubyGemRequirement, GoPackageRequirement, + GoRequirement, DhAddonRequirement, PhpClassRequirement, PhpPackageRequirement, @@ -306,6 +307,10 @@ def resolve_go_package_req(apt_mgr, req): ) +def resolve_go_req(apt_mgr, req): + return [AptRequirement.simple('golang-%s' % req.version)] + + def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] return find_reqs_simple(apt_mgr, paths) @@ -556,6 +561,7 @@ APT_REQUIREMENT_RESOLVERS = [ (ValaPackageRequirement, resolve_vala_package_req), (RubyGemRequirement, resolve_ruby_gem_req), (GoPackageRequirement, resolve_go_package_req), + (GoRequirement, resolve_go_req), (DhAddonRequirement, resolve_dh_addon_req), (PhpClassRequirement, resolve_php_class_req), (PhpPackageRequirement, resolve_php_package_req), From f08359c73c021f48143f81c1a5ecb7f5c9b104f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 23:04:14 +0000 Subject: [PATCH 212/252] Support lists in go files. --- ognibuild/buildsystem.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index e5718f8..6442ee5 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1144,6 +1144,24 @@ class Cargo(BuildSystem): return Cargo(os.path.join(path, "Cargo.toml")) +def _parse_go_mod(f): + line = f.readline() + while f: + parts = line.strip().split(' ') + if not parts: + continue + if len(parts) == 2 and parts[1] == '(': + line = f.readline() + while line.strip() != ')': + yield [parts[0]] + list(line.strip().split(' ')) + line = f.readline() + if not line: + raise AssertionError('list of %s interrupted?' % parts[0]) + else: + yield parts + line = f.readline() + + class Golang(BuildSystem): """Go builds.""" @@ -1171,17 +1189,16 @@ class Golang(BuildSystem): go_mod_path = os.path.join(self.path, 'go.mod') if not os.path.exists(go_mod_path): with open(go_mod_path, 'r') as f: - for line in f: - parts = line.strip().split(' ') - if not parts: - continue + for parts in _parse_go_mod(f): if parts[0] == 'go': yield "build", GoRequirement(parts[1]) elif parts[0] == 'require': yield "build", GoPackageRequirement( parts[1], parts[2] if len(parts) > 2 else None) - elif parts[0] in ('module', 'exclude', 'replace'): - pass + elif parts[0] == 'exclude': + pass # TODO(jelmer): Create conflicts? + elif parts[0] == 'replace': + pass # TODO(jelmer): do.. something? else: logging.warning( 'Unknown directive %s in go.mod', From 445c2c19d2ee45df695b89ead2a4ee326f1836f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 23:05:52 +0000 Subject: [PATCH 213/252] Fix syntax. --- ognibuild/buildsystem.py | 4 ++-- ognibuild/requirements.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 6442ee5..c50e26e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1146,7 +1146,7 @@ class Cargo(BuildSystem): def _parse_go_mod(f): line = f.readline() - while f: + while line: parts = line.strip().split(' ') if not parts: continue @@ -1187,7 +1187,7 @@ class Golang(BuildSystem): def get_declared_dependencies(self, session, fixers=None): go_mod_path = os.path.join(self.path, 'go.mod') - if not os.path.exists(go_mod_path): + if os.path.exists(go_mod_path): with open(go_mod_path, 'r') as f: for parts in _parse_go_mod(f): if parts[0] == 'go': diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 2b0efaa..ee9aa13 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -284,7 +284,7 @@ class GoRequirement(Requirement): version: Optional[str] def __init__(self, version: Optional[str] = None): - super(GoPackageRequirement, self).__init__("go") + super(GoRequirement, self).__init__("go") self.version = version From e232c603eb7dcff03647a1cd7e2269e34b3b8c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 23:12:07 +0000 Subject: [PATCH 214/252] Fix go parsing. --- ognibuild/buildsystem.py | 15 ++++++++++----- ognibuild/requirements.py | 8 ++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index c50e26e..fa95842 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1145,21 +1145,26 @@ class Cargo(BuildSystem): def _parse_go_mod(f): - line = f.readline() + def readline(): + line = f.readline() + if not line: + return line + return line.split('//')[0] + '\n' + line = readline() while line: parts = line.strip().split(' ') if not parts: continue if len(parts) == 2 and parts[1] == '(': - line = f.readline() + line = readline() while line.strip() != ')': yield [parts[0]] + list(line.strip().split(' ')) - line = f.readline() + line = readline() if not line: raise AssertionError('list of %s interrupted?' % parts[0]) else: yield parts - line = f.readline() + line = readline() class Golang(BuildSystem): @@ -1194,7 +1199,7 @@ class Golang(BuildSystem): yield "build", GoRequirement(parts[1]) elif parts[0] == 'require': yield "build", GoPackageRequirement( - parts[1], parts[2] if len(parts) > 2 else None) + parts[1], parts[2].lstrip('v') if len(parts) > 2 else None) elif parts[0] == 'exclude': pass # TODO(jelmer): Create conflicts? elif parts[0] == 'replace': diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index ee9aa13..c1f9e6c 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -278,6 +278,11 @@ class GoPackageRequirement(Requirement): self.package = package self.version = version + def __str__(self): + if self.version: + return "go package: %s (= %s)" % (self.package, self.version) + return "go package: %s" % self.package + class GoRequirement(Requirement): @@ -287,6 +292,9 @@ class GoRequirement(Requirement): super(GoRequirement, self).__init__("go") self.version = version + def __str__(self): + return "go %s" % self.version + class DhAddonRequirement(Requirement): From fb848a86e5a71cfc0dca28d3cbc0db624be8507a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 23:15:17 +0000 Subject: [PATCH 215/252] Fix warnings. --- ognibuild/buildsystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index fa95842..5224161 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1153,7 +1153,7 @@ def _parse_go_mod(f): line = readline() while line: parts = line.strip().split(' ') - if not parts: + if not parts or parts == ['']: continue if len(parts) == 2 and parts[1] == '(': line = readline() @@ -1204,6 +1204,8 @@ class Golang(BuildSystem): pass # TODO(jelmer): Create conflicts? elif parts[0] == 'replace': pass # TODO(jelmer): do.. something? + elif parts[0] == 'module': + pass else: logging.warning( 'Unknown directive %s in go.mod', From d35b4092de714dc0e1a4cb89f83a1d567fca5eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 25 Mar 2021 23:55:08 +0000 Subject: [PATCH 216/252] Add manual map. --- ognibuild/resolver/apt.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 1bc4f78..85275fd 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -235,11 +235,18 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] +vague_map = { + 'the Gnu Scientific Library': 'libgsl-dev', +} + + def resolve_vague_dep_req(apt_mgr, req): name = req.name + options = [] + if name in vague_map: + options.append(AptRequirement.simple(vague_map[name])) if name.startswith('gnu '): name = name[4:] - options = [] for x in req.expand(): options.extend(resolve_requirement_apt(apt_mgr, x)) return options From 1ba4014c2a58eeb36bf8942ac3047f2508cebf95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:10:32 +0000 Subject: [PATCH 217/252] Fix import. --- ognibuild/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index c1f9e6c..2d0d0cc 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -146,7 +146,7 @@ class VagueDependencyRequirement(Requirement): def expand(self): yield BinaryRequirement(self.name) yield LibraryRequirement(self.name) - from resolver.apt import AptRequirement + from .resolver.apt import AptRequirement yield AptRequirement(self.name) def met(self, session): From 56f3e3f87c4b26177058d65ac102fb2c746a2a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:22:03 +0000 Subject: [PATCH 218/252] Support parsing cpan files. --- ognibuild/buildsystem.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 5224161..52153f7 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -21,7 +21,6 @@ import logging import os import re import shlex -import stat from typing import Optional, Tuple import warnings @@ -937,6 +936,12 @@ class RunTests(BuildSystem): run_with_build_fixers(session, ["/bin/bash", "./runtests.sh"], fixers) +def _read_cpanfile(session, args, kind): + output = session.check_output(['cpanfile-dump'] + args) + for line in output.splitlines(False): + yield kind, PerlModuleRequirement(line.decode().strip()) + + class Make(BuildSystem): name = "make" @@ -1060,6 +1065,7 @@ class Make(BuildSystem): return dc.copy_single(target_directory) def get_declared_dependencies(self, session, fixers=None): + something = False # TODO(jelmer): Split out the perl-specific stuff? if os.path.exists(os.path.join(self.path, "META.yml")): # See http://module-build.sourceforge.net/META-spec-v1.4.html for @@ -1075,7 +1081,12 @@ class Make(BuildSystem): return for require in data.get("requires", []): yield "build", PerlModuleRequirement(require) - else: + something = True + if os.path.exists(os.path.join(self.path, "cpanfile")): + yield from _read_cpanfile(session, ['--configure', '--build'], 'build') + yield from _read_cpanfile(session, ['--test'], 'test') + something = True + if not something: raise NotImplementedError @classmethod From fd6c9fdeecaf4df907257734a2d33e42711d2650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:27:01 +0000 Subject: [PATCH 219/252] Also read cpanfile when using dist-inkt. --- ognibuild/buildsystem.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 52153f7..0fb28cf 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -912,6 +912,8 @@ class DistZilla(BuildSystem): out = session.check_output(["dzil", "authordeps"]) for entry in out.splitlines(): yield "build", PerlModuleRequirement(entry.decode().strip()) + if os.path.exists(os.path.join(os.path.dirname(self.path), "cpanfile")): + yield from _declared_deps_from_cpanfile(session) class RunTests(BuildSystem): @@ -942,6 +944,11 @@ def _read_cpanfile(session, args, kind): yield kind, PerlModuleRequirement(line.decode().strip()) +def _declared_deps_from_cpanfile(session): + yield from _read_cpanfile(session, ['--configure', '--build'], 'build') + yield from _read_cpanfile(session, ['--test'], 'test') + + class Make(BuildSystem): name = "make" @@ -1083,8 +1090,7 @@ class Make(BuildSystem): yield "build", PerlModuleRequirement(require) something = True if os.path.exists(os.path.join(self.path, "cpanfile")): - yield from _read_cpanfile(session, ['--configure', '--build'], 'build') - yield from _read_cpanfile(session, ['--test'], 'test') + yield from _declared_deps_from_cpanfile(session) something = True if not something: raise NotImplementedError From b9ae8e7e2c0a4daae40dabb1e22a3091b1d28675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:36:28 +0000 Subject: [PATCH 220/252] Specify -T. --- ognibuild/resolver/__init__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index cd6614c..40b66d2 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -39,9 +39,10 @@ class Resolver(object): class CPANResolver(Resolver): - def __init__(self, session, user_local=False): + def __init__(self, session, user_local=False, skip_tests=True): self.session = session self.user_local = user_local + self.skip_tests = skip_tests def __str__(self): return "cpan" @@ -49,6 +50,13 @@ class CPANResolver(Resolver): def __repr__(self): return "%s(%r)" % (type(self).__name__, self.session) + def _cmd(self, reqs): + ret = ["cpan", "-i"] + if self.skip_tests: + ret.append('-T') + ret.extend([req.module for req in reqs]) + return ret + def explain(self, requirements): from ..requirements import PerlModuleRequirement @@ -58,7 +66,7 @@ class CPANResolver(Resolver): continue perlreqs.append(requirement) if perlreqs: - yield (["cpan", "-i"] + [req.module for req in perlreqs], [perlreqs]) + yield (self._cmd(perlreqs), [perlreqs]) def install(self, requirements): from ..requirements import PerlModuleRequirement @@ -79,9 +87,9 @@ class CPANResolver(Resolver): if not isinstance(requirement, PerlModuleRequirement): missing.append(requirement) continue - # TODO(jelmer): Specify -T to skip tests? - run_detecting_problems(self.session, - ["cpan", "-i", requirement.module], + run_detecting_problems( + self.session, + self._cmd([requirement]), env=env, user=user, ) From 6809f3c5e069d96da30c6e2e4a00f69730879de0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:52:37 +0000 Subject: [PATCH 221/252] Sanitize. --- ognibuild/requirements.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 2d0d0cc..3579048 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -144,10 +144,11 @@ class VagueDependencyRequirement(Requirement): self.name = name def expand(self): - yield BinaryRequirement(self.name) - yield LibraryRequirement(self.name) - from .resolver.apt import AptRequirement - yield AptRequirement(self.name) + if ' ' not in self.name: + yield BinaryRequirement(self.name) + yield LibraryRequirement(self.name) + from .resolver.apt import AptRequirement + yield AptRequirement(self.name) def met(self, session): for x in self.expand(): From ab8ecbd8bd9f4e6b92c952c487acee495df32434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:53:08 +0000 Subject: [PATCH 222/252] Fixes. --- ognibuild/fix_build.py | 7 ++++--- ognibuild/resolver/__init__.py | 2 ++ ognibuild/resolver/apt.py | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index ca3f375..c7b363e 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -89,9 +89,7 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): ) raise f try: - if not resolve_error(f.error, None, fixers=fixers): - logging.warning("Failed to find resolution for error %r. Giving up.", f.error) - raise f + resolved = resolve_error(f.error, None, fixers=fixers) except DetailedFailure as n: logging.info('New error %r while resolving %r', n, f) if n in to_resolve: @@ -99,6 +97,9 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): to_resolve.append(f) to_resolve.append(n) else: + if not resolved: + logging.warning("Failed to find resolution for error %r. Giving up.", f.error) + raise f fixed_errors.append(f.error) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 40b66d2..73b3c89 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -436,6 +436,8 @@ class StackedResolver(Resolver): requirements = e.requirements else: return + if requirements: + raise UnsatisfiedRequirements(requirements) NATIVE_RESOLVER_CLS = [ diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 85275fd..cfd67a3 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -73,6 +73,8 @@ from ..requirements import ( class AptRequirement(Requirement): def __init__(self, relations): super(AptRequirement, self).__init__("apt") + if not isinstance(relations, list): + raise TypeError(relations) self.relations = relations @classmethod From 16a59557f2ee40aeb9f4201913c6add48cb45a61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:57:00 +0000 Subject: [PATCH 223/252] Fix library finding. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index cfd67a3..e98374a 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -364,7 +364,7 @@ def resolve_library_req(apt_mgr, req): posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), ] - return find_reqs_simple(apt_mgr, paths) + return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_ruby_file_req(apt_mgr, req): From 7556399b35e657d774368fb6c09a072496224d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 01:59:28 +0000 Subject: [PATCH 224/252] Fix golang installs. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index e98374a..dd19be3 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -317,7 +317,7 @@ def resolve_go_package_req(apt_mgr, req): def resolve_go_req(apt_mgr, req): - return [AptRequirement.simple('golang-%s' % req.version)] + return [AptRequirement.simple('golang-go', minimum_version='2:%s' % req.version)] def resolve_dh_addon_req(apt_mgr, req): From 423774519754823b915bcf44721a08e85f3848ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:05:10 +0000 Subject: [PATCH 225/252] Add --include-controldir. --- ognibuild/debian/file_search.py | 1 + ognibuild/dist.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 21c570f..429852e 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -286,6 +286,7 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( "/usr/bin/rst2html": "python3-docutils", # aclocal is a symlink to aclocal-1.XY "/usr/bin/aclocal": "automake", + "/usr/bin/automake": "automake", # maven lives in /usr/share "/usr/bin/mvn": "maven", } diff --git a/ognibuild/dist.py b/ognibuild/dist.py index b7c56c9..ce8bcdc 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -146,6 +146,9 @@ if __name__ == "__main__": "--target-directory", type=str, default="..", help="Target directory" ) parser.add_argument("--verbose", action="store_true", help="Be verbose") + parser.add_argument( + '--include-controldir', action='store_true', + help='Clone rather than export.') args = parser.parse_args() @@ -172,6 +175,7 @@ if __name__ == "__main__": target_dir=os.path.abspath(args.target_directory), packaging_tree=packaging_tree, chroot=args.chroot, + include_controldir=args.include_controldir, ) except (NoBuildToolsFound, NotImplementedError): logging.info("No build tools found, falling back to simple export.") From 4a8d514f13cdba7ea7aab474d40b672b9832c45e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:20:30 +0000 Subject: [PATCH 226/252] Install dzil if it is missing. --- ognibuild/buildsystem.py | 8 ++++---- ognibuild/fix_build.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 0fb28cf..44c1091 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -44,7 +44,7 @@ from .requirements import ( GoRequirement, GoPackageRequirement, ) -from .fix_build import run_with_build_fixers +from .fix_build import run_with_build_fixers, run_detecting_problems from .session import which @@ -909,9 +909,9 @@ class DistZilla(BuildSystem): return cls(os.path.join(path, "dist.ini")) def get_declared_dependencies(self, session, fixers=None): - out = session.check_output(["dzil", "authordeps"]) - for entry in out.splitlines(): - yield "build", PerlModuleRequirement(entry.decode().strip()) + lines = run_with_build_fixers(session, ["dzil", "authordeps"], fixers) + for entry in lines: + yield "build", PerlModuleRequirement(entry.strip()) if os.path.exists(os.path.join(os.path.dirname(self.path), "cpanfile")): yield from _declared_deps_from_cpanfile(session) diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index c7b363e..3411cb2 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -54,7 +54,7 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs): retcode = 1 else: if retcode == 0: - return + return contents lines = ''.join(contents).splitlines(False) match, error = find_build_failure_description(lines) if error is None: From fe0475bd32bd1712673869d2e3f1b3a03554bc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:23:23 +0000 Subject: [PATCH 227/252] Install cpanfile-dump if it's missing. --- ognibuild/buildsystem.py | 17 ++++++++--------- ognibuild/session/__init__.py | 2 ++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 44c1091..9eaea1d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -913,7 +913,7 @@ class DistZilla(BuildSystem): for entry in lines: yield "build", PerlModuleRequirement(entry.strip()) if os.path.exists(os.path.join(os.path.dirname(self.path), "cpanfile")): - yield from _declared_deps_from_cpanfile(session) + yield from _declared_deps_from_cpanfile(session, fixers) class RunTests(BuildSystem): @@ -938,15 +938,14 @@ class RunTests(BuildSystem): run_with_build_fixers(session, ["/bin/bash", "./runtests.sh"], fixers) -def _read_cpanfile(session, args, kind): - output = session.check_output(['cpanfile-dump'] + args) - for line in output.splitlines(False): - yield kind, PerlModuleRequirement(line.decode().strip()) +def _read_cpanfile(session, args, kind, fixers): + for line in run_with_build_fixers(session, ['cpanfile-dump'] + args, fixers): + yield kind, PerlModuleRequirement(line) -def _declared_deps_from_cpanfile(session): - yield from _read_cpanfile(session, ['--configure', '--build'], 'build') - yield from _read_cpanfile(session, ['--test'], 'test') +def _declared_deps_from_cpanfile(session, fixers): + yield from _read_cpanfile(session, ['--configure', '--build'], 'build', fixers) + yield from _read_cpanfile(session, ['--test'], 'test', fixers) class Make(BuildSystem): @@ -1090,7 +1089,7 @@ class Make(BuildSystem): yield "build", PerlModuleRequirement(require) something = True if os.path.exists(os.path.join(self.path, "cpanfile")): - yield from _declared_deps_from_cpanfile(session) + yield from _declared_deps_from_cpanfile(session, fixers) something = True if not something: raise NotImplementedError diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 801c125..24d0bd9 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -108,6 +108,8 @@ class SessionSetupFailure(Exception): def run_with_tee(session: Session, args: List[str], **kwargs): + if 'stdin' not in kwargs: + kwargs['stdin'] = subprocess.DEVNULL p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) contents = [] while p.poll() is None: From f40a611ae231849749488fc9093a49df94b4255c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:25:16 +0000 Subject: [PATCH 228/252] Don't install preemptively. --- ognibuild/buildsystem.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9eaea1d..451a35e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -984,14 +984,6 @@ class Make(BuildSystem): raise elif session.exists("configure.ac") or session.exists("configure.in"): - resolver.install( - [ - BinaryRequirement("autoconf"), - BinaryRequirement("automake"), - BinaryRequirement("gettextize"), - BinaryRequirement("libtoolize"), - ] - ) run_with_build_fixers(session, ["autoreconf", "-i"], fixers) if not makefile_exists() and session.exists("configure"): From 0220d5c52439941397af592a72c7ce7e04bfe2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:36:25 +0000 Subject: [PATCH 229/252] Rerun autoconf if it didn't expand all macros. --- ognibuild/dist.py | 8 ++++++-- ognibuild/fixers.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index ce8bcdc..aa23aa9 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -71,7 +71,9 @@ def create_dist( from .buildsystem import detect_buildsystems from .buildlog import InstallFixer from .fix_build import BuildFixer - from .fixers import GitIdentityFixer, SecretGpgKeyFixer + from .fixers import ( + GitIdentityFixer, SecretGpgKeyFixer, + UnexpandedAutoconfMacroFixer, ) if subdir is None: subdir = "package" @@ -86,7 +88,9 @@ def create_dist( # TODO(jelmer): use scan_buildsystems to also look in subdirectories buildsystems = list(detect_buildsystems(export_directory)) resolver = auto_resolver(session) - fixers: List[BuildFixer] = [InstallFixer(resolver)] + fixers: List[BuildFixer] = [UnexpandedAutoconfMacroFixer(session, resolver)] + + fixers.append(InstallFixer(resolver)) if session.is_temporary: # Only muck about with temporary sessions diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index c5ed6db..995f2c1 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -22,7 +22,10 @@ from buildlog_consultant import Problem from buildlog_consultant.common import ( MissingGitIdentity, MissingSecretGpgKey, + MissingAutoconfMacro, ) +from ognibuild.requirements import AutoconfMacroRequirement +from ognibuild.resolver import UnsatisfiedRequirements from .fix_build import BuildFixer @@ -70,3 +73,30 @@ Passphrase: "" if p.returncode == 0: return True return False + + +class UnexpandedAutoconfMacroFixer(BuildFixer): + + def __init__(self, session, resolver): + self.session = session + self.resolver = resolver + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.resolver) + + def __str__(self): + return "unexpanded m4 macro fixer (%s)" % self.resolver + + def can_fix(self, error): + return isinstance(error, MissingAutoconfMacro) + + def _fix(self, error, phase): + try: + self.resolver.install([AutoconfMacroRequirement(error.macro)]) + except UnsatisfiedRequirements: + return False + from .fix_build import run_detecting_problems + + run_detecting_problems(self.session, ['autoconf']) + + return True From be14708c5f0e576c39c42395c08c5b71b6eee979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:40:52 +0000 Subject: [PATCH 230/252] Hardcode another option. --- ognibuild/resolver/apt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index dd19be3..2f99753 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -239,6 +239,7 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): vague_map = { 'the Gnu Scientific Library': 'libgsl-dev', + 'the required FreeType library': 'libfreetype-dev', } From edbf0f281c23f00d20366b5a24802aa5a23094ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:50:05 +0000 Subject: [PATCH 231/252] Add another special case for python versions. --- ognibuild/resolver/apt.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 2f99753..3202205 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -151,6 +151,13 @@ def python_spec_to_apt_rels(pkg_name, specs): elif spec[0] == '!=': rels.extend([{"name": pkg_name, "version": ('>>', deb_version)}, {"name": pkg_name, "version": ('<<', deb_version)}]) + elif spec[1].endswith('.*') and spec[0] == '==': + s = spec[1].split('.') + s.pop(-1) + n = list(s) + n[-1] = str(int(n[-1])+1) + rels.extend([{"name": pkg_name, "version": ('>=', Version('.'.join(s)))}, + {"name": pkg_name, "version": ('<<', Version('.'.join(n)))}]) else: c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] rels.append([{"name": pkg_name, "version": (c, deb_version)}]) From a1b4bba4e8510bc2d48238396f3ce3efcdaa83f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 02:51:56 +0000 Subject: [PATCH 232/252] Specify -f. --- ognibuild/fixers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index 995f2c1..b9e9734 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -97,6 +97,6 @@ class UnexpandedAutoconfMacroFixer(BuildFixer): return False from .fix_build import run_detecting_problems - run_detecting_problems(self.session, ['autoconf']) + run_detecting_problems(self.session, ['autoconf', '-f']) return True From b29c9cecf05a792b6c708872445cdd298310490e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 03:28:41 +0000 Subject: [PATCH 233/252] Try lower case version of library names. --- ognibuild/requirements.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 3579048..26c0f95 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -147,8 +147,10 @@ class VagueDependencyRequirement(Requirement): if ' ' not in self.name: yield BinaryRequirement(self.name) yield LibraryRequirement(self.name) + if self.name.lower() != self.name: + yield LibraryRequirement(self.name.lower()) from .resolver.apt import AptRequirement - yield AptRequirement(self.name) + yield AptRequirement(self.name.lower()) def met(self, session): for x in self.expand(): From 46c42872d74304aebb7c0c567e7a5f830932387a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 03:29:24 +0000 Subject: [PATCH 234/252] Try lower case version of library names. --- ognibuild/requirements.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 26c0f95..df540ef 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -147,8 +147,10 @@ class VagueDependencyRequirement(Requirement): if ' ' not in self.name: yield BinaryRequirement(self.name) yield LibraryRequirement(self.name) + yield PkgConfigRequirement(self.name) if self.name.lower() != self.name: yield LibraryRequirement(self.name.lower()) + yield PkgConfigRequirement(self.name.lower()) from .resolver.apt import AptRequirement yield AptRequirement(self.name.lower()) From 61c87f500beeb6b2b7e00da5fd077c0a0dcd4c3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 03:38:01 +0000 Subject: [PATCH 235/252] Search for lowercase version of binary as well. --- ognibuild/requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index df540ef..c7e8fe8 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -149,6 +149,7 @@ class VagueDependencyRequirement(Requirement): yield LibraryRequirement(self.name) yield PkgConfigRequirement(self.name) if self.name.lower() != self.name: + yield BinaryRequirement(self.name.lower()) yield LibraryRequirement(self.name.lower()) yield PkgConfigRequirement(self.name.lower()) from .resolver.apt import AptRequirement From 2435f7825a3ad6acaada8f31527c4cd5c9bcf88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 04:35:00 +0000 Subject: [PATCH 236/252] Fix expansion of vague requirement. --- ognibuild/requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index c7e8fe8..3e5cba4 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -153,7 +153,7 @@ class VagueDependencyRequirement(Requirement): yield LibraryRequirement(self.name.lower()) yield PkgConfigRequirement(self.name.lower()) from .resolver.apt import AptRequirement - yield AptRequirement(self.name.lower()) + yield AptRequirement.simple(self.name.lower()) def met(self, session): for x in self.expand(): From 5931e1fbb69f9d507c8641f26f17f1c14f975664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 11:49:23 +0000 Subject: [PATCH 237/252] Mention when we're needing to break a tie. --- ognibuild/resolver/apt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 3202205..812cd53 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -678,6 +678,7 @@ class AptResolver(Resolver): return None if len(ret) == 1: return ret[0] + logging.info('Need to break tie between %r', ret) for tie_breaker in self.tie_breakers: winner = tie_breaker(ret) if winner is not None: From 7b3b1711349f1b609bde55632c51850dcba70304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 12:12:35 +0000 Subject: [PATCH 238/252] Also mention tie breakers. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 812cd53..c0f3dc5 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -678,7 +678,7 @@ class AptResolver(Resolver): return None if len(ret) == 1: return ret[0] - logging.info('Need to break tie between %r', ret) + logging.info('Need to break tie between %r with %r', ret, self.tie_breakers) for tie_breaker in self.tie_breakers: winner = tie_breaker(ret) if winner is not None: From 3d914f5b4ee0e687004a15272066a1359de91725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 12:14:25 +0000 Subject: [PATCH 239/252] Fix style. --- ognibuild/__init__.py | 6 +- ognibuild/__main__.py | 27 +- ognibuild/buildlog.py | 10 +- ognibuild/buildsystem.py | 401 +++++++++++++---------- ognibuild/debian/apt.py | 15 +- ognibuild/debian/build.py | 1 - ognibuild/debian/build_deps.py | 17 +- ognibuild/debian/file_search.py | 76 +++-- ognibuild/debian/fix_build.py | 141 ++++---- ognibuild/debian/udd.py | 13 +- ognibuild/dist.py | 53 +-- ognibuild/dist_catcher.py | 13 +- ognibuild/fix_build.py | 20 +- ognibuild/fixers.py | 25 +- ognibuild/requirements.py | 75 +++-- ognibuild/resolver/__init__.py | 43 +-- ognibuild/resolver/apt.py | 264 +++++++++------ ognibuild/session/__init__.py | 10 +- ognibuild/session/plain.py | 34 +- ognibuild/session/schroot.py | 25 +- ognibuild/tests/test_debian_fix_build.py | 9 +- 21 files changed, 754 insertions(+), 524 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index da463df..a80945c 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -41,7 +41,11 @@ class UnidentifiedError(Exception): def __repr__(self): return "<%s(%r, %r, ..., secondary=%r)>" % ( - type(self).__name__, self.retcode, self.argv, self.secondary) + type(self).__name__, + self.retcode, + self.argv, + self.secondary, + ) def shebang_binary(p): diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 6d6d430..514056f 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -20,7 +20,12 @@ import os import shlex import sys from . import UnidentifiedError, DetailedFailure -from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall, install_missing_reqs +from .buildlog import ( + InstallFixer, + ExplainInstallFixer, + ExplainInstall, + install_missing_reqs, +) from .buildsystem import NoBuildToolsFound, detect_buildsystems from .resolver import ( auto_resolver, @@ -34,8 +39,7 @@ def display_explain_commands(commands): for command, reqs in commands: if isinstance(command, list): command = shlex.join(command) - logging.info( - ' %s (to install %s)', command, ', '.join(map(str, reqs))) + logging.info(" %s (to install %s)", command, ", ".join(map(str, reqs))) def get_necessary_declared_requirements(resolver, requirements, stages): @@ -46,7 +50,9 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(session, resolver, fixers, buildsystems, stages, explain=False): +def install_necessary_declared_requirements( + session, resolver, fixers, buildsystems, stages, explain=False +): relevant = [] declared_reqs = [] for buildsystem in buildsystems: @@ -153,8 +159,7 @@ def main(): # noqa: C901 logging.info("Using requirement resolver: %s", resolver) try: bss = list(detect_buildsystems(args.directory)) - logging.info( - "Detected buildsystems: %s", ', '.join(map(str, bss))) + logging.info("Detected buildsystems: %s", ", ".join(map(str, bss))) fixers = determine_fixers(session, resolver, explain=args.explain) if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] @@ -162,7 +167,8 @@ def main(): # noqa: C901 logging.info("Checking that declared requirements are present") try: install_necessary_declared_requirements( - session, resolver, fixers, bss, stages, explain=args.explain) + session, resolver, fixers, bss, stages, explain=args.explain + ) except ExplainInstall as e: display_explain_commands(e.commands) return 1 @@ -170,8 +176,11 @@ def main(): # noqa: C901 from .dist import run_dist run_dist( - session=session, buildsystems=bss, resolver=resolver, fixers=fixers, - target_directory='.' + session=session, + buildsystems=bss, + resolver=resolver, + fixers=fixers, + target_directory=".", ) if args.subcommand == "build": from .build import run_build diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 115b3f2..1381ccc 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -141,11 +141,12 @@ def problem_to_upstream_requirement(problem): # noqa: C901 elif isinstance(problem, MissingJavaClass): return JavaClassRequirement(problem.classname) elif isinstance(problem, MissingHaskellDependencies): - return [HaskellPackageRequirement.from_string(dep) - for dep in problem.deps] + return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps] elif isinstance(problem, MissingMavenArtifacts): - return [MavenArtifactRequirement.from_str(artifact) - for artifact in problem.artifacts] + return [ + MavenArtifactRequirement.from_str(artifact) + for artifact in problem.artifacts + ] elif isinstance(problem, MissingCSharpCompiler): return BinaryRequirement("msc") elif isinstance(problem, GnomeCommonMissing): @@ -236,7 +237,6 @@ class InstallFixer(BuildFixer): class ExplainInstall(Exception): - def __init__(self, commands): self.commands = commands diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 451a35e..9762220 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -44,7 +44,7 @@ from .requirements import ( GoRequirement, GoPackageRequirement, ) -from .fix_build import run_with_build_fixers, run_detecting_problems +from .fix_build import run_with_build_fixers from .session import which @@ -75,7 +75,9 @@ class BuildSystem(object): def __str__(self): return self.name - def dist(self, session, resolver, fixers, target_directory: str, quiet=False) -> str: + def dist( + self, session, resolver, fixers, target_directory: str, quiet=False + ) -> str: raise NotImplementedError(self.dist) def test(self, session, resolver, fixers): @@ -103,11 +105,12 @@ class BuildSystem(object): def xmlparse_simplify_namespaces(path, namespaces): import xml.etree.ElementTree as ET - namespaces = ['{%s}' % ns for ns in namespaces] + + namespaces = ["{%s}" % ns for ns in namespaces] tree = ET.iterparse(path) for _, el in tree: for namespace in namespaces: - el.tag = el.tag.replace(namespace, '') + el.tag = el.tag.replace(namespace, "") return tree.root @@ -123,7 +126,7 @@ class Pear(BuildSystem): def dist(self, session, resolver, fixers, target_directory: str, quiet=False): self.setup(resolver) - with DistCatcher([session.external_path('.')]) as dc: + with DistCatcher([session.external_path(".")]) as dc: run_with_build_fixers(session, ["pear", "package"], fixers) return dc.copy_single(target_directory) @@ -146,28 +149,34 @@ class Pear(BuildSystem): def get_declared_dependencies(self, session, fixers=None): path = os.path.join(self.path, "package.xml") import xml.etree.ElementTree as ET + try: - root = xmlparse_simplify_namespaces(path, [ - 'http://pear.php.net/dtd/package-2.0', - 'http://pear.php.net/dtd/package-2.1']) + root = xmlparse_simplify_namespaces( + path, + [ + "http://pear.php.net/dtd/package-2.0", + "http://pear.php.net/dtd/package-2.1", + ], + ) except ET.ParseError as e: - logging.warning('Unable to parse package.xml: %s', e) + logging.warning("Unable to parse package.xml: %s", e) return - assert root.tag == 'package', 'root tag is %r' % root.tag - dependencies_tag = root.find('dependencies') + assert root.tag == "package", "root tag is %r" % root.tag + dependencies_tag = root.find("dependencies") if dependencies_tag is not None: - required_tag = root.find('dependencies') + required_tag = root.find("dependencies") if required_tag is not None: - for package_tag in root.findall('package'): - name = package_tag.find('name').text - min_tag = package_tag.find('min') - max_tag = package_tag.find('max') - channel_tag = package_tag.find('channel') + for package_tag in root.findall("package"): + name = package_tag.find("name").text + min_tag = package_tag.find("min") + max_tag = package_tag.find("max") + channel_tag = package_tag.find("channel") yield "core", PhpPackageRequirement( name, channel=(channel_tag.text if channel_tag else None), min_version=(min_tag.text if min_tag else None), - max_version=(max_tag.text if max_tag else None)) + max_version=(max_tag.text if max_tag else None), + ) @classmethod def probe(cls, path): @@ -179,6 +188,7 @@ class Pear(BuildSystem): # run_setup, but setting __name__ # Imported from Python's distutils.core, Copyright (C) PSF + def run_setup(script_name, script_args=None, stop_after="run"): from distutils import core import sys @@ -260,11 +270,11 @@ with open(%(output_path)s, 'w') as f: class SetupPy(BuildSystem): name = "setup.py" - DEFAULT_PYTHON = 'python3' + DEFAULT_PYTHON = "python3" def __init__(self, path): self.path = path - if os.path.exists(os.path.join(self.path, 'setup.py')): + if os.path.exists(os.path.join(self.path, "setup.py")): self.has_setup_py = True else: self.has_setup_py = False @@ -280,7 +290,9 @@ class SetupPy(BuildSystem): self.pyproject = None self.build_backend = None else: - self.build_backend = self.pyproject.get("build-system", {}).get('build-backend') + self.build_backend = self.pyproject.get("build-system", {}).get( + "build-backend" + ) def load_toml(self): import toml @@ -290,7 +302,8 @@ class SetupPy(BuildSystem): def load_setup_cfg(self): from setuptools.config import read_configuration - p = os.path.join(self.path, 'setup.cfg') + + p = os.path.join(self.path, "setup.cfg") if os.path.exists(p): return read_configuration(p) raise FileNotFoundError(p) @@ -304,7 +317,7 @@ class SetupPy(BuildSystem): return self._extract_setup_in_session(session, fixers) def _extract_setup_direct(self): - p = os.path.join(self.path, 'setup.py') + p = os.path.join(self.path, "setup.py") try: d = run_setup(os.path.abspath(p), stop_after="init") except RuntimeError as e: @@ -313,37 +326,42 @@ class SetupPy(BuildSystem): if d is None: logging.warning( "'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p)) + "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p) + ) return None return { - 'name': d.name, - 'setup_requires': getattr(d, "setup_requires", []), - 'install_requires': getattr(d, "install_requires", []), - 'tests_require': getattr(d, "tests_require", []) or [], - 'scripts': getattr(d, "scripts", []), - 'entry_points': getattr(d, "entry_points", None) or {}, - 'packages': getattr(d, "packages", []), - 'requires': d.get_requires() or [], - } + "name": d.name, + "setup_requires": getattr(d, "setup_requires", []), + "install_requires": getattr(d, "install_requires", []), + "tests_require": getattr(d, "tests_require", []) or [], + "scripts": getattr(d, "scripts", []), + "entry_points": getattr(d, "entry_points", None) or {}, + "packages": getattr(d, "packages", []), + "requires": d.get_requires() or [], + } def _extract_setup_in_session(self, session, fixers=None): import tempfile import json + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) if interpreter is None: interpreter = self.DEFAULT_PYTHON output_f = tempfile.NamedTemporaryFile( - dir=os.path.join(session.location, 'tmp'), mode='w+t') + dir=os.path.join(session.location, "tmp"), mode="w+t" + ) with output_f: # TODO(jelmer): Perhaps run this in session, so we can install # missing dependencies? - argv = [interpreter, "-c", - _setup_wrapper - .replace('%(script_name)s', '"setup.py"') - .replace('%(output_path)s', - '"/' + os.path.relpath(output_f.name, session.location) + - '"')] + argv = [ + interpreter, + "-c", + _setup_wrapper.replace("%(script_name)s", '"setup.py"').replace( + "%(output_path)s", + '"/' + os.path.relpath(output_f.name, session.location) + '"', + ), + ] try: if fixers is not None: run_with_build_fixers(session, argv, fixers) @@ -359,15 +377,17 @@ class SetupPy(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def test(self, session, resolver, fixers): - if os.path.exists(os.path.join(self.path, 'tox.ini')): - run_with_build_fixers(session, ['tox'], fixers) + if os.path.exists(os.path.join(self.path, "tox.ini")): + run_with_build_fixers(session, ["tox"], fixers) elif self.pyproject: - run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers) + run_with_build_fixers( + session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers + ) elif self.has_setup_py: # Pre-emptively insall setuptools, since distutils doesn't provide # a 'test' subcommand and some packages fall back to distutils # if setuptools is not available. - resolver.install([PythonPackageRequirement('setuptools')]) + resolver.install([PythonPackageRequirement("setuptools")]) self._run_setup(session, resolver, ["test"], fixers) else: raise NotImplementedError @@ -386,13 +406,17 @@ class SetupPy(BuildSystem): preargs.append("--quiet") # Preemptively install setuptools since some packages fail in # some way without it. - resolver.install([PythonPackageRequirement('setuptools')]) - with DistCatcher([session.external_path('dist')]) as dc: + resolver.install([PythonPackageRequirement("setuptools")]) + with DistCatcher([session.external_path("dist")]) as dc: self._run_setup(session, resolver, preargs + ["sdist"], fixers) return dc.copy_single(target_directory) elif self.pyproject: - with DistCatcher([session.external_path('dist')]) as dc: - run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], fixers) + with DistCatcher([session.external_path("dist")]) as dc: + run_with_build_fixers( + session, + [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], + fixers, + ) return dc.copy_single(target_directory) raise AssertionError("no setup.py or pyproject.toml") @@ -413,79 +437,83 @@ class SetupPy(BuildSystem): def _run_setup(self, session, resolver, args, fixers): from .buildlog import install_missing_reqs + # Install the setup_requires beforehand, since otherwise # setuptools might fetch eggs instead of our preferred resolver. install_missing_reqs(session, resolver, list(self._setup_requires())) - interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) if interpreter is None: interpreter = self.DEFAULT_PYTHON argv = [interpreter, "./setup.py"] + args env = {} # Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment - if 'SETUPTOOLS_SCM_PRETEND_VERSION' in os.environ: - env['SETUPTOOLS_SCM_PRETEND_VERSION'] = ( - os.environ['SETUPTOOLS_SCM_PRETEND_VERSION']) + if "SETUPTOOLS_SCM_PRETEND_VERSION" in os.environ: + env["SETUPTOOLS_SCM_PRETEND_VERSION"] = os.environ[ + "SETUPTOOLS_SCM_PRETEND_VERSION" + ] run_with_build_fixers(session, argv, fixers, env=env) def _setup_requires(self): if self.pyproject: if "build-system" in self.pyproject: - for require in self.pyproject['build-system'].get("requires", []): + for require in self.pyproject["build-system"].get("requires", []): yield PythonPackageRequirement.from_requirement_str(require) if self.config: - options = self.config.get('options', {}) - for require in options.get('setup_requires', []): + options = self.config.get("options", {}) + for require in options.get("setup_requires", []): yield PythonPackageRequirement.from_requirement_str(require) def get_declared_dependencies(self, session, fixers=None): distribution = self._extract_setup(session, fixers) if distribution is not None: - for require in distribution['requires']: + for require in distribution["requires"]: yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - for require in distribution['setup_requires']: + for require in distribution["setup_requires"]: yield "build", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - for require in distribution['install_requires']: + for require in distribution["install_requires"]: yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - for require in distribution['tests_require']: + for require in distribution["tests_require"]: yield "test", PythonPackageRequirement.from_requirement_str(require) if self.pyproject: if "build-system" in self.pyproject: - for require in self.pyproject['build-system'].get("requires", []): - yield "build", PythonPackageRequirement.from_requirement_str(require) + for require in self.pyproject["build-system"].get("requires", []): + yield "build", PythonPackageRequirement.from_requirement_str( + require + ) if self.config: - options = self.config.get('options', {}) - for require in options.get('setup_requires', []): + options = self.config.get("options", {}) + for require in options.get("setup_requires", []): yield "build", PythonPackageRequirement.from_requirement_str(require) - for require in options.get('install_requires', []): + for require in options.get("install_requires", []): yield "core", PythonPackageRequirement.from_requirement_str(require) def get_declared_outputs(self, session, fixers=None): distribution = self._extract_setup(session, fixers) all_packages = set() if distribution is not None: - for script in distribution['scripts']: + for script in distribution["scripts"]: yield BinaryOutput(os.path.basename(script)) for script in distribution["entry_points"].get("console_scripts", []): yield BinaryOutput(script.split("=")[0]) - all_packages.update(distribution['packages']) + all_packages.update(distribution["packages"]) if self.config: - options = self.config.get('options', {}) - all_packages.update(options.get('packages', [])) - for script in options.get('scripts', []): + options = self.config.get("options", {}) + all_packages.update(options.get("packages", [])) + for script in options.get("scripts", []): yield BinaryOutput(os.path.basename(script)) for script in options.get("entry_points", {}).get("console_scripts", []): yield BinaryOutput(script.split("=")[0]) packages = set() for package in sorted(all_packages): - pts = package.split('.') + pts = package.split(".") b = [] for e in pts: b.append(e) - if '.'.join(b) in packages: + if ".".join(b) in packages: break else: packages.add(package) @@ -521,12 +549,12 @@ class Octave(BuildSystem): return False # Urgh, isn't there a better way to see if this is an octave package? for entry in os.scandir(path): - if entry.name.endswith('.m'): + if entry.name.endswith(".m"): return True if not entry.is_dir(): continue for subentry in os.scandir(entry.path): - if subentry.name.endswith('.m'): + if subentry.name.endswith(".m"): return True return False @@ -537,17 +565,19 @@ class Octave(BuildSystem): return cls(path) def _read_description(self): - path = os.path.join(self.path, 'DESCRIPTION') + path = os.path.join(self.path, "DESCRIPTION") from email.parser import BytesParser - with open(path, 'rb') as f: + + with open(path, "rb") as f: return BytesParser().parse(f) def get_declared_dependencies(self, session, fixers=None): def parse_list(t): - return [s.strip() for s in t.split(',') if s.strip()] + return [s.strip() for s in t.split(",") if s.strip()] + description = self._read_description() - if 'Depends' in description: - for s in parse_list(description['Depends']): + if "Depends" in description: + for s in parse_list(description["Depends"]): yield "build", OctavePackageRequirement.from_str(s) @@ -564,14 +594,14 @@ class Gradle(BuildSystem): @classmethod def exists(cls, path): - return ( - os.path.exists(os.path.join(path, "build.gradle")) or - os.path.exists(os.path.join(path, "build.gradle.kts"))) + return os.path.exists(os.path.join(path, "build.gradle")) or os.path.exists( + os.path.join(path, "build.gradle.kts") + ) @classmethod def from_path(cls, path): if os.path.exists(os.path.join(path, "gradlew")): - return cls(path, './gradlew') + return cls(path, "./gradlew") return cls(path) @classmethod @@ -581,44 +611,50 @@ class Gradle(BuildSystem): return cls.from_path(path) def setup(self, resolver): - if not self.executable.startswith('./'): + if not self.executable.startswith("./"): resolver.install([BinaryRequirement(self.executable)]) def _run(self, session, resolver, task, args, fixers): self.setup(resolver) argv = [] - if self.executable.startswith('./') and ( - not os.access(os.path.join(self.path, self.executable), os.X_OK)): - argv.append('sh') + if self.executable.startswith("./") and ( + not os.access(os.path.join(self.path, self.executable), os.X_OK) + ): + argv.append("sh") argv.extend([self.executable, task]) argv.extend(args) try: run_with_build_fixers(session, argv, fixers) except UnidentifiedError as e: - if any([re.match( - r"Task '" + task + "' not found in root project '.*'\.", - line) for line in e.lines]): + if any( + [ + re.match( + r"Task '" + task + r"' not found in root project '.*'\.", line + ) + for line in e.lines + ] + ): raise NotImplementedError raise def clean(self, session, resolver, fixers): - self._run(session, resolver, 'clean', [], fixers) + self._run(session, resolver, "clean", [], fixers) def build(self, session, resolver, fixers): - self._run(session, resolver, 'build', [], fixers) + self._run(session, resolver, "build", [], fixers) def test(self, session, resolver, fixers): - self._run(session, resolver, 'test', [], fixers) + self._run(session, resolver, "test", [], fixers) def dist(self, session, resolver, fixers, target_directory, quiet=False): - with DistCatcher([session.external_path('.')]) as dc: - self._run(session, resolver, 'distTar', [], fixers) + with DistCatcher([session.external_path(".")]) as dc: + self._run(session, resolver, "distTar", [], fixers) return dc.copy_single(target_directory) def install(self, session, resolver, fixers, install_target): raise NotImplementedError # TODO(jelmer): installDist just creates files under build/install/... - self._run(session, resolver, 'installDist', [], fixers) + self._run(session, resolver, "installDist", [], fixers) class R(BuildSystem): @@ -638,7 +674,7 @@ class R(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): r_path = guaranteed_which(session, resolver, "R") - with DistCatcher([session.external_path('.')]) as dc: + with DistCatcher([session.external_path(".")]) as dc: run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) return dc.copy_single(target_directory) @@ -652,34 +688,37 @@ class R(BuildSystem): @classmethod def probe(cls, path): - if (os.path.exists(os.path.join(path, 'DESCRIPTION')) and - os.path.exists(os.path.join(path, 'NAMESPACE'))): + if os.path.exists(os.path.join(path, "DESCRIPTION")) and os.path.exists( + os.path.join(path, "NAMESPACE") + ): return cls(path) def _read_description(self): - path = os.path.join(self.path, 'DESCRIPTION') + path = os.path.join(self.path, "DESCRIPTION") from email.parser import BytesParser - with open(path, 'rb') as f: + + with open(path, "rb") as f: return BytesParser().parse(f) def get_declared_dependencies(self, session, fixers=None): def parse_list(t): - return [s.strip() for s in t.split(',') if s.strip()] + return [s.strip() for s in t.split(",") if s.strip()] + description = self._read_description() - if 'Suggests' in description: - for s in parse_list(description['Suggests']): + if "Suggests" in description: + for s in parse_list(description["Suggests"]): yield "build", RPackageRequirement.from_str(s) - if 'Depends' in description: - for s in parse_list(description['Depends']): + if "Depends" in description: + for s in parse_list(description["Depends"]): yield "build", RPackageRequirement.from_str(s) - if 'Imports' in description: - for s in parse_list(description['Imports']): + if "Imports" in description: + for s in parse_list(description["Imports"]): yield "build", RPackageRequirement.from_str(s) def get_declared_outputs(self, session, fixers=None): description = self._read_description() - if 'Package' in description: - yield RPackageOutput(description['Package']) + if "Package" in description: + yield RPackageOutput(description["Package"]) class Meson(BuildSystem): @@ -693,8 +732,8 @@ class Meson(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def _setup(self, session, fixers): - if not session.exists('build'): - session.check_call(['mkdir', 'build']) + if not session.exists("build"): + session.check_call(["mkdir", "build"]) run_with_build_fixers(session, ["meson", "setup", "build"], fixers) def clean(self, session, resolver, fixers): @@ -707,7 +746,7 @@ class Meson(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): self._setup(session, fixers) - with DistCatcher([session.external_path('build/meson-dist')]) as dc: + with DistCatcher([session.external_path("build/meson-dist")]) as dc: run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) return dc.copy_single(target_directory) @@ -742,7 +781,7 @@ class Npm(BuildSystem): return "%s(%r)" % (type(self).__name__, self.path) def get_declared_dependencies(self, session, fixers=None): - if 'dependencies' in self.package: + if "dependencies" in self.package: for name, unused_version in self.package["dependencies"].items(): # TODO(jelmer): Look at version yield "core", NodePackageRequirement(name) @@ -756,13 +795,13 @@ class Npm(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(resolver) - with DistCatcher([session.external_path('.')]) as dc: + with DistCatcher([session.external_path(".")]) as dc: run_with_build_fixers(session, ["npm", "pack"], fixers) return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(resolver) - test_script = self.package['scripts'].get('test') + test_script = self.package["scripts"].get("test") if test_script: run_with_build_fixers(session, shlex.split(test_script), fixers) else: @@ -770,7 +809,7 @@ class Npm(BuildSystem): def build(self, session, resolver, fixers): self.setup(resolver) - build_script = self.package['scripts'].get('build') + build_script = self.package["scripts"].get("build") if build_script: run_with_build_fixers(session, shlex.split(build_script), fixers) else: @@ -778,7 +817,7 @@ class Npm(BuildSystem): def clean(self, session, resolver, fixers): self.setup(resolver) - clean_script = self.package['scripts'].get('clean') + clean_script = self.package["scripts"].get("clean") if clean_script: run_with_build_fixers(session, shlex.split(clean_script), fixers) else: @@ -803,7 +842,7 @@ class Waf(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) - with DistCatcher.default(session.external_path('.')) as dc: + with DistCatcher.default(session.external_path(".")) as dc: run_with_build_fixers(session, ["./waf", "dist"], fixers) return dc.copy_single(target_directory) @@ -835,13 +874,15 @@ class Gem(BuildSystem): ] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") - with DistCatcher.default(session.external_path('.')) as dc: + with DistCatcher.default(session.external_path(".")) as dc: run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) return dc.copy_single(target_directory) @classmethod def probe(cls, path): - gemfiles = [entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")] + gemfiles = [ + entry.path for entry in os.scandir(path) if entry.name.endswith(".gem") + ] if gemfiles: return cls(gemfiles[0]) @@ -881,13 +922,13 @@ class DistZilla(BuildSystem): self.setup(resolver) if self.name == "dist-inkt": resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) - with DistCatcher.default(session.external_path('.')) as dc: + with DistCatcher.default(session.external_path(".")) as dc: run_with_build_fixers(session, ["distinkt-dist"], fixers) return dc.copy_single(target_directory) else: # Default to invoking Dist::Zilla resolver.install([PerlModuleRequirement("Dist::Zilla")]) - with DistCatcher.default(session.external_path('.')) as dc: + with DistCatcher.default(session.external_path(".")) as dc: run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers) return dc.copy_single(target_directory) @@ -939,13 +980,13 @@ class RunTests(BuildSystem): def _read_cpanfile(session, args, kind, fixers): - for line in run_with_build_fixers(session, ['cpanfile-dump'] + args, fixers): + for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers): yield kind, PerlModuleRequirement(line) def _declared_deps_from_cpanfile(session, fixers): - yield from _read_cpanfile(session, ['--configure', '--build'], 'build', fixers) - yield from _read_cpanfile(session, ['--test'], 'test', fixers) + yield from _read_cpanfile(session, ["--configure", "--build"], "build", fixers) + yield from _read_cpanfile(session, ["--test"], "test", fixers) class Make(BuildSystem): @@ -989,7 +1030,9 @@ class Make(BuildSystem): if not makefile_exists() and session.exists("configure"): run_with_build_fixers(session, ["./configure"], fixers) - if not makefile_exists() and any([n.name.endswith('.pro') for n in session.scandir(".")]): + if not makefile_exists() and any( + [n.name.endswith(".pro") for n in session.scandir(".")] + ): run_with_build_fixers(session, ["qmake"], fixers) def build(self, session, resolver, fixers): @@ -1010,7 +1053,7 @@ class Make(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) - with DistCatcher.default(session.external_path('.')) as dc: + with DistCatcher.default(session.external_path(".")) as dc: try: run_with_build_fixers(session, ["make", "dist"], fixers) except UnidentifiedError as e: @@ -1106,7 +1149,7 @@ class Make(BuildSystem): return cls(path) for n in os.scandir(path): # qmake - if n.name.endswith('.pro'): + if n.name.endswith(".pro"): return cls(path) @@ -1133,8 +1176,9 @@ class Cargo(BuildSystem): # TODO(jelmer): Look at details['version'] yield "build", CargoCrateRequirement( name, - features=details.get('features', []), - version=details.get("version")) + features=details.get("features", []), + version=details.get("version"), + ) def test(self, session, resolver, fixers): run_with_build_fixers(session, ["cargo", "test"], fixers) @@ -1157,19 +1201,20 @@ def _parse_go_mod(f): line = f.readline() if not line: return line - return line.split('//')[0] + '\n' + return line.split("//")[0] + "\n" + line = readline() while line: - parts = line.strip().split(' ') - if not parts or parts == ['']: + parts = line.strip().split(" ") + if not parts or parts == [""]: continue - if len(parts) == 2 and parts[1] == '(': + if len(parts) == 2 and parts[1] == "(": line = readline() - while line.strip() != ')': - yield [parts[0]] + list(line.strip().split(' ')) + while line.strip() != ")": + yield [parts[0]] + list(line.strip().split(" ")) line = readline() if not line: - raise AssertionError('list of %s interrupted?' % parts[0]) + raise AssertionError("list of %s interrupted?" % parts[0]) else: yield parts line = readline() @@ -1199,31 +1244,30 @@ class Golang(BuildSystem): session.check_call(["go", "clean"]) def get_declared_dependencies(self, session, fixers=None): - go_mod_path = os.path.join(self.path, 'go.mod') + go_mod_path = os.path.join(self.path, "go.mod") if os.path.exists(go_mod_path): - with open(go_mod_path, 'r') as f: + with open(go_mod_path, "r") as f: for parts in _parse_go_mod(f): - if parts[0] == 'go': + if parts[0] == "go": yield "build", GoRequirement(parts[1]) - elif parts[0] == 'require': + elif parts[0] == "require": yield "build", GoPackageRequirement( - parts[1], parts[2].lstrip('v') if len(parts) > 2 else None) - elif parts[0] == 'exclude': + parts[1], parts[2].lstrip("v") if len(parts) > 2 else None + ) + elif parts[0] == "exclude": pass # TODO(jelmer): Create conflicts? - elif parts[0] == 'replace': + elif parts[0] == "replace": pass # TODO(jelmer): do.. something? - elif parts[0] == 'module': + elif parts[0] == "module": pass else: - logging.warning( - 'Unknown directive %s in go.mod', - parts[0]) + logging.warning("Unknown directive %s in go.mod", parts[0]) @classmethod def probe(cls, path): - if os.path.exists(os.path.join(path, 'go.mod')): + if os.path.exists(os.path.join(path, "go.mod")): return Golang(path) - if os.path.exists(os.path.join(path, 'go.sum')): + if os.path.exists(os.path.join(path, "go.sum")): return Golang(path) for entry in os.scandir(path): if entry.name.endswith(".go"): @@ -1266,20 +1310,23 @@ class Maven(BuildSystem): def get_declared_dependencies(self, session, fixers=None): import xml.etree.ElementTree as ET + try: - root = xmlparse_simplify_namespaces(self.path, - ['http://maven.apache.org/POM/4.0.0']) + root = xmlparse_simplify_namespaces( + self.path, ["http://maven.apache.org/POM/4.0.0"] + ) except ET.ParseError as e: - logging.warning('Unable to parse package.xml: %s', e) + logging.warning("Unable to parse package.xml: %s", e) return - assert root.tag == 'project', 'root tag is %r' % root.tag - deps_tag = root.find('dependencies') + assert root.tag == "project", "root tag is %r" % root.tag + deps_tag = root.find("dependencies") if deps_tag: - for dep in deps_tag.findall('dependency'): + for dep in deps_tag.findall("dependency"): yield "core", MavenArtifactRequirement( - dep.find('groupId').text, - dep.find('artifactId').text, - dep.find('version').text) + dep.find("groupId").text, + dep.find("artifactId").text, + dep.find("version").text, + ) class Cabal(BuildSystem): @@ -1310,9 +1357,12 @@ class Cabal(BuildSystem): self._run(session, ["test"], fixers) def dist(self, session, resolver, fixers, target_directory, quiet=False): - with DistCatcher([ - session.external_path('dist-newstyle/sdist'), - session.external_path('dist')]) as dc: + with DistCatcher( + [ + session.external_path("dist-newstyle/sdist"), + session.external_path("dist"), + ] + ) as dc: self._run(session, ["sdist"], fixers) return dc.copy_single(target_directory) @@ -1377,23 +1427,38 @@ class PerlBuildTiny(BuildSystem): BUILDSYSTEM_CLSES = [ - Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, - DistZilla, Gem, PerlBuildTiny, Golang, R, Octave, + Pear, + SetupPy, + Npm, + Waf, + Cargo, + Meson, + Cabal, + Gradle, + Maven, + DistZilla, + Gem, + PerlBuildTiny, + Golang, + R, + Octave, # Make is intentionally at the end of the list. - Make, Composer, RunTests] + Make, + Composer, + RunTests, +] def scan_buildsystems(path): """Detect build systems.""" ret = [] - ret.extend([('.', bs) for bs in detect_buildsystems(path)]) + ret.extend([(".", bs) for bs in detect_buildsystems(path)]) if not ret: # Nothing found. Try the next level? for entry in os.scandir(path): if entry.is_dir(): - ret.extend( - [(entry.name, bs) for bs in detect_buildsystems(entry.path)]) + ret.extend([(entry.name, bs) for bs in detect_buildsystems(entry.path)]) return ret diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index c435b56..7c168aa 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -26,10 +26,17 @@ from buildlog_consultant.apt import ( from .. import DetailedFailure, UnidentifiedError from ..session import Session, run_with_tee, get_user -from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_packages_for_paths +from .file_search import ( + FileSearcher, + AptCachedContentsFileSearcher, + GENERATED_FILE_SEARCHER, + get_packages_for_paths, +) -def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None: +def run_apt( + session: Session, args: List[str], prefix: Optional[List[str]] = None +) -> None: """Run apt.""" if prefix is None: prefix = [] @@ -84,7 +91,9 @@ class AptManager(object): def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): logging.debug("Searching for packages containing %r", paths) # TODO(jelmer): Make sure we use whatever is configured in self.session - return get_packages_for_paths(paths, self.searchers(), regex=regex, case_insensitive=case_insensitive) + return get_packages_for_paths( + paths, self.searchers(), regex=regex, case_insensitive=case_insensitive + ) def missing(self, packages): root = getattr(self.session, "location", "/") diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index ea5a40a..cca50fb 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -33,7 +33,6 @@ import sys from debian.changelog import Changelog from debmutate.changelog import get_maintainer, format_datetime -from breezy import osutils from breezy.mutabletree import MutableTree from breezy.plugins.debian.builder import BuildFailedError from breezy.tree import Tree diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index 5a5f811..888325f 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -22,7 +22,6 @@ import logging class BuildDependencyTieBreaker(object): - def __init__(self, rootdir): self.rootdir = rootdir self._counts = None @@ -37,8 +36,9 @@ class BuildDependencyTieBreaker(object): def _count(self): counts = {} import apt_pkg + apt_pkg.init() - apt_pkg.config.set('Dir', self.rootdir) + apt_pkg.config.set("Dir", self.rootdir) apt_cache = apt_pkg.SourceRecords() apt_cache.restart() while apt_cache.step(): @@ -65,17 +65,20 @@ class BuildDependencyTieBreaker(object): return None top = max(by_count.items(), key=lambda k: k[1]) logging.info( - 'Breaking tie between %r to %r based on build-depends count', - [repr(r) for r in reqs], top[0]) + "Breaking tie between %r to %r based on build-depends count", + [repr(r) for r in reqs], + top[0], + ) return top[0] -if __name__ == '__main__': +if __name__ == "__main__": import argparse from ..resolver.apt import AptRequirement + parser = argparse.ArgumentParser() - parser.add_argument('req', nargs='+') + parser.add_argument("req", nargs="+") args = parser.parse_args() reqs = [AptRequirement.from_str(req) for req in args.req] - tie_breaker = BuildDependencyTieBreaker('/') + tie_breaker = BuildDependencyTieBreaker("/") print(tie_breaker(reqs)) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index 429852e..d0d9ff4 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -21,7 +21,7 @@ from datetime import datetime from debian.deb822 import Release import os import re -from typing import Iterator, List, Optional, Set +from typing import Iterator, List import logging @@ -29,7 +29,9 @@ from .. import USER_AGENT class FileSearcher(object): - def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]: + def search_files( + self, path: str, regex: bool = False, case_insensitive: bool = False + ) -> Iterator[str]: raise NotImplementedError(self.search_files) @@ -66,14 +68,16 @@ def contents_urls_from_sources_entry(source, arches, load_url): try: response = load_url(release_url) except FileNotFoundError as e: - logging.warning('Unable to download %s or %s: %s', inrelease_url, release_url, e) + logging.warning( + "Unable to download %s or %s: %s", inrelease_url, release_url, e + ) return existing_names = {} release = Release(response.read()) - for hn in ['MD5Sum', 'SHA1Sum', 'SHA256Sum']: + for hn in ["MD5Sum", "SHA1Sum", "SHA256Sum"]: for entry in release.get(hn, []): - existing_names[os.path.splitext(entry['name'])[0]] = entry['name'] + existing_names[os.path.splitext(entry["name"])[0]] = entry["name"] contents_files = set() if components: @@ -117,8 +121,7 @@ def load_direct_url(url): for ext in [".xz", ".gz", ""]: try: - request = Request( - url + ext, headers={"User-Agent": USER_AGENT}) + request = Request(url + ext, headers={"User-Agent": USER_AGENT}) response = urlopen(request) except HTTPError as e: if e.status == 404: @@ -141,16 +144,17 @@ def load_url_with_cache(url, cache_dirs): def load_apt_cache_file(url, cache_dir): fn = apt_pkg.uri_to_filename(url) - for ext in ['.xz', '.gz', '.lz4', '']: + for ext in [".xz", ".gz", ".lz4", ""]: p = os.path.join(cache_dir, fn + ext) if not os.path.exists(p): continue # return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) logging.debug("Loading cached contents file %s", p) - if ext == '.lz4': + if ext == ".lz4": import lz4.frame + return lz4.frame.open(p, mode="rb") - return _unwrap(open(p, 'rb'), ext) + return _unwrap(open(p, "rb"), ext) raise FileNotFoundError(url) @@ -174,14 +178,15 @@ class AptCachedContentsFileSearcher(FileSearcher): sl.load("/etc/apt/sources.list") from .build import get_build_architecture + cache_dirs = set(["/var/lib/apt/lists"]) def load_url(url): return load_url_with_cache(url, cache_dirs) urls = list( - contents_urls_from_sourceslist( - sl, get_build_architecture(), load_url)) + contents_urls_from_sourceslist(sl, get_build_architecture(), load_url) + ) self._load_urls(urls, cache_dirs, load_url) def load_from_session(self, session): @@ -193,16 +198,19 @@ class AptCachedContentsFileSearcher(FileSearcher): from .build import get_build_architecture - cache_dirs = set([ - os.path.join(session.location, "var/lib/apt/lists"), - "/var/lib/apt/lists", - ]) + cache_dirs = set( + [ + os.path.join(session.location, "var/lib/apt/lists"), + "/var/lib/apt/lists", + ] + ) def load_url(url): return load_url_with_cache(url, cache_dirs) urls = list( - contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)) + contents_urls_from_sourceslist(sl, get_build_architecture(), load_url) + ) self._load_urls(urls, cache_dirs, load_url) def _load_urls(self, urls, cache_dirs, load_url): @@ -217,7 +225,7 @@ class AptCachedContentsFileSearcher(FileSearcher): self._db[path] = package def search_files(self, path, regex=False, case_insensitive=False): - path = path.lstrip('/').encode('utf-8', 'surrogateescape') + path = path.lstrip("/").encode("utf-8", "surrogateescape") if case_insensitive and not regex: regex = True path = re.escape(path) @@ -230,12 +238,12 @@ class AptCachedContentsFileSearcher(FileSearcher): for p, rest in self._db.items(): if c.match(p): pkg = rest.split(b"/")[-1] - ret.append((p, pkg.decode('utf-8'))) + ret.append((p, pkg.decode("utf-8"))) for p, pkg in sorted(ret): yield pkg else: try: - yield self._db[path].split(b"/")[-1].decode('utf-8') + yield self._db[path].split(b"/")[-1].decode("utf-8") except KeyError: pass @@ -243,7 +251,7 @@ class AptCachedContentsFileSearcher(FileSearcher): start_time = datetime.now() for path, rest in read_contents_file(f.readlines()): self[path] = rest - logging.debug('Read %s in %s', url, datetime.now() - start_time) + logging.debug("Read %s in %s", url, datetime.now() - start_time) class GeneratedFileSearcher(FileSearcher): @@ -257,12 +265,14 @@ class GeneratedFileSearcher(FileSearcher): return self def load_from_path(self, path): - with open(path, 'r') as f: + with open(path, "r") as f: for line in f: (path, pkg) = line.strip().split(None, 1) self._db[path] = pkg - def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]: + def search_files( + self, path: str, regex: bool = False, case_insensitive: bool = False + ) -> Iterator[str]: for p, pkg in sorted(self._db.items()): if regex: flags = 0 @@ -294,13 +304,17 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher( def get_packages_for_paths( - paths: List[str], searchers: List[FileSearcher], regex: bool = False, - case_insensitive: bool = False + paths: List[str], + searchers: List[FileSearcher], + regex: bool = False, + case_insensitive: bool = False, ) -> List[str]: candidates: List[str] = list() for path in paths: for searcher in searchers: - for pkg in searcher.search_files(path, regex=regex, case_insensitive=case_insensitive): + for pkg in searcher.search_files( + path, regex=regex, case_insensitive=case_insensitive + ): if pkg not in candidates: candidates.append(pkg) return candidates @@ -308,10 +322,11 @@ def get_packages_for_paths( def main(argv): import argparse + parser = argparse.ArgumentParser() - parser.add_argument('path', help='Path to search for.', type=str, nargs='*') - parser.add_argument('--regex', '-x', help='Search for regex.', action='store_true') - parser.add_argument('--debug', action='store_true') + parser.add_argument("path", help="Path to search for.", type=str, nargs="*") + parser.add_argument("--regex", "-x", help="Search for regex.", action="store_true") + parser.add_argument("--debug", action="store_true") args = parser.parse_args() if args.debug: @@ -328,6 +343,7 @@ def main(argv): print(package) -if __name__ == '__main__': +if __name__ == "__main__": import sys + sys.exit(main(sys.argv)) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 9635a8f..b6b2e6f 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -19,13 +19,12 @@ __all__ = [ "build_incrementally", ] -from datetime import datetime from functools import partial import logging import os import shutil import sys -from typing import List, Set, Optional, Type, Tuple +from typing import List, Set, Optional, Type from debian.deb822 import ( Deb822, @@ -33,7 +32,6 @@ from debian.deb822 import ( ) from breezy.commit import PointlessCommit -from breezy.mutabletree import MutableTree from breezy.tree import Tree from debmutate.changelog import ChangelogEditor from debmutate.control import ( @@ -54,18 +52,20 @@ from debmutate.reformatting import ( try: from breezy.workspace import reset_tree except ImportError: # breezy < 3.2 + def delete_items(deletables, dry_run=False): """Delete files in the deletables iterable""" import errno import shutil + def onerror(function, path, excinfo): - """Show warning for errors seen by rmtree. - """ + """Show warning for errors seen by rmtree.""" # Handle only permission error while removing files. # Other errors are re-raised. if function is not os.remove or excinfo[1].errno != errno.EACCES: raise - logging.warning('unable to remove %s' % path) + logging.warning("unable to remove %s" % path) + for path, subp in deletables: if os.path.isdir(path): shutil.rmtree(path, onerror=onerror) @@ -78,13 +78,18 @@ except ImportError: # breezy < 3.2 raise e logging.warning('unable to remove "%s": %s.', path, e.strerror) - def reset_tree(local_tree, subpath=''): + def reset_tree(local_tree, subpath=""): from breezy.transform import revert from breezy.clean_tree import iter_deletables - revert(local_tree, local_tree.branch.basis_tree(), - [subpath] if subpath not in ('.', '') else None) - deletables = list(iter_deletables( - local_tree, unknown=True, ignored=False, detritus=False)) + + revert( + local_tree, + local_tree.branch.basis_tree(), + [subpath] if subpath not in (".", "") else None, + ) + deletables = list( + iter_deletables(local_tree, unknown=True, ignored=False, detritus=False) + ) delete_items(deletables) @@ -103,8 +108,6 @@ from buildlog_consultant.common import ( MissingAutomakeInput, MissingConfigure, NeedPgBuildExtUpdateControl, - MissingPythonModule, - MissingPythonDistribution, MissingPerlFile, ) from buildlog_consultant.sbuild import ( @@ -130,8 +133,9 @@ class CircularDependency(Exception): class DebianPackagingContext(object): - - def __init__(self, tree, subpath, committer, update_changelog, commit_reporter=None): + def __init__( + self, tree, subpath, committer, update_changelog, commit_reporter=None + ): self.tree = tree self.subpath = subpath self.committer = committer @@ -144,14 +148,18 @@ class DebianPackagingContext(object): with self.tree.lock_write(): try: if update_changelog: - cl_path = self.tree.abspath(os.path.join(self.subpath, "debian/changelog")) + cl_path = self.tree.abspath( + os.path.join(self.subpath, "debian/changelog") + ) with ChangelogEditor(cl_path) as editor: editor.add_entry([summary]) debcommit(self.tree, committer=self.committer, subpath=self.subpath) else: self.tree.commit( - message=summary, committer=self.committer, specific_files=[self.subpath], - reporter=self.commit_reporter + message=summary, + committer=self.committer, + specific_files=[self.subpath], + reporter=self.commit_reporter, ) except PointlessCommit: return False @@ -160,7 +168,6 @@ class DebianPackagingContext(object): class PackageDependencyFixer(BuildFixer): - def __init__(self, context, apt_resolver): self.apt_resolver = apt_resolver self.context = context @@ -203,7 +210,7 @@ def add_dependency(context, phase, requirement: AptRequirement): elif phase[0] == "build": return add_build_dependency(context, requirement) else: - logging.warning('Unknown phase %r', phase) + logging.warning("Unknown phase %r", phase) return False @@ -239,7 +246,9 @@ def add_test_dependency(context, testname, requirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - tests_control_path = os.path.join(context.tree.abspath(context.subpath), "debian/tests/control") + tests_control_path = os.path.join( + context.tree.abspath(context.subpath), "debian/tests/control" + ) try: with Deb822Editor(path=tests_control_path) as updater: @@ -293,18 +302,22 @@ def python_tie_breaker(tree, subpath, reqs): return None def same(pkg, python_version): - if pkg.startswith(python_version + '-'): + if pkg.startswith(python_version + "-"): return True - if pkg.startswith('lib%s-' % python_version): + if pkg.startswith("lib%s-" % python_version): return True return False + for python_version in targeted: for req in reqs: if any(same(name, python_version) for name in req.package_names()): logging.info( - 'Breaking tie between %r to %r, since package already ' - 'has %r build-dependencies', [str(req) for req in reqs], - str(req), python_version) + "Breaking tie between %r to %r, since package already " + "has %r build-dependencies", + [str(req) for req in reqs], + str(req), + python_version, + ) return req return None @@ -327,7 +340,9 @@ def enable_dh_autoreconf(context, phase): return dh_invoke_add_with(line, b"autoreconf") if update_rules(command_line_cb=add_with_autoreconf): - return add_dependency(context, phase, AptRequirement.simple("dh-autoreconf")) + return add_dependency( + context, phase, AptRequirement.simple("dh-autoreconf") + ) return False @@ -379,16 +394,18 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer): return isinstance(problem, NeedPgBuildExtUpdateControl) def __repr__(self): - return "%s()" % (type(self).__name__, ) + return "%s()" % (type(self).__name__,) def _fix(self, error, context): logging.info("Running 'pg_buildext updatecontrol'") self.session.check_call(["pg_buildext", "updatecontrol"]) shutil.copy( - self.session.external_path('debian/control'), - context.tree.abspath(os.path.join(context.subpath, 'debian/control'))) + self.session.external_path("debian/control"), + context.tree.abspath(os.path.join(context.subpath, "debian/control")), + ) return self.context.commit( - "Run 'pgbuildext updatecontrol'.", update_changelog=False) + "Run 'pgbuildext updatecontrol'.", update_changelog=False + ) def fix_missing_makefile_pl(error, phase, context): @@ -410,7 +427,10 @@ class SimpleBuildFixer(BuildFixer): def __repr__(self): return "%s(%s, %s)" % ( - type(self).__name__, self._problem_cls.__name__, self._fn.__name__) + type(self).__name__, + self._problem_cls.__name__, + self._fn.__name__, + ) def can_fix(self, problem: Problem): return isinstance(problem, self._problem_cls) @@ -428,7 +448,10 @@ class DependencyBuildFixer(BuildFixer): def __repr__(self): return "%s(%s, %s)" % ( - type(self).__name__, self._problem_cls.__name__, self._fn.__name__) + type(self).__name__, + self._problem_cls.__name__, + self._fn.__name__, + ) def can_fix(self, problem: Problem): return isinstance(problem, self._problem_cls) @@ -441,8 +464,12 @@ def versioned_package_fixers(session, packaging_context): return [ PgBuildExtOutOfDateControlFixer(packaging_context, session), SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure), - SimpleBuildFixer(packaging_context, MissingAutomakeInput, fix_missing_automake_input), - SimpleBuildFixer(packaging_context, MissingConfigStatusInput, fix_missing_config_status_input), + SimpleBuildFixer( + packaging_context, MissingAutomakeInput, fix_missing_automake_input + ), + SimpleBuildFixer( + packaging_context, MissingConfigStatusInput, fix_missing_config_status_input + ), SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl), ] @@ -451,14 +478,17 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]: from ..resolver.apt import AptResolver from .udd import popcon_tie_breaker from .build_deps import BuildDependencyTieBreaker + apt_tie_breakers = [ partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), BuildDependencyTieBreaker.from_session(apt.session), popcon_tie_breaker, - ] + ] resolver = AptResolver(apt, apt_tie_breakers) return [ - DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure), + DependencyBuildFixer( + packaging_context, apt, AptFetchFailure, retry_apt_failure + ), PackageDependencyFixer(packaging_context, resolver), ] @@ -479,9 +509,11 @@ def build_incrementally( ): fixed_errors = [] packaging_context = DebianPackagingContext( - local_tree, subpath, committer, update_changelog) - fixers = (versioned_package_fixers(apt.session, packaging_context) + - apt_fixers(apt, packaging_context)) + local_tree, subpath, committer, update_changelog + ) + fixers = versioned_package_fixers(apt.session, packaging_context) + apt_fixers( + apt, packaging_context + ) logging.info("Using fixers: %r", fixers) while True: try: @@ -534,11 +566,8 @@ def build_incrementally( ): i += 1 target_path = os.path.join(output_directory, "build.log.%d" % i) - os.rename( - os.path.join(output_directory, "build.log"), - target_path - ) - logging.debug('Storing build log at %s', target_path) + os.rename(os.path.join(output_directory, "build.log"), target_path) + logging.debug("Storing build log at %s", target_path) def main(argv=None): @@ -577,19 +606,13 @@ def main(argv=None): help="force updating of the changelog", default=None, ) - parser.add_argument( - '--schroot', - type=str, - help='chroot to use.') - parser.add_argument( - '--verbose', - action='store_true', - help='Be verbose') + parser.add_argument("--schroot", type=str, help="chroot to use.") + parser.add_argument("--verbose", action="store_true", help="Be verbose") args = parser.parse_args() from breezy.workingtree import WorkingTree - import breezy.git - import breezy.bzr + import breezy.git # noqa: F401 + import breezy.bzr # noqa: F401 from .apt import AptManager from ..session.plain import PlainSession from ..session.schroot import SchrootSession @@ -632,15 +655,15 @@ def main(argv=None): ) except SbuildFailure as e: if e.phase is None: - phase = 'unknown phase' + phase = "unknown phase" elif len(e.phase) == 1: phase = e.phase[0] else: - phase = '%s (%s)' % (e.phase[0], e.phase[1]) + phase = "%s (%s)" % (e.phase[0], e.phase[1]) if e.error: - logging.fatal('Error during %s: %s', phase, e.error) + logging.fatal("Error during %s: %s", phase, e.error) else: - logging.fatal('Error during %s: %s', phase, e.description) + logging.fatal("Error during %s: %s", phase, e.description) return 1 diff --git a/ognibuild/debian/udd.py b/ognibuild/debian/udd.py index 223eb3c..0c73818 100644 --- a/ognibuild/debian/udd.py +++ b/ognibuild/debian/udd.py @@ -21,7 +21,6 @@ import logging class UDD(object): - def connect(self): import psycopg2 @@ -36,8 +35,9 @@ class UDD(object): def get_most_popular(self, packages): cursor = self._conn.cursor() cursor.execute( - 'SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1', - (tuple(packages), )) + "SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1", + (tuple(packages),), + ) return cursor.fetchone()[0] @@ -47,15 +47,14 @@ def popcon_tie_breaker(candidates): try: from .udd import UDD except ModuleNotFoundError: - logging.warning('Unable to import UDD, not ranking by popcon') + logging.warning("Unable to import UDD, not ranking by popcon") return sorted(candidates, key=len)[0] udd = UDD() udd.connect() names = {list(c.package_names())[0]: c for c in candidates} winner = udd.get_most_popular(list(names.keys())) if winner is None: - logging.warning( - 'No relevant popcon information found, not ranking by popcon') + logging.warning("No relevant popcon information found, not ranking by popcon") return None - logging.info('Picked winner using popcon') + logging.info("Picked winner using popcon") return names[winner] diff --git a/ognibuild/dist.py b/ognibuild/dist.py index aa23aa9..cdf1265 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -16,16 +16,15 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA __all__ = [ - 'UnidentifiedError', - 'DetailedFailure', - 'create_dist', - 'create_dist_schroot', - ] + "UnidentifiedError", + "DetailedFailure", + "create_dist", + "create_dist_schroot", +] import errno import logging import os -import shutil import sys from typing import Optional, List @@ -54,7 +53,8 @@ def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=Fa for buildsystem in buildsystems: filename = buildsystem.dist( - session, resolver, fixers, target_directory, quiet=quiet) + session, resolver, fixers, target_directory, quiet=quiet + ) return filename raise NoBuildToolsFound() @@ -66,20 +66,23 @@ def create_dist( target_dir: str, include_controldir: bool = True, subdir: Optional[str] = None, - cleanup: bool = False + cleanup: bool = False, ) -> Optional[str]: from .buildsystem import detect_buildsystems from .buildlog import InstallFixer from .fix_build import BuildFixer from .fixers import ( - GitIdentityFixer, SecretGpgKeyFixer, - UnexpandedAutoconfMacroFixer, ) + GitIdentityFixer, + SecretGpgKeyFixer, + UnexpandedAutoconfMacroFixer, + ) if subdir is None: subdir = "package" try: export_directory, reldir = session.setup_from_vcs( - tree, include_controldir=include_controldir, subdir=subdir) + tree, include_controldir=include_controldir, subdir=subdir + ) except OSError as e: if e.errno == errno.ENOSPC: raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) @@ -94,8 +97,7 @@ def create_dist( if session.is_temporary: # Only muck about with temporary sessions - fixers.extend([ - GitIdentityFixer(session), SecretGpgKeyFixer(session)]) + fixers.extend([GitIdentityFixer(session), SecretGpgKeyFixer(session)]) session.chdir(reldir) return run_dist(session, buildsystems, resolver, fixers, target_dir) @@ -109,7 +111,7 @@ def create_dist_schroot( packaging_subpath: Optional[str] = None, include_controldir: bool = True, subdir: Optional[str] = None, - cleanup: bool = False + cleanup: bool = False, ) -> Optional[str]: with SchrootSession(chroot) as session: if packaging_tree is not None: @@ -117,10 +119,13 @@ def create_dist_schroot( satisfy_build_deps(session, packaging_tree, packaging_subpath) return create_dist( - session, tree, target_dir, + session, + tree, + target_dir, include_controldir=include_controldir, subdir=subdir, - cleanup=cleanup) + cleanup=cleanup, + ) if __name__ == "__main__": @@ -151,8 +156,8 @@ if __name__ == "__main__": ) parser.add_argument("--verbose", action="store_true", help="Be verbose") parser.add_argument( - '--include-controldir', action='store_true', - help='Clone rather than export.') + "--include-controldir", action="store_true", help="Clone rather than export." + ) args = parser.parse_args() @@ -185,15 +190,17 @@ if __name__ == "__main__": logging.info("No build tools found, falling back to simple export.") export(tree, "dist.tar.gz", "tgz", None) except NotImplementedError: - logging.info("Build system does not support dist tarball creation, " - "falling back to simple export.") + logging.info( + "Build system does not support dist tarball creation, " + "falling back to simple export." + ) export(tree, "dist.tar.gz", "tgz", None) except UnidentifiedError as e: - logging.fatal('Unidentified error: %r', e.lines) + logging.fatal("Unidentified error: %r", e.lines) except DetailedFailure as e: - logging.fatal('Identified error during dist creation: %s', e.error) + logging.fatal("Identified error during dist creation: %s", e.error) except DistNoTarball: - logging.fatal('dist operation did not create a tarball') + logging.fatal("dist operation did not create a tarball") else: logging.info("Created %s", ret) sys.exit(0) diff --git a/ognibuild/dist_catcher.py b/ognibuild/dist_catcher.py index d9d9bca..2739773 100644 --- a/ognibuild/dist_catcher.py +++ b/ognibuild/dist_catcher.py @@ -53,17 +53,17 @@ class DistCatcher(object): @classmethod def default(cls, directory): - return cls([ - os.path.join(directory, 'dist'), - directory, - os.path.join(directory, '..')]) + return cls( + [os.path.join(directory, "dist"), directory, os.path.join(directory, "..")] + ) def __enter__(self): self.existing_files = {} for directory in self.directories: try: self.existing_files[directory] = { - entry.name: entry for entry in os.scandir(directory)} + entry.name: entry for entry in os.scandir(directory) + } except FileNotFoundError: self.existing_files[directory] = {} return self @@ -92,7 +92,8 @@ class DistCatcher(object): return entry.name elif len(possible_new) > 1: logging.warning( - "Found multiple tarballs %r in %s.", possible_new, directory) + "Found multiple tarballs %r in %s.", possible_new, directory + ) return if len(possible_updated) == 1: diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index 3411cb2..e83b84b 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -17,17 +17,15 @@ from functools import partial import logging -from typing import List, Optional, Tuple, Callable, Any +from typing import List, Tuple, Callable, Any from buildlog_consultant import Problem from buildlog_consultant.common import ( find_build_failure_description, MissingCommand, ) -from breezy.mutabletree import MutableTree from . import DetailedFailure, UnidentifiedError -from .debian.apt import AptManager from .session import Session, run_with_tee @@ -55,7 +53,7 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs): else: if retcode == 0: return contents - lines = ''.join(contents).splitlines(False) + lines = "".join(contents).splitlines(False) match, error = find_build_failure_description(lines) if error is None: if match: @@ -91,22 +89,26 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): try: resolved = resolve_error(f.error, None, fixers=fixers) except DetailedFailure as n: - logging.info('New error %r while resolving %r', n, f) + logging.info("New error %r while resolving %r", n, f) if n in to_resolve: raise to_resolve.append(f) to_resolve.append(n) else: if not resolved: - logging.warning("Failed to find resolution for error %r. Giving up.", f.error) + logging.warning( + "Failed to find resolution for error %r. Giving up.", f.error + ) raise f fixed_errors.append(f.error) -def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): +def run_with_build_fixers( + session: Session, args: List[str], fixers: List[BuildFixer], **kwargs +): return iterate_with_build_fixers( - fixers, - partial(run_detecting_problems, session, args, **kwargs)) + fixers, partial(run_detecting_problems, session, args, **kwargs) + ) def resolve_error(error, phase, fixers): diff --git a/ognibuild/fixers.py b/ognibuild/fixers.py index b9e9734..c413f20 100644 --- a/ognibuild/fixers.py +++ b/ognibuild/fixers.py @@ -23,7 +23,7 @@ from buildlog_consultant.common import ( MissingGitIdentity, MissingSecretGpgKey, MissingAutoconfMacro, - ) +) from ognibuild.requirements import AutoconfMacroRequirement from ognibuild.resolver import UnsatisfiedRequirements @@ -31,7 +31,6 @@ from .fix_build import BuildFixer class GitIdentityFixer(BuildFixer): - def __init__(self, session): self.session = session @@ -39,16 +38,17 @@ class GitIdentityFixer(BuildFixer): return isinstance(problem, MissingGitIdentity) def _fix(self, problem: Problem, phase: Tuple[str, ...]): - for name in ['user.email', 'user.name']: - value = subprocess.check_output( - ['git', 'config', '--global', name]).decode().strip() - self.session.check_call( - ['git', 'config', '--global', name, value]) + for name in ["user.email", "user.name"]: + value = ( + subprocess.check_output(["git", "config", "--global", name]) + .decode() + .strip() + ) + self.session.check_call(["git", "config", "--global", name, value]) return True class SecretGpgKeyFixer(BuildFixer): - def __init__(self, session): self.session = session @@ -67,8 +67,10 @@ Expire-Date: 0 Passphrase: "" """ p = self.session.Popen( - ['gpg', '--gen-key', '--batch', '/dev/stdin'], - stdin=subprocess.PIPE, stdout=subprocess.PIPE) + ["gpg", "--gen-key", "--batch", "/dev/stdin"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) p.communicate(SCRIPT) if p.returncode == 0: return True @@ -76,7 +78,6 @@ Passphrase: "" class UnexpandedAutoconfMacroFixer(BuildFixer): - def __init__(self, session, resolver): self.session = session self.resolver = resolver @@ -97,6 +98,6 @@ class UnexpandedAutoconfMacroFixer(BuildFixer): return False from .fix_build import run_detecting_problems - run_detecting_problems(self.session, ['autoconf', '-f']) + run_detecting_problems(self.session, ["autoconf", "-f"]) return True diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 3e5cba4..fea0ea4 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -19,7 +19,7 @@ import posixpath import re import subprocess -from typing import Optional, List, Tuple, Set +from typing import Optional, List, Set from . import Requirement @@ -72,19 +72,24 @@ class PythonPackageRequirement(Requirement): cmd = "python3" else: raise NotImplementedError - text = self.package + ','.join([''.join(spec) for spec in self.specs]) + text = self.package + ",".join(["".join(spec) for spec in self.specs]) p = session.Popen( [cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) p.communicate() return p.returncode == 0 class PhpPackageRequirement(Requirement): - - def __init__(self, package: str, channel: Optional[str] = None, - min_version: Optional[str] = None, - max_version: Optional[str] = None): + def __init__( + self, + package: str, + channel: Optional[str] = None, + min_version: Optional[str] = None, + max_version: Optional[str] = None, + ): self.package = package self.channel = channel self.min_version = min_version @@ -92,8 +97,12 @@ class PhpPackageRequirement(Requirement): def __repr__(self): return "%s(%r, %r, %r, %r)" % ( - type(self).__name__, self.package, self.channel, - self.min_version, self.max_version) + type(self).__name__, + self.package, + self.channel, + self.min_version, + self.max_version, + ) class BinaryRequirement(Requirement): @@ -109,8 +118,10 @@ class BinaryRequirement(Requirement): def met(self, session): p = session.Popen( - ["which", self.binary_name], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) + ["which", self.binary_name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) p.communicate() return p.returncode == 0 @@ -144,7 +155,7 @@ class VagueDependencyRequirement(Requirement): self.name = name def expand(self): - if ' ' not in self.name: + if " " not in self.name: yield BinaryRequirement(self.name) yield LibraryRequirement(self.name) yield PkgConfigRequirement(self.name) @@ -153,6 +164,7 @@ class VagueDependencyRequirement(Requirement): yield LibraryRequirement(self.name.lower()) yield PkgConfigRequirement(self.name.lower()) from .resolver.apt import AptRequirement + yield AptRequirement.simple(self.name.lower()) def met(self, session): @@ -208,17 +220,18 @@ class CargoCrateRequirement(Requirement): type(self).__name__, self.crate, self.features, - self.version + self.version, ) def __str__(self): if self.features: return "cargo crate: %s %s (%s)" % ( - self.crate, self.version or '', - ', '.join(sorted(self.features))) + self.crate, + self.version or "", + ", ".join(sorted(self.features)), + ) else: - return "cargo crate: %s %s" % ( - self.crate, self.version or '') + return "cargo crate: %s %s" % (self.crate, self.version or "") class PkgConfigRequirement(Requirement): @@ -346,10 +359,10 @@ class RPackageRequirement(Requirement): @classmethod def from_str(cls, text): # TODO(jelmer): More complex parser - m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text) + m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text) if m: return cls(m.group(1), m.group(2)) - m = re.fullmatch(r'([^ ]+)', text) + m = re.fullmatch(r"([^ ]+)", text) if m: return cls(m.group(1)) raise ValueError(text) @@ -381,10 +394,10 @@ class OctavePackageRequirement(Requirement): @classmethod def from_str(cls, text): # TODO(jelmer): More complex parser - m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text) + m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text) if m: return cls(m.group(1), m.group(2)) - m = re.fullmatch(r'([^ ]+)', text) + m = re.fullmatch(r"([^ ]+)", text) if m: return cls(m.group(1)) raise ValueError(text) @@ -468,11 +481,14 @@ class MavenArtifactRequirement(Requirement): def __str__(self): return "maven requirement: %s:%s:%s" % ( - self.group_id, self.artifact_id, self.version) + self.group_id, + self.artifact_id, + self.version, + ) @classmethod def from_str(cls, text): - return cls.from_tuple(text.split(':')) + return cls.from_tuple(text.split(":")) @classmethod def from_tuple(cls, parts): @@ -486,8 +502,7 @@ class MavenArtifactRequirement(Requirement): (group_id, artifact_id) = parts kind = "jar" else: - raise ValueError( - "invalid number of parts to artifact %r" % parts) + raise ValueError("invalid number of parts to artifact %r" % parts) return cls(group_id, artifact_id, version, kind) @@ -512,31 +527,26 @@ class JDKFileRequirement(Requirement): class JDKRequirement(Requirement): - def __init__(self): super(JDKRequirement, self).__init__("jdk") class JRERequirement(Requirement): - def __init__(self): super(JRERequirement, self).__init__("jre") class QTRequirement(Requirement): - def __init__(self): super(QTRequirement, self).__init__("qt") class X11Requirement(Requirement): - def __init__(self): super(X11Requirement, self).__init__("x11") class CertificateAuthorityRequirement(Requirement): - def __init__(self, url): super(CertificateAuthorityRequirement, self).__init__("ca-cert") self.url = url @@ -561,7 +571,6 @@ class AutoconfMacroRequirement(Requirement): class LibtoolRequirement(Requirement): - def __init__(self): super(LibtoolRequirement, self).__init__("libtool") @@ -593,6 +602,8 @@ class PythonModuleRequirement(Requirement): raise NotImplementedError p = session.Popen( [cmd, "-c", "import %s" % self.module], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) p.communicate() return p.returncode == 0 diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 73b3c89..7451481 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -19,6 +19,7 @@ import subprocess from ..fix_build import run_detecting_problems + class UnsatisfiedRequirements(Exception): def __init__(self, reqs): self.requirements = reqs @@ -53,7 +54,7 @@ class CPANResolver(Resolver): def _cmd(self, reqs): ret = ["cpan", "-i"] if self.skip_tests: - ret.append('-T') + ret.append("-T") ret.extend([req.module for req in reqs]) return ret @@ -75,7 +76,7 @@ class CPANResolver(Resolver): "PERL_MM_USE_DEFAULT": "1", "PERL_MM_OPT": "", "PERL_MB_OPT": "", - } + } if not self.user_local: user = "root" @@ -111,7 +112,11 @@ class RResolver(Resolver): def _cmd(self, req): # TODO(jelmer: Handle self.user_local - return ["R", "-e", "install.packages('%s', repos=%r)" % (req.package, self.repos)] + return [ + "R", + "-e", + "install.packages('%s', repos=%r)" % (req.package, self.repos), + ] def explain(self, requirements): from ..requirements import RPackageRequirement @@ -187,16 +192,17 @@ class OctaveForgeResolver(Resolver): class CRANResolver(RResolver): - def __init__(self, session, user_local=False): - super(CRANResolver, self).__init__(session, 'http://cran.r-project.org', user_local=user_local) + super(CRANResolver, self).__init__( + session, "http://cran.r-project.org", user_local=user_local + ) class BioconductorResolver(RResolver): - def __init__(self, session, user_local=False): super(BioconductorResolver, self).__init__( - session, 'https://hedgehog.fhcrc.org/bioconductor', user_local=user_local) + session, "https://hedgehog.fhcrc.org/bioconductor", user_local=user_local + ) class HackageResolver(Resolver): @@ -213,7 +219,7 @@ class HackageResolver(Resolver): def _cmd(self, reqs): extra_args = [] if self.user_local: - extra_args.append('--user') + extra_args.append("--user") return ["cabal", "install"] + extra_args + [req.package for req in reqs] def install(self, requirements): @@ -259,7 +265,7 @@ class PypiResolver(Resolver): def _cmd(self, reqs): extra_args = [] if self.user_local: - extra_args.append('--user') + extra_args.append("--user") return ["pip", "install"] + extra_args + [req.package for req in reqs] def install(self, requirements): @@ -276,8 +282,7 @@ class PypiResolver(Resolver): missing.append(requirement) continue try: - self.session.check_call( - self._cmd([requirement]), user=user) + self.session.check_call(self._cmd([requirement]), user=user) except subprocess.CalledProcessError: missing.append(requirement) if missing: @@ -296,7 +301,6 @@ class PypiResolver(Resolver): class GoResolver(Resolver): - def __init__(self, session, user_local): self.session = session self.user_local = user_local @@ -314,7 +318,7 @@ class GoResolver(Resolver): env = {} else: # TODO(jelmer): Isn't this Debian-specific? - env = {'GOPATH': '/usr/share/gocode'} + env = {"GOPATH": "/usr/share/gocode"} missing = [] for requirement in requirements: @@ -334,8 +338,7 @@ class GoResolver(Resolver): continue goreqs.append(requirement) if goreqs: - yield (["go", "get"] + [req.package for req in goreqs], - goreqs) + yield (["go", "get"] + [req.package for req in goreqs], goreqs) NPM_COMMAND_PACKAGES = { @@ -361,7 +364,7 @@ class NpmResolver(Resolver): NodePackageRequirement, NodeModuleRequirement, BinaryRequirement, - ) + ) if self.user_local: user = None @@ -379,13 +382,13 @@ class NpmResolver(Resolver): requirement = NodePackageRequirement(package) if isinstance(requirement, NodeModuleRequirement): # TODO: Is this legit? - requirement = NodePackageRequirement(requirement.module.split('/')[0]) + requirement = NodePackageRequirement(requirement.module.split("/")[0]) if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue self.session.check_call( - ["npm", "-g", "install", requirement.package], - user=user) + ["npm", "-g", "install", requirement.package], user=user + ) if missing: raise UnsatisfiedRequirements(missing) @@ -449,7 +452,7 @@ NATIVE_RESOLVER_CLS = [ CRANResolver, BioconductorResolver, OctaveForgeResolver, - ] +] def native_resolvers(session, user_local): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index c0f3dc5..bce5834 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -115,19 +115,27 @@ class AptRequirement(Requirement): return False -def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False) -> List[str]: +def find_package_names( + apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False +) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) def find_reqs_simple( - apt_mgr: AptManager, paths: List[str], regex: bool = False, - minimum_version=None, case_insensitive=False) -> List[str]: + apt_mgr: AptManager, + paths: List[str], + regex: bool = False, + minimum_version=None, + case_insensitive=False, +) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) - return [AptRequirement.simple(package, minimum_version=minimum_version) - for package in find_package_names(apt_mgr, paths, regex, case_insensitive)] + return [ + AptRequirement.simple(package, minimum_version=minimum_version) + for package in find_package_names(apt_mgr, paths, regex, case_insensitive) + ] def python_spec_to_apt_rels(pkg_name, specs): @@ -138,36 +146,57 @@ def python_spec_to_apt_rels(pkg_name, specs): rels = [] for spec in specs: deb_version = Version(spec[1]) - if spec[0] == '~=': + if spec[0] == "~=": # PEP 440: For a given release identifier V.N , the compatible # release clause is approximately equivalent to the pair of # comparison clauses: >= V.N, == V.* - parts = spec[1].split('.') + parts = spec[1].split(".") parts.pop(-1) - parts[-1] = str(int(parts[-1])+1) - next_maj_deb_version = Version('.'.join(parts)) - rels.extend([{"name": pkg_name, "version": ('>=', deb_version)}, - {"name": pkg_name, "version": ('<<', next_maj_deb_version)}]) - elif spec[0] == '!=': - rels.extend([{"name": pkg_name, "version": ('>>', deb_version)}, - {"name": pkg_name, "version": ('<<', deb_version)}]) - elif spec[1].endswith('.*') and spec[0] == '==': - s = spec[1].split('.') + parts[-1] = str(int(parts[-1]) + 1) + next_maj_deb_version = Version(".".join(parts)) + rels.extend( + [ + {"name": pkg_name, "version": (">=", deb_version)}, + {"name": pkg_name, "version": ("<<", next_maj_deb_version)}, + ] + ) + elif spec[0] == "!=": + rels.extend( + [ + {"name": pkg_name, "version": (">>", deb_version)}, + {"name": pkg_name, "version": ("<<", deb_version)}, + ] + ) + elif spec[1].endswith(".*") and spec[0] == "==": + s = spec[1].split(".") s.pop(-1) n = list(s) - n[-1] = str(int(n[-1])+1) - rels.extend([{"name": pkg_name, "version": ('>=', Version('.'.join(s)))}, - {"name": pkg_name, "version": ('<<', Version('.'.join(n)))}]) + n[-1] = str(int(n[-1]) + 1) + rels.extend( + [ + {"name": pkg_name, "version": (">=", Version(".".join(s)))}, + {"name": pkg_name, "version": ("<<", Version(".".join(n)))}, + ] + ) else: c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] rels.append([{"name": pkg_name, "version": (c, deb_version)}]) return rels -def get_package_for_python_package(apt_mgr, package, python_version: Optional[str], specs=None): - pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) - cpython2_regex = "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) - cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) +def get_package_for_python_package( + apt_mgr, package, python_version: Optional[str], specs=None +): + pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape( + package.replace("-", "_") + ) + cpython2_regex = ( + "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" + % re.escape(package.replace("-", "_")) + ) + cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape( + package.replace("-", "_") + ) if python_version == "pypy": paths = [pypy_regex] elif python_version == "cpython2": @@ -177,59 +206,64 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st elif python_version is None: paths = [cpython3_regex, cpython2_regex, pypy_regex] else: - raise NotImplementedError('unsupported python version %s' % python_version) + raise NotImplementedError("unsupported python version %s" % python_version) names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True) return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] def get_package_for_python_module(apt_mgr, module, python_version, specs): cpython3_regexes = [ - posixpath.join( - "/usr/lib/python3/dist-packages", - re.escape(module.replace(".", "/")), - "__init__.py", - ), - posixpath.join( - "/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")) + ".py" - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/lib-dynload", - re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" - ), - posixpath.join( - "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")), "__init__.py" - ), - ] + posixpath.join( + "/usr/lib/python3/dist-packages", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + posixpath.join( + "/usr/lib/python3/dist-packages", + re.escape(module.replace(".", "/")) + ".py", + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/lib-dynload", + re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" + ), + posixpath.join( + "/usr/lib/python3\\.[0-9]+/", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + ] cpython2_regexes = [ - posixpath.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - re.escape(module.replace(".", "/")), - "__init__.py", - ), - posixpath.join( - "/usr/lib/python2\\.[0-9]/dist-packages", - re.escape(module.replace(".", "/")) + ".py", - ), - posixpath.join( - "/usr/lib/python2.\\.[0-9]/lib-dynload", - re.escape(module.replace(".", "/")) + ".so", - ), - ] + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist-packages", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + posixpath.join( + "/usr/lib/python2\\.[0-9]/dist-packages", + re.escape(module.replace(".", "/")) + ".py", + ), + posixpath.join( + "/usr/lib/python2.\\.[0-9]/lib-dynload", + re.escape(module.replace(".", "/")) + ".so", + ), + ] pypy_regexes = [ - posixpath.join( - "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")), "__init__.py" - ), - posixpath.join( - "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" - ), - posixpath.join( - "/usr/lib/pypy/dist-packages", - re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", - ), - ] + posixpath.join( + "/usr/lib/pypy/dist-packages", + re.escape(module.replace(".", "/")), + "__init__.py", + ), + posixpath.join( + "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" + ), + posixpath.join( + "/usr/lib/pypy/dist-packages", + re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", + ), + ] if python_version == "cpython3": paths = cpython3_regexes elif python_version == "cpython2": @@ -245,8 +279,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): vague_map = { - 'the Gnu Scientific Library': 'libgsl-dev', - 'the required FreeType library': 'libfreetype-dev', + "the Gnu Scientific Library": "libgsl-dev", + "the required FreeType library": "libfreetype-dev", } @@ -255,7 +289,7 @@ def resolve_vague_dep_req(apt_mgr, req): options = [] if name in vague_map: options.append(AptRequirement.simple(vague_map[name])) - if name.startswith('gnu '): + if name.startswith("gnu "): name = name[4:] for x in req.expand(): options.extend(resolve_requirement_apt(apt_mgr, x)) @@ -273,13 +307,23 @@ def resolve_binary_req(apt_mgr, req): def resolve_pkg_config_req(apt_mgr, req): - names = find_package_names(apt_mgr, - [posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], - regex=True) + names = find_package_names( + apt_mgr, + [ + posixpath.join( + "/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc" + ) + ], + regex=True, + ) if not names: names = find_package_names( - apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")]) - return [AptRequirement.simple(name, minimum_version=req.minimum_version) for name in names] + apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")] + ) + return [ + AptRequirement.simple(name, minimum_version=req.minimum_version) + for name in names + ] def resolve_path_req(apt_mgr, req): @@ -288,12 +332,13 @@ def resolve_path_req(apt_mgr, req): def resolve_c_header_req(apt_mgr, req): reqs = find_reqs_simple( - apt_mgr, - [posixpath.join("/usr/include", req.header)], regex=False) + apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False + ) if not reqs: reqs = find_reqs_simple( apt_mgr, - [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True + [posixpath.join("/usr/include", ".*", re.escape(req.header))], + regex=True, ) return reqs @@ -314,18 +359,21 @@ def resolve_ruby_gem_req(apt_mgr, req): "specifications/%s-.*\\.gemspec" % re.escape(req.gem) ) ] - return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version) + return find_reqs_simple( + apt_mgr, paths, regex=True, minimum_version=req.minimum_version + ) def resolve_go_package_req(apt_mgr, req): return find_reqs_simple( apt_mgr, - [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True + [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], + regex=True, ) def resolve_go_req(apt_mgr, req): - return [AptRequirement.simple('golang-go', minimum_version='2:%s' % req.version)] + return [AptRequirement.simple("golang-go", minimum_version="2:%s" % req.version)] def resolve_dh_addon_req(apt_mgr, req): @@ -339,11 +387,15 @@ def resolve_php_class_req(apt_mgr, req): def resolve_php_package_req(apt_mgr, req): - return [AptRequirement.simple('php-%s' % req.package, minimum_version=req.min_version)] + return [ + AptRequirement.simple("php-%s" % req.package, minimum_version=req.min_version) + ] def resolve_r_package_req(apt_mgr, req): - paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] + paths = [ + posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package)) + ] return find_reqs_simple(apt_mgr, paths, regex=True) @@ -441,16 +493,18 @@ def resolve_maven_artifact_req(apt_mgr, req): else: version = req.version regex = False + def escape(x): return x - kind = req.kind or 'jar' + + kind = req.kind or "jar" path = posixpath.join( - escape("/usr/share/maven-repo"), - escape(req.group_id.replace(".", "/")), - escape(req.artifact_id), - version, - escape("%s-") + version + escape("." + kind) - ) + escape("/usr/share/maven-repo"), + escape(req.group_id.replace(".", "/")), + escape(req.artifact_id), + version, + escape("%s-") + version + escape("." + kind), + ) return find_reqs_simple(apt_mgr, [path], regex=regex) @@ -465,15 +519,15 @@ def resolve_jdk_file_req(apt_mgr, req): def resolve_jdk_req(apt_mgr, req): - return [AptRequirement.simple('default-jdk')] + return [AptRequirement.simple("default-jdk")] def resolve_jre_req(apt_mgr, req): - return [AptRequirement.simple('default-jre')] + return [AptRequirement.simple("default-jre")] def resolve_x11_req(apt_mgr, req): - return [AptRequirement.simple('libx11-dev')] + return [AptRequirement.simple("libx11-dev")] def resolve_qt_req(apt_mgr, req): @@ -554,13 +608,12 @@ def resolve_python_package_req(apt_mgr, req): def resolve_cargo_crate_req(apt_mgr, req): - paths = [ - '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)] + paths = ["/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml" % re.escape(req.crate)] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_ca_req(apt_mgr, req): - return [AptRequirement.simple('ca-certificates')] + return [AptRequirement.simple("ca-certificates")] def resolve_apt_req(apt_mgr, req): @@ -670,7 +723,16 @@ class AptResolver(Resolver): if apt_req is not None: apt_requirements.append((r, apt_req)) if apt_requirements: - yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements]) + yield ( + self.apt.satisfy_command( + [ + PkgRelation.str( + chain(*[r.relations for o, r in apt_requirements]) + ) + ] + ), + [o for o, r in apt_requirements], + ) def resolve(self, req: Requirement): ret = resolve_requirement_apt(self.apt, req) @@ -678,14 +740,12 @@ class AptResolver(Resolver): return None if len(ret) == 1: return ret[0] - logging.info('Need to break tie between %r with %r', ret, self.tie_breakers) + logging.info("Need to break tie between %r with %r", ret, self.tie_breakers) for tie_breaker in self.tie_breakers: winner = tie_breaker(ret) if winner is not None: if not isinstance(winner, AptRequirement): raise TypeError(winner) return winner - logging.info( - 'Unable to break tie over %r, picking first: %r', - ret, ret[0]) + logging.info("Unable to break tie over %r, picking first: %r", ret, ret[0]) return ret[0] diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index 24d0bd9..464381d 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -55,7 +55,7 @@ class Session(object): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, - close_fds: bool = True + close_fds: bool = True, ): raise NotImplementedError(self.check_call) @@ -90,8 +90,8 @@ class Session(object): raise NotImplementedError(self.scandir) def setup_from_vcs( - self, tree, include_controldir: Optional[bool] = None, - subdir="package") -> Tuple[str, str]: + self, tree, include_controldir: Optional[bool] = None, subdir="package" + ) -> Tuple[str, str]: raise NotImplementedError(self.setup_from_vcs) def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]: @@ -108,8 +108,8 @@ class SessionSetupFailure(Exception): def run_with_tee(session: Session, args: List[str], **kwargs): - if 'stdin' not in kwargs: - kwargs['stdin'] = subprocess.DEVNULL + if "stdin" not in kwargs: + kwargs["stdin"] = subprocess.DEVNULL p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) contents = [] while p.poll() is None: diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 8d9de34..c634f6a 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -38,12 +38,13 @@ class PlainSession(Session): raise NoSessionOpen(self) if user is not None: import getpass + if user != getpass.getuser(): args = ["sudo", "-u", user] + args return args def __repr__(self): - return "%s()" % (type(self).__name__, ) + return "%s()" % (type(self).__name__,) def __enter__(self) -> "Session": if self.es is not None: @@ -63,19 +64,23 @@ class PlainSession(Session): pass def check_call( - self, argv: List[str], - cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None, - close_fds: bool = True): + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + close_fds: bool = True, + ): argv = self._prepend_user(user, argv) return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds) def check_output( - self, argv: List[str], - cwd: Optional[str] = None, - user: Optional[str] = None, - env: Optional[Dict[str, str]] = None) -> bytes: + self, + argv: List[str], + cwd: Optional[str] = None, + user: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + ) -> bytes: argv = self._prepend_user(user, argv) return subprocess.check_output(argv, cwd=cwd, env=env) @@ -95,15 +100,16 @@ class PlainSession(Session): def external_path(self, path): return os.path.abspath(path) - def setup_from_vcs( - self, tree, include_controldir=None, subdir="package"): + def setup_from_vcs(self, tree, include_controldir=None, subdir="package"): from ..vcs import dupe_vcs_tree, export_vcs_tree + if include_controldir is False or ( - not hasattr(tree, 'base') and include_controldir is None): + not hasattr(tree, "base") and include_controldir is None + ): td = self.es.enter_context(tempfile.TemporaryDirectory()) export_vcs_tree(tree, td) return td, td - elif not hasattr(tree, 'base'): + elif not hasattr(tree, "base"): td = self.es.enter_context(tempfile.TemporaryDirectory()) dupe_vcs_tree(tree, td) return td, td diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 10fc9ca..f7f45b2 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -57,12 +57,17 @@ class SchrootSession(Session): if self.session_id is None: raise NoSessionOpen(self) try: - subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"], stderr=subprocess.PIPE) + subprocess.check_output( + ["schroot", "-c", "session:" + self.session_id, "-e"], + stderr=subprocess.PIPE, + ) except subprocess.CalledProcessError as e: for line in e.stderr.splitlines(False): - if line.startswith(b'E: '): - logging.error('%s', line[3:].decode(errors='replace')) - logging.warning('Failed to close schroot session %s, leaving stray.', self.session_id) + if line.startswith(b"E: "): + logging.error("%s", line[3:].decode(errors="replace")) + logging.warning( + "Failed to close schroot session %s, leaving stray.", self.session_id + ) self.session_id = None return False self.session_id = None @@ -134,10 +139,12 @@ class SchrootSession(Session): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, - close_fds: bool = True + close_fds: bool = True, ): try: - subprocess.check_call(self._run_argv(argv, cwd, user, env=env), close_fds=close_fds) + subprocess.check_call( + self._run_argv(argv, cwd, user, env=env), close_fds=close_fds + ) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv) @@ -191,9 +198,10 @@ class SchrootSession(Session): return os.scandir(fullpath) def setup_from_vcs( - self, tree, include_controldir: Optional[bool] = None, - subdir="package"): + self, tree, include_controldir: Optional[bool] = None, subdir="package" + ): from ..vcs import dupe_vcs_tree, export_vcs_tree + build_dir = os.path.join(self.location, "build") directory = tempfile.mkdtemp(dir=build_dir) @@ -209,6 +217,7 @@ class SchrootSession(Session): def setup_from_directory(self, path, subdir="package"): import shutil + build_dir = os.path.join(self.location, "build") directory = tempfile.mkdtemp(dir=build_dir) reldir = "/" + os.path.relpath(directory, self.location) diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index 4434a5f..b007955 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -99,11 +99,14 @@ blah (0.1) UNRELEASED; urgency=medium apt = AptManager(session) apt._searchers = [DummyAptSearcher(self._apt_files)] context = DebianPackagingContext( - self.tree, subpath="", committer="ognibuild ", + self.tree, + subpath="", + committer="ognibuild ", update_changelog=True, - commit_reporter=NullCommitReporter()) + commit_reporter=NullCommitReporter(), + ) fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context) - return resolve_error(error, ("build", ), fixers) + return resolve_error(error, ("build",), fixers) def get_build_deps(self): with open(self.tree.abspath("debian/control"), "r") as f: From 7bc15f3cd10c8255d99d8a183d4ca4105ec09b06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 12:28:24 +0000 Subject: [PATCH 240/252] remove unused code. --- ognibuild/resolver/apt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index bce5834..383ef77 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -289,8 +289,6 @@ def resolve_vague_dep_req(apt_mgr, req): options = [] if name in vague_map: options.append(AptRequirement.simple(vague_map[name])) - if name.startswith("gnu "): - name = name[4:] for x in req.expand(): options.extend(resolve_requirement_apt(apt_mgr, x)) return options From 81a9f6f53fcf261d49f530607c4e6e2263a9e97b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 12:29:56 +0000 Subject: [PATCH 241/252] Support minimum version in VagueDependency. --- ognibuild/buildlog.py | 2 +- ognibuild/requirements.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 1381ccc..b5d2c8d 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -129,7 +129,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 elif isinstance(problem, MissingNodePackage): return NodePackageRequirement(problem.package) elif isinstance(problem, MissingVagueDependency): - return VagueDependencyRequirement(problem.name) + return VagueDependencyRequirement(problem.name, minimum_version=problem.minimum_version) elif isinstance(problem, MissingLibrary): return LibraryRequirement(problem.library) elif isinstance(problem, MissingRubyFile): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index fea0ea4..802d5ba 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -149,23 +149,25 @@ class PerlModuleRequirement(Requirement): class VagueDependencyRequirement(Requirement): name: str + minimum_version: Optional[str] = None - def __init__(self, name): + def __init__(self, name, minimum_version=None): super(VagueDependencyRequirement, self).__init__("vague") self.name = name + self.minimum_version = minimum_version def expand(self): if " " not in self.name: yield BinaryRequirement(self.name) yield LibraryRequirement(self.name) - yield PkgConfigRequirement(self.name) + yield PkgConfigRequirement(self.name, minimum_version=self.minimum_version) if self.name.lower() != self.name: yield BinaryRequirement(self.name.lower()) yield LibraryRequirement(self.name.lower()) - yield PkgConfigRequirement(self.name.lower()) + yield PkgConfigRequirement(self.name.lower(), minimum_version=self.minimum_version) from .resolver.apt import AptRequirement - yield AptRequirement.simple(self.name.lower()) + yield AptRequirement.simple(self.name.lower(), minimum_version=self.minimum_version) def met(self, session): for x in self.expand(): From 9bfac89e7e98e5917bd22a4afe484d9bbd07ab1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 12:40:27 +0000 Subject: [PATCH 242/252] Depend on testtools. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3e9c72..d011923 100755 --- a/setup.py +++ b/setup.py @@ -35,6 +35,6 @@ setup(name="ognibuild", extras_require={ 'debian': ['debmutate', 'python_debian', 'python_apt'], }, - tests_require=['python_debian', 'buildlog-consultant', 'breezy'], + tests_require=['python_debian', 'buildlog-consultant', 'breezy', 'testtools'], test_suite='ognibuild.tests.test_suite', ) From 8fd1d4035d0383f8833db622639a2060cc925789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 12:52:11 +0000 Subject: [PATCH 243/252] Fix apt expansion for vague dependencies. --- ognibuild/requirements.py | 1 + ognibuild/resolver/apt.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 802d5ba..7491b19 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -168,6 +168,7 @@ class VagueDependencyRequirement(Requirement): from .resolver.apt import AptRequirement yield AptRequirement.simple(self.name.lower(), minimum_version=self.minimum_version) + yield AptRequirement.simple('lib%s-dev' % self.name.lower(), minimum_version=self.minimum_version) def met(self, session): for x in self.expand(): diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 383ef77..f672836 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -615,6 +615,10 @@ def resolve_ca_req(apt_mgr, req): def resolve_apt_req(apt_mgr, req): + # TODO(jelmer): This should be checking whether versions match as well. + for package_name in req.package_names(): + if not apt_mgr.package_exists(package_name): + return [] return [req] From 5904fa214a162129c25093de6cf95036e6cfae6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 16:37:08 +0000 Subject: [PATCH 244/252] Print results. --- ognibuild/debian/fix_build.py | 39 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index b6b2e6f..66ed2fb 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -142,15 +142,16 @@ class DebianPackagingContext(object): self.update_changelog = update_changelog self.commit_reporter = commit_reporter + def abspath(self, *parts): + return self.tree.abspath(os.path.join(self.subpath, *parts)) + def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool: if update_changelog is None: update_changelog = self.update_changelog with self.tree.lock_write(): try: if update_changelog: - cl_path = self.tree.abspath( - os.path.join(self.subpath, "debian/changelog") - ) + cl_path = self.abspath("debian/changelog") with ChangelogEditor(cl_path) as editor: editor.add_entry([summary]) debcommit(self.tree, committer=self.committer, subpath=self.subpath) @@ -218,7 +219,7 @@ def add_build_dependency(context, requirement: AptRequirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - control_path = os.path.join(context.tree.abspath(context.subpath), "debian/control") + control_path = context.abspath("debian/control") try: with ControlEditor(path=control_path) as updater: for binary in updater.binaries: @@ -246,9 +247,7 @@ def add_test_dependency(context, testname, requirement): if not isinstance(requirement, AptRequirement): raise TypeError(requirement) - tests_control_path = os.path.join( - context.tree.abspath(context.subpath), "debian/tests/control" - ) + tests_control_path = context.abspath("debian/tests/control") try: with Deb822Editor(path=tests_control_path) as updater: @@ -386,9 +385,10 @@ def fix_missing_config_status_input(error, phase, context): class PgBuildExtOutOfDateControlFixer(BuildFixer): - def __init__(self, packaging_context, session): + def __init__(self, packaging_context, session, apt): self.session = session self.context = packaging_context + self.apt = apt def can_fix(self, problem): return isinstance(problem, NeedPgBuildExtUpdateControl) @@ -396,12 +396,17 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer): def __repr__(self): return "%s()" % (type(self).__name__,) - def _fix(self, error, context): + def _fix(self, error, phase): logging.info("Running 'pg_buildext updatecontrol'") + self.apt.install(['postgresql-common']) + external_dir, internal_dir = self.session.setup_from_vcs( + self.context.tree, include_controldir=None, + subdir=self.context.subpath) + self.session.chdir(internal_dir) self.session.check_call(["pg_buildext", "updatecontrol"]) shutil.copy( - self.session.external_path("debian/control"), - context.tree.abspath(os.path.join(context.subpath, "debian/control")), + os.path.join(external_dir, error.generated_path), + self.context.abspath(error.generated_path) ) return self.context.commit( "Run 'pgbuildext updatecontrol'.", update_changelog=False @@ -460,9 +465,9 @@ class DependencyBuildFixer(BuildFixer): return self._fn(problem, phase, self.apt_resolver, self.context) -def versioned_package_fixers(session, packaging_context): +def versioned_package_fixers(session, packaging_context, apt): return [ - PgBuildExtOutOfDateControlFixer(packaging_context, session), + PgBuildExtOutOfDateControlFixer(packaging_context, session, apt), SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure), SimpleBuildFixer( packaging_context, MissingAutomakeInput, fix_missing_automake_input @@ -511,7 +516,7 @@ def build_incrementally( packaging_context = DebianPackagingContext( local_tree, subpath, committer, update_changelog ) - fixers = versioned_package_fixers(apt.session, packaging_context) + apt_fixers( + fixers = versioned_package_fixers(apt.session, packaging_context, apt) + apt_fixers( apt, packaging_context ) logging.info("Using fixers: %r", fixers) @@ -642,7 +647,7 @@ def main(argv=None): apt = AptManager(session) try: - build_incrementally( + (changes_filename, cl_version) = build_incrementally( tree, apt, args.suffix, @@ -666,6 +671,10 @@ def main(argv=None): logging.fatal("Error during %s: %s", phase, e.description) return 1 + logging.info( + 'Built %s - changes file at %s.', + os.path.join(output_directory, changes_filename)) + if __name__ == "__main__": sys.exit(main(sys.argv)) From 8ab342f21c4c4f88ee8b87ef7d9e4b3ef6b30291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 17:23:28 +0000 Subject: [PATCH 245/252] ADd TODO. --- ognibuild/debian/fix_build.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 66ed2fb..5e12726 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -203,11 +203,7 @@ class PackageDependencyFixer(BuildFixer): def add_dependency(context, phase, requirement: AptRequirement): if phase[0] == "autopkgtest": - return add_test_dependency( - context, - phase[1], - requirement, - ) + return add_test_dependency(context, phase[1], requirement) elif phase[0] == "build": return add_build_dependency(context, requirement) else: @@ -249,6 +245,9 @@ def add_test_dependency(context, testname, requirement): tests_control_path = context.abspath("debian/tests/control") + # TODO(jelmer): If requirement is for one of our binary packages + # but "@" is already present then don't do anything. + try: with Deb822Editor(path=tests_control_path) as updater: command_counter = 1 From 996b135c8035431b3790d311b2b87db4641446f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 18:16:24 +0000 Subject: [PATCH 246/252] Avoid pre-emptively installing. --- ognibuild/__main__.py | 20 +++++++----- ognibuild/buildsystem.py | 69 +++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 44 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 514056f..8f82ca2 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -173,15 +173,19 @@ def main(): # noqa: C901 display_explain_commands(e.commands) return 1 if args.subcommand == "dist": - from .dist import run_dist + from .dist import run_dist, DistNoTarball - run_dist( - session=session, - buildsystems=bss, - resolver=resolver, - fixers=fixers, - target_directory=".", - ) + try: + run_dist( + session=session, + buildsystems=bss, + resolver=resolver, + fixers=fixers, + target_directory=".", + ) + except DistNoTarball: + logging.fatal('No tarball created.') + return 1 if args.subcommand == "build": from .build import run_build diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9762220..0776e57 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -121,30 +121,23 @@ class Pear(BuildSystem): def __init__(self, path): self.path = path - def setup(self, resolver): - resolver.install([BinaryRequirement("pear")]) - def dist(self, session, resolver, fixers, target_directory: str, quiet=False): - self.setup(resolver) with DistCatcher([session.external_path(".")]) as dc: - run_with_build_fixers(session, ["pear", "package"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "package"], fixers) return dc.copy_single(target_directory) def test(self, session, resolver, fixers): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "run-tests"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "run-tests"], fixers) def build(self, session, resolver, fixers): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "build", self.path], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "build", self.path], fixers) def clean(self, session, resolver, fixers): self.setup(resolver) # TODO def install(self, session, resolver, fixers, install_target): - self.setup(resolver) - run_with_build_fixers(session, ["pear", "install", self.path], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "pear"), "install", self.path], fixers) def get_declared_dependencies(self, session, fixers=None): path = os.path.join(self.path, "package.xml") @@ -387,7 +380,9 @@ class SetupPy(BuildSystem): # Pre-emptively insall setuptools, since distutils doesn't provide # a 'test' subcommand and some packages fall back to distutils # if setuptools is not available. - resolver.install([PythonPackageRequirement("setuptools")]) + setuptools_req = PythonPackageRequirement("setuptools") + if not setuptools_req.met(session): + resolver.install([setuptools_req]) self._run_setup(session, resolver, ["test"], fixers) else: raise NotImplementedError @@ -406,7 +401,9 @@ class SetupPy(BuildSystem): preargs.append("--quiet") # Preemptively install setuptools since some packages fail in # some way without it. - resolver.install([PythonPackageRequirement("setuptools")]) + setuptools_req = PythonPackageRequirement("setuptools") + if not setuptools_req.met(session): + resolver.install([setuptools_req]) with DistCatcher([session.external_path("dist")]) as dc: self._run_setup(session, resolver, preargs + ["sdist"], fixers) return dc.copy_single(target_directory) @@ -610,12 +607,14 @@ class Gradle(BuildSystem): logging.debug("Found build.gradle, assuming gradle package.") return cls.from_path(path) - def setup(self, resolver): + def setup(self, session, resolver): if not self.executable.startswith("./"): - resolver.install([BinaryRequirement(self.executable)]) + binary_req = BinaryRequirement(self.executable) + if not binary_req.met(session): + resolver.install([binary_req]) def _run(self, session, resolver, task, args, fixers): - self.setup(resolver) + self.setup(session, resolver) argv = [] if self.executable.startswith("./") and ( not os.access(os.path.join(self.path, self.executable), os.X_OK) @@ -790,17 +789,19 @@ class Npm(BuildSystem): # TODO(jelmer): Look at version yield "build", NodePackageRequirement(name) - def setup(self, resolver): - resolver.install([BinaryRequirement("npm")]) + def setup(self, session, resolver): + binary_req = BinaryRequirement("npm") + if not binary_req.met(session): + resolver.install([binary_req]) def dist(self, session, resolver, fixers, target_directory, quiet=False): - self.setup(resolver) + self.setup(session, resolver) with DistCatcher([session.external_path(".")]) as dc: run_with_build_fixers(session, ["npm", "pack"], fixers) return dc.copy_single(target_directory) def test(self, session, resolver, fixers): - self.setup(resolver) + self.setup(session, resolver) test_script = self.package["scripts"].get("test") if test_script: run_with_build_fixers(session, shlex.split(test_script), fixers) @@ -808,7 +809,7 @@ class Npm(BuildSystem): raise NotImplementedError def build(self, session, resolver, fixers): - self.setup(resolver) + self.setup(session, resolver) build_script = self.package["scripts"].get("build") if build_script: run_with_build_fixers(session, shlex.split(build_script), fixers) @@ -816,7 +817,7 @@ class Npm(BuildSystem): raise NotImplementedError def clean(self, session, resolver, fixers): - self.setup(resolver) + self.setup(session, resolver) clean_script = self.package["scripts"].get("clean") if clean_script: run_with_build_fixers(session, shlex.split(clean_script), fixers) @@ -838,7 +839,9 @@ class Waf(BuildSystem): self.path = path def setup(self, session, resolver, fixers): - resolver.install([BinaryRequirement("python3")]) + binary_req = BinaryRequirement("python3") + if not binary_req.met(session): + resolver.install([binary_req]) def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, resolver, fixers) @@ -864,18 +867,16 @@ class Gem(BuildSystem): def __init__(self, path): self.path = path - def setup(self, resolver): - resolver.install([BinaryRequirement("gem2deb")]) - def dist(self, session, resolver, fixers, target_directory, quiet=False): - self.setup(resolver) gemfiles = [ entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") ] if len(gemfiles) > 1: logging.warning("More than one gemfile. Trying the first?") with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) + run_with_build_fixers( + session, + [guaranteed_which(session, resolver, "gem2tgz"), gemfiles[0]], fixers) return dc.copy_single(target_directory) @classmethod @@ -921,26 +922,22 @@ class DistZilla(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(resolver) if self.name == "dist-inkt": - resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers(session, ["distinkt-dist"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "distinkt-dist")], fixers) return dc.copy_single(target_directory) else: # Default to invoking Dist::Zilla - resolver.install([PerlModuleRequirement("Dist::Zilla")]) with DistCatcher.default(session.external_path(".")) as dc: - run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "build", "--tgz"], fixers) return dc.copy_single(target_directory) def test(self, session, resolver, fixers): self.setup(resolver) - resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers(session, ["dzil", "test"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "test"], fixers) def build(self, session, resolver, fixers): self.setup(resolver) - resolver.install([PerlModuleRequirement("Dist::Zilla")]) - run_with_build_fixers(session, ["dzil", "build"], fixers) + run_with_build_fixers(session, [guaranteed_which(session, resolver, "dzil"), "build"], fixers) @classmethod def probe(cls, path): From d6f35c965719c405bf0218ff994531eb461f86b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 18:19:54 +0000 Subject: [PATCH 247/252] Support stdin for plain session. --- ognibuild/session/plain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index c634f6a..0084d8d 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -84,9 +84,9 @@ class PlainSession(Session): argv = self._prepend_user(user, argv) return subprocess.check_output(argv, cwd=cwd, env=env) - def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None, env=None): + def Popen(self, args, stdout=None, stderr=None, stdin=None, user=None, cwd=None, env=None): args = self._prepend_user(user, args) - return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd, env=env) + return subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, cwd=cwd, env=env) def exists(self, path): return os.path.exists(path) From 7456ee7d12139c750ecffb0b7d6ea6f875c42e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 18:21:08 +0000 Subject: [PATCH 248/252] Cope with leading whitespace in shebang. --- ognibuild/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index a80945c..da6dea3 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -55,7 +55,7 @@ def shebang_binary(p): firstline = f.readline() if not firstline.startswith(b"#!"): return None - args = firstline[2:].split(b" ") + args = firstline[2:].strip().split(b" ") if args[0] in (b"/usr/bin/env", b"env"): return os.path.basename(args[1].decode()).strip() return os.path.basename(args[0].decode()).strip() From de2c26664e220a88501b61397b14ea614928311f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 26 Mar 2021 21:40:21 +0000 Subject: [PATCH 249/252] Bump to 0.0.3. --- ognibuild/__init__.py | 3 +++ releaser.conf | 5 +++++ setup.py | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index da6dea3..ab42dbb 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -20,6 +20,9 @@ import os import stat +__version__ = (0, 0, 3) + + USER_AGENT = "Ognibuild" diff --git a/releaser.conf b/releaser.conf index deea576..c4c19a4 100644 --- a/releaser.conf +++ b/releaser.conf @@ -7,3 +7,8 @@ update_version { match: "^ version=\"(.*)\",$" new_line: " version=\"$VERSION\"," } +update_version { + path: "ognibuild/__init__.py" + match: "^__version__ = \\((.*)\\)$" + new_line: "__version__ = $TUPLED_VERSION" +} diff --git a/setup.py b/setup.py index d011923..d3676b2 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup(name="ognibuild", description="Detect and run any build system", - version="0.0.1", + version="0.0.3", maintainer="Jelmer Vernooij", maintainer_email="jelmer@jelmer.uk", license="GNU GPLv2 or later", From e1b8a90747d952b7c56635dd7eb50153a826a18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Mar 2021 09:07:59 +0000 Subject: [PATCH 250/252] Add support for finding cmake files. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 9 +++++++++ ognibuild/resolver/apt.py | 7 +++++++ 3 files changed, 20 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index b5d2c8d..fabb279 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -56,6 +56,7 @@ from buildlog_consultant.common import ( GnomeCommonMissing, MissingGnomeCommonDependency, UnknownCertificateAuthority, + CMakeFilesMissing, MissingLibtool, MissingQt, MissingX11, @@ -80,6 +81,7 @@ from .requirements import ( XmlEntityRequirement, SprocketsFileRequirement, JavaClassRequirement, + CMakefileRequirement, HaskellPackageRequirement, MavenArtifactRequirement, GnomeCommonRequirement, @@ -140,6 +142,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return SprocketsFileRequirement(problem.content_type, problem.name) elif isinstance(problem, MissingJavaClass): return JavaClassRequirement(problem.classname) + elif isinstance(problem, CMakeFilesMissing): + return [CMakefileRequirement(filename) for filename in problem.filenames] elif isinstance(problem, MissingHaskellDependencies): return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps] elif isinstance(problem, MissingMavenArtifacts): diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 7491b19..014bee9 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -453,6 +453,15 @@ class JavaClassRequirement(Requirement): self.classname = classname +class CMakefileRequirement(Requirement): + + filename: str + + def __init__(self, filename: str): + super(CMakefileRequirement, self).__init__("cmake-file") + self.filename = filename + + class HaskellPackageRequirement(Requirement): package: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index f672836..618206c 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -51,6 +51,7 @@ from ..requirements import ( XmlEntityRequirement, SprocketsFileRequirement, JavaClassRequirement, + CMakefileRequirement, HaskellPackageRequirement, MavenArtifactRequirement, GnomeCommonRequirement, @@ -478,6 +479,11 @@ def resolve_java_class_req(apt_mgr, req): return find_reqs_simple(apt_mgr, [classpath]) +def resolve_cmake_file_req(apt_mgr, req): + paths = ['/usr/lib/.*/cmake/.*/%s' % re.escape(req.filename)] + return find_reqs_simple(apt_mgr, paths, regex=True) + + def resolve_haskell_package_req(apt_mgr, req): path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) return find_reqs_simple(apt_mgr, [path], regex=True) @@ -645,6 +651,7 @@ APT_REQUIREMENT_RESOLVERS = [ (XmlEntityRequirement, resolve_xml_entity_req), (SprocketsFileRequirement, resolve_sprockets_file_req), (JavaClassRequirement, resolve_java_class_req), + (CMakefileRequirement, resolve_cmake_file_req), (HaskellPackageRequirement, resolve_haskell_package_req), (MavenArtifactRequirement, resolve_maven_artifact_req), (GnomeCommonRequirement, resolve_gnome_common_req), From a5f9bc3da6984ca1ed1e37c85528b19fb8f36b5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Mar 2021 14:23:34 +0000 Subject: [PATCH 251/252] Fix test. --- ognibuild/tests/test_debian_fix_build.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ognibuild/tests/test_debian_fix_build.py b/ognibuild/tests/test_debian_fix_build.py index b007955..e3288b5 100644 --- a/ognibuild/tests/test_debian_fix_build.py +++ b/ognibuild/tests/test_debian_fix_build.py @@ -45,10 +45,17 @@ class DummyAptSearcher(FileSearcher): def __init__(self, files): self._apt_files = files - def search_files(self, path, regex=False): + def search_files(self, path, regex=False, case_insensitive=False): for p, pkg in sorted(self._apt_files.items()): + if case_insensitive: + flags = re.I + else: + flags = 0 if regex: - if re.match(path, p): + if re.match(path, p, flags): + yield pkg + elif case_insensitive: + if path.lower() == p.lower(): yield pkg else: if path == p: @@ -105,7 +112,7 @@ blah (0.1) UNRELEASED; urgency=medium update_changelog=True, commit_reporter=NullCommitReporter(), ) - fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context) + fixers = versioned_package_fixers(session, context, apt) + apt_fixers(apt, context) return resolve_error(error, ("build",), fixers) def get_build_deps(self): From 9fe6ec8dbcaf82b63156bb720afcf58d369f7947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Mar 2021 14:25:37 +0000 Subject: [PATCH 252/252] Release 0.0.3.