From f5aa0b3a3acef85d9d0f1fe872c687a6556b6905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Mar 2021 18:05:27 +0000 Subject: [PATCH 01/65] Don't just check for setup.cfg. --- ognibuild/buildsystem.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 0776e57..4f8c271 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -522,9 +522,6 @@ class SetupPy(BuildSystem): if os.path.exists(os.path.join(path, "setup.py")): logging.debug("Found setup.py, assuming python project.") return cls(path) - if os.path.exists(os.path.join(path, "setup.cfg")): - logging.debug("Found setup.py, assuming python project.") - return cls(path) if os.path.exists(os.path.join(path, "pyproject.toml")): logging.debug("Found pyproject.toml, assuming python project.") return cls(path) From cb6c10a992ac15fee42bb2441138334c93cbc49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Mar 2021 19:26:01 +0000 Subject: [PATCH 02/65] Mention dependency resolver in output. --- ognibuild/dist.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/dist.py b/ognibuild/dist.py index cdf1265..8206920 100644 --- a/ognibuild/dist.py +++ b/ognibuild/dist.py @@ -51,6 +51,8 @@ def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=Fa # e.g. pip caches in ~/.cache session.create_home() + logging.info('Using dependency resolver: %s', resolver) + for buildsystem in buildsystems: filename = buildsystem.dist( session, resolver, fixers, target_directory, quiet=quiet From ed6d9c8af19f7bf91bf25694a0bc90a652bb95b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 27 Mar 2021 20:42:55 +0000 Subject: [PATCH 03/65] Use run_detecting_problems more. --- ognibuild/resolver/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 7451481..4e72bc5 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -142,7 +142,7 @@ class RResolver(Resolver): if not isinstance(requirement, RPackageRequirement): missing.append(requirement) continue - self.session.check_call(self._cmd(requirement), user=user) + run_detecting_problems(self.session, self._cmd(requirement), user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -186,7 +186,7 @@ class OctaveForgeResolver(Resolver): if not isinstance(requirement, OctavePackageRequirement): missing.append(requirement) continue - self.session.check_call(self._cmd(requirement), user=user) + run_detecting_problems(self.session, self._cmd(requirement), user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -235,7 +235,7 @@ class HackageResolver(Resolver): if not isinstance(requirement, HaskellPackageRequirement): missing.append(requirement) continue - self.session.check_call(self._cmd([requirement]), user=user) + run_detecting_problems(self.session, self._cmd([requirement]), user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -282,7 +282,7 @@ class PypiResolver(Resolver): missing.append(requirement) continue try: - self.session.check_call(self._cmd([requirement]), user=user) + run_detecting_problems(self.session, self._cmd([requirement]), user=user) except subprocess.CalledProcessError: missing.append(requirement) if missing: @@ -325,7 +325,7 @@ class GoResolver(Resolver): if not isinstance(requirement, GoPackageRequirement): missing.append(requirement) continue - self.session.check_call(["go", "get", requirement.package], env=env) + run_detecting_problems(self.session, ["go", "get", requirement.package], env=env) if missing: raise UnsatisfiedRequirements(missing) @@ -386,7 +386,8 @@ class NpmResolver(Resolver): if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - self.session.check_call( + run_detecting_problems( + self.session, ["npm", "-g", "install", requirement.package], user=user ) if missing: From 6106f9a74c8c07488f7dfc91ee7915a40628fc73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sun, 28 Mar 2021 00:38:41 +0000 Subject: [PATCH 04/65] Add another npm command. --- ognibuild/resolver/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 4e72bc5..5460fec 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -344,6 +344,7 @@ class GoResolver(Resolver): NPM_COMMAND_PACKAGES = { "del-cli": "del-cli", "husky": "husky", + "cross-env": "cross-env", } From fc64cf1da1b8e32d1632a30c59e8d79649d89753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 00:55:13 +0100 Subject: [PATCH 05/65] Create first entry. --- ognibuild/debian/build.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index cca50fb..e530cf2 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -24,6 +24,7 @@ __all__ = [ ] from datetime import datetime +from debmutate.changelog import ChangelogEditor import logging import os import re @@ -119,25 +120,16 @@ def add_dummy_changelog_entry( maintainer = get_maintainer() if timestamp is None: timestamp = datetime.now() - with tree.get_file(path) as f: - cl = Changelog() - cl.parse_changelog(f, max_blocks=None, allow_empty_author=True, strict=False) - version = cl[0].version + with ChangelogEditor(tree.abspath(os.path.join(path))) as editor: + version = editor[0].version if version.debian_revision: version.debian_revision = add_suffix(version.debian_revision, suffix) else: version.upstream_version = add_suffix(version.upstream_version, suffix) - cl.new_block( - package=cl[0].package, - version=version, - urgency="low", - distributions=suite, - author="%s <%s>" % maintainer, - date=format_datetime(timestamp), - changes=["", " * " + message, ""], - ) - cl_str = cl._format(allow_missing_author=True) - tree.put_file_bytes_non_atomic(path, cl_str.encode(cl._encoding)) + editor.auto_version(version) + editor.add_entry( + summary=[message], maintainer=maintainer, timestamp=timestamp, urgency='low') + editor[0].distributions = suite def get_latest_changelog_version(local_tree, subpath=""): From 332263fd9ffa956e53802da7e4240f1f7f22134b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 01:06:01 +0100 Subject: [PATCH 06/65] Run gbp dch when not updating changelog. --- ognibuild/debian/build.py | 3 +++ ognibuild/debian/fix_build.py | 1 + 2 files changed, 4 insertions(+) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index e530cf2..505199c 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -222,6 +222,7 @@ def attempt_build( build_changelog_entry=None, subpath="", source_date_epoch=None, + run_gbp_dch=False ): """Attempt a build, with a custom distribution set. @@ -236,6 +237,8 @@ def attempt_build( source_date_epoch: Source date epoch to set Returns: Tuple with (changes_name, cl_version) """ + if run_gbp_dch: + gbp_dch(local_tree.abspath(subpath)) if build_changelog_entry is not None: add_dummy_changelog_entry( local_tree, subpath, suffix, build_suite, build_changelog_entry diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 5e12726..2ae4487 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -530,6 +530,7 @@ def build_incrementally( build_changelog_entry, subpath=subpath, source_date_epoch=source_date_epoch, + run_gbp_dch=(update_changelog is False) ) except SbuildFailure as e: if e.error is None: From 9432bbf6a03e2e1dff43a15c9076b35d94f1bee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 12:44:43 +0100 Subject: [PATCH 07/65] Ignore branch when applying 'gbp dch' --- ognibuild/debian/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 505199c..b5524b6 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -210,7 +210,7 @@ def build_once( def gbp_dch(path): - subprocess.check_call(["gbp", "dch"], cwd=path) + subprocess.check_call(["gbp", "dch", "--ignore_branch"], cwd=path) def attempt_build( From fc73f0ead22e058d7f562ee11fcab995d7e97df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 13:09:39 +0100 Subject: [PATCH 08/65] Parse out build/configure requires from META.yml. --- ognibuild/buildsystem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 4f8c271..fd40ee6 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1115,6 +1115,10 @@ class Make(BuildSystem): warnings.warn("Unable to parse META.yml: %s" % e) return for require in data.get("requires", []): + yield "core", PerlModuleRequirement(require) + for require in data.get("build_requires", []): + yield "build", PerlModuleRequirement(require) + for require in data.get("configure_requires", []): yield "build", PerlModuleRequirement(require) something = True if os.path.exists(os.path.join(self.path, "cpanfile")): From 7abf6bfba5f7c575ba2e1cdba24be3b1d7133618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 14:34:30 +0100 Subject: [PATCH 09/65] Fix flag for gbp dch. --- ognibuild/debian/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index b5524b6..63d9b50 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -210,7 +210,7 @@ def build_once( def gbp_dch(path): - subprocess.check_call(["gbp", "dch", "--ignore_branch"], cwd=path) + subprocess.check_call(["gbp", "dch", "--ignore-branch"], cwd=path) def attempt_build( From 3bcfc950cb82ab5d0e537b9df04bc813b06ee768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 14:59:43 +0100 Subject: [PATCH 10/65] Fix tests. --- ognibuild/debian/build.py | 20 +++++++++++--------- ognibuild/debian/fix_build.py | 6 +++--- ognibuild/tests/test_debian_build.py | 27 ++++++--------------------- 3 files changed, 20 insertions(+), 33 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 63d9b50..deafe39 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -16,7 +16,6 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA __all__ = [ - "changes_filename", "get_build_architecture", "add_dummy_changelog_entry", "build", @@ -54,11 +53,15 @@ class MissingChangesFile(Exception): self.filename = filename -def changes_filename(package, version, arch): +def find_changes_files(path, package, version): non_epoch_version = version.upstream_version if version.debian_version is not None: non_epoch_version += "-%s" % version.debian_version - return "%s_%s_%s.changes" % (package, non_epoch_version, arch) + c = re.compile('%s_%s_(.*).changes' % (re.escape(package), re.escape(non_epoch_version))) + for entry in os.scandir(path): + m = c.match(entry.name) + if m: + yield m.group(1), entry def get_build_architecture(): @@ -126,7 +129,7 @@ def add_dummy_changelog_entry( version.debian_revision = add_suffix(version.debian_revision, suffix) else: version.upstream_version = add_suffix(version.upstream_version, suffix) - editor.auto_version(version) + editor.auto_version(version, timestamp=timestamp) editor.add_entry( summary=[message], maintainer=maintainer, timestamp=timestamp, urgency='low') editor[0].distributions = suite @@ -202,11 +205,10 @@ def build_once( raise worker_failure_from_sbuild_log(f) (cl_package, cl_version) = get_latest_changelog_version(local_tree, subpath) - changes_name = changes_filename(cl_package, cl_version, get_build_architecture()) - changes_path = os.path.join(output_directory, changes_name) - if not os.path.exists(changes_path): - raise MissingChangesFile(changes_name) - return (changes_name, cl_version) + changes_names = [] + for kind, entry in find_changes_files(output_directory, cl_package, cl_version): + changes_names.append((entry.name)) + return (changes_names, cl_version) def gbp_dch(path): diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 2ae4487..4de6403 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -647,7 +647,7 @@ def main(argv=None): apt = AptManager(session) try: - (changes_filename, cl_version) = build_incrementally( + (changes_filenames, cl_version) = build_incrementally( tree, apt, args.suffix, @@ -672,8 +672,8 @@ def main(argv=None): return 1 logging.info( - 'Built %s - changes file at %s.', - os.path.join(output_directory, changes_filename)) + 'Built %s - changes file at %r.', + os.path.join(output_directory, changes_files)) if __name__ == "__main__": diff --git a/ognibuild/tests/test_debian_build.py b/ognibuild/tests/test_debian_build.py index 6e5970a..0b06869 100644 --- a/ognibuild/tests/test_debian_build.py +++ b/ognibuild/tests/test_debian_build.py @@ -53,17 +53,12 @@ janitor (0.1-1) UNRELEASED; urgency=medium ) self.assertFileEqual( """\ -janitor (0.1-1jan+some1) some-fixes; urgency=low +janitor (0.1-1jan+some1) some-fixes; urgency=medium + * Initial release. (Closes: #XXXXXX) * Dummy build. -- Jelmer Vernooij Sat, 05 Sep 2020 12:35:04 -0000 - -janitor (0.1-1) UNRELEASED; urgency=medium - - * Initial release. (Closes: #XXXXXX) - - -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 """, "debian/changelog", ) @@ -97,17 +92,12 @@ janitor (0.1) UNRELEASED; urgency=medium ) self.assertFileEqual( """\ -janitor (0.1jan+some1) some-fixes; urgency=low +janitor (0.1jan+some1) some-fixes; urgency=medium + * Initial release. (Closes: #XXXXXX) * Dummy build. -- Jelmer Vernooij Sat, 05 Sep 2020 12:35:04 -0000 - -janitor (0.1) UNRELEASED; urgency=medium - - * Initial release. (Closes: #XXXXXX) - - -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 """, "debian/changelog", ) @@ -141,17 +131,12 @@ janitor (0.1-1jan+some1) UNRELEASED; urgency=medium ) self.assertFileEqual( """\ -janitor (0.1-1jan+some2) some-fixes; urgency=low +janitor (0.1-1jan+some2) some-fixes; urgency=medium + * Initial release. (Closes: #XXXXXX) * Dummy build. -- Jelmer Vernooij Sat, 05 Sep 2020 12:35:04 -0000 - -janitor (0.1-1jan+some1) UNRELEASED; urgency=medium - - * Initial release. (Closes: #XXXXXX) - - -- Jelmer Vernooij Sat, 04 Apr 2020 14:12:13 +0000 """, "debian/changelog", ) From 5f511b2b80bce29b2f45ad0ea5b981620cc30f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 17:53:58 +0100 Subject: [PATCH 11/65] Support calling out to apt-file. --- ognibuild/buildsystem.py | 1 - ognibuild/debian/apt.py | 5 +- ognibuild/debian/file_search.py | 86 ++++++++++++++++++++++++++++----- 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index fd40ee6..f33dd59 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -221,7 +221,6 @@ import sys script_name = %(script_name)s -save_argv = sys.argv.copy() g = {"__file__": script_name, "__name__": "__main__"} try: core._setup_stop_after = "init" diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 7c168aa..2451a2b 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -28,7 +28,7 @@ from .. import DetailedFailure, UnidentifiedError from ..session import Session, run_with_tee, get_user from .file_search import ( FileSearcher, - AptCachedContentsFileSearcher, + get_apt_contents_file_searcher, GENERATED_FILE_SEARCHER, get_packages_for_paths, ) @@ -76,7 +76,7 @@ class AptManager(object): def searchers(self): if self._searchers is None: self._searchers = [ - AptCachedContentsFileSearcher.from_session(self.session), + get_apt_contents_file_searcher(self.session), GENERATED_FILE_SEARCHER, ] return self._searchers @@ -90,7 +90,6 @@ class AptManager(object): def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): logging.debug("Searching for packages containing %r", paths) - # TODO(jelmer): Make sure we use whatever is configured in self.session return get_packages_for_paths( paths, self.searchers(), regex=regex, case_insensitive=case_insensitive ) diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index d0d9ff4..b7054f5 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -21,11 +21,13 @@ from datetime import datetime from debian.deb822 import Release import os import re +import subprocess from typing import Iterator, List import logging from .. import USER_AGENT +from ..session import Session class FileSearcher(object): @@ -158,7 +160,68 @@ def load_apt_cache_file(url, cache_dir): raise FileNotFoundError(url) -class AptCachedContentsFileSearcher(FileSearcher): +class AptFileFileSearcher(FileSearcher): + + CACHE_IS_EMPTY_PATH = '/usr/share/apt-file/is-cache-empty' + + def __init__(self, session: Session): + self.session = session + + @classmethod + def has_cache(cls, session: Session) -> bool: + if not os.path.exists(session.external_path(cls.CACHE_IS_EMPTY_PATH)): + return True + try: + session.check_call([cls.CACHE_IS_EMPTY_PATH]) + except subprocess.CalledProcessError as e: + if e.returncode == 1: + return True + raise + else: + return False + + @classmethod + def from_session(cls, session): + logging.info('Using apt-file to search apt contents') + if not os.path.exists(session.external_path(cls.CACHE_IS_EMPTY_PATH)): + from .apt import AptManager + AptManager.from_session(session).install(['apt-file']) + if not cls.has_cache(session): + session.check_call(['apt-file', 'update'], user='root') + return cls(session) + + def search_files(self, path, regex=False, case_insensitive=False): + args = [] + if regex: + args.append('-x') + else: + args.append('-F') + if case_insensitive: + args.append('-i') + args.append(path) + try: + output = self.session.check_output(['/usr/bin/apt-file', 'search'] + args) + except subprocess.CalledProcessError as e: + if e.returncode == 1: + # No results + return + if e.returncode == 3: + raise Exception('apt-file cache is empty') + raise + + for line in output.splitlines(False): + pkg, path = line.split(b': ') + yield pkg.decode('utf-8') + + +def get_apt_contents_file_searcher(session): + if AptFileFileSearcher.has_cache(session): + return AptFileFileSearcher.from_session(session) + + return RemoteContentsFileSearcher.from_session(session) + + +class RemoteContentsFileSearcher(FileSearcher): def __init__(self): self._db = {} @@ -268,12 +331,12 @@ class GeneratedFileSearcher(FileSearcher): with open(path, "r") as f: for line in f: (path, pkg) = line.strip().split(None, 1) - self._db[path] = pkg + self._db.append(path, pkg) def search_files( self, path: str, regex: bool = False, case_insensitive: bool = False ) -> Iterator[str]: - for p, pkg in sorted(self._db.items()): + for p, pkg in self._db: if regex: flags = 0 if case_insensitive: @@ -290,16 +353,16 @@ class GeneratedFileSearcher(FileSearcher): # TODO(jelmer): read from a file GENERATED_FILE_SEARCHER = GeneratedFileSearcher( - { - "/etc/locale.gen": "locales", + [ + ("/etc/locale.gen", "locales"), # Alternative - "/usr/bin/rst2html": "python3-docutils", + ("/usr/bin/rst2html", "python3-docutils"), # aclocal is a symlink to aclocal-1.XY - "/usr/bin/aclocal": "automake", - "/usr/bin/automake": "automake", + ("/usr/bin/aclocal", "automake"), + ("/usr/bin/automake", "automake"), # maven lives in /usr/share - "/usr/bin/mvn": "maven", - } + ("/usr/bin/mvn", "maven"), + ] ) @@ -322,6 +385,7 @@ def get_packages_for_paths( def main(argv): import argparse + from ..session.plain import PlainSession parser = argparse.ArgumentParser() parser.add_argument("path", help="Path to search for.", type=str, nargs="*") @@ -334,7 +398,7 @@ def main(argv): else: logging.basicConfig(level=logging.INFO) - main_searcher = AptCachedContentsFileSearcher() + main_searcher = get_apt_contents_file_searcher(PlainSession()) main_searcher.load_local() searchers = [main_searcher, GENERATED_FILE_SEARCHER] From ce3e477c4ca602e7b95428cb6dc4d320eb5e2195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 18:11:48 +0100 Subject: [PATCH 12/65] Support introspection typelibs, improve default tie breaking. --- ognibuild/buildlog.py | 4 ++++ ognibuild/debian/build_deps.py | 6 +++--- ognibuild/debian/file_search.py | 2 +- ognibuild/requirements.py | 5 +++++ ognibuild/resolver/apt.py | 19 ++++++++++++++++++- 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index fabb279..a851e14 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -53,6 +53,7 @@ from buildlog_consultant.common import ( MissingVagueDependency, DhAddonLoadFailure, MissingMavenArtifacts, + MissingIntrospectionTypelib, GnomeCommonMissing, MissingGnomeCommonDependency, UnknownCertificateAuthority, @@ -99,6 +100,7 @@ from .requirements import ( X11Requirement, LibtoolRequirement, VagueDependencyRequirement, + IntrospectionTypelibRequirement, ) from .resolver import UnsatisfiedRequirements @@ -112,6 +114,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return PkgConfigRequirement(problem.module, problem.minimum_version) elif isinstance(problem, MissingCHeader): return CHeaderRequirement(problem.header) + elif isinstance(problem, MissingIntrospectionTypelib): + return IntrospectionTypelibRequirement(problem.library) elif isinstance(problem, MissingJavaScriptRuntime): return JavaScriptRuntimeRequirement() elif isinstance(problem, MissingRubyGem): diff --git a/ognibuild/debian/build_deps.py b/ognibuild/debian/build_deps.py index 888325f..246a058 100644 --- a/ognibuild/debian/build_deps.py +++ b/ognibuild/debian/build_deps.py @@ -65,9 +65,9 @@ class BuildDependencyTieBreaker(object): return None top = max(by_count.items(), key=lambda k: k[1]) logging.info( - "Breaking tie between %r to %r based on build-depends count", - [repr(r) for r in reqs], - top[0], + "Breaking tie between [%s] to %s based on build-depends count", + ', '.join([repr(r.pkg_relation_str()) for r in reqs]), + repr(top[0].pkg_relation_str()), ) return top[0] diff --git a/ognibuild/debian/file_search.py b/ognibuild/debian/file_search.py index b7054f5..4f33ebf 100644 --- a/ognibuild/debian/file_search.py +++ b/ognibuild/debian/file_search.py @@ -170,7 +170,7 @@ class AptFileFileSearcher(FileSearcher): @classmethod def has_cache(cls, session: Session) -> bool: if not os.path.exists(session.external_path(cls.CACHE_IS_EMPTY_PATH)): - return True + return False try: session.check_call([cls.CACHE_IS_EMPTY_PATH]) except subprocess.CalledProcessError as e: diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 014bee9..71219df 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -587,6 +587,11 @@ class LibtoolRequirement(Requirement): super(LibtoolRequirement, self).__init__("libtool") +class IntrospectionTypelibRequirement(Requirement): + def __init__(self, library): + self.library = library + + class PythonModuleRequirement(Requirement): module: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 618206c..80dd090 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -68,6 +68,7 @@ from ..requirements import ( CertificateAuthorityRequirement, LibtoolRequirement, VagueDependencyRequirement, + IntrospectionTypelibRequirement, ) @@ -620,6 +621,12 @@ def resolve_ca_req(apt_mgr, req): return [AptRequirement.simple("ca-certificates")] +def resolve_introspection_typelib_req(apt_mgr, req): + return find_reqs_simple( + apt_mgr, ['/usr/lib/.*/girepository-.*/%s-.*\.typelib' % re.escape(req.library)], + regex=True) + + def resolve_apt_req(apt_mgr, req): # TODO(jelmer): This should be checking whether versions match as well. for package_name in req.package_names(): @@ -668,6 +675,7 @@ APT_REQUIREMENT_RESOLVERS = [ (PythonPackageRequirement, resolve_python_package_req), (CertificateAuthorityRequirement, resolve_ca_req), (CargoCrateRequirement, resolve_cargo_crate_req), + (IntrospectionTypelibRequirement, resolve_introspection_typelib_req), ] @@ -683,11 +691,20 @@ def resolve_requirement_apt(apt_mgr, req: Requirement) -> List[AptRequirement]: raise NotImplementedError(type(req)) +def default_tie_breakers(session): + from ..debian.udd import popcon_tie_breaker + from ..debian.build_deps import BuildDependencyTieBreaker + return [ + BuildDependencyTieBreaker.from_session(session), + popcon_tie_breaker, + ] + + class AptResolver(Resolver): def __init__(self, apt, tie_breakers=None): self.apt = apt if tie_breakers is None: - tie_breakers = [] + tie_breakers = default_tie_breakers(apt.session) self.tie_breakers = tie_breakers def __str__(self): From 02aa7372419aa7672372c2778527691768173160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 29 Mar 2021 18:42:09 +0100 Subject: [PATCH 13/65] Don't require cwd when path is absolute. --- ognibuild/session/schroot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index f7f45b2..ff5b436 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -185,6 +185,8 @@ class SchrootSession(Session): self.check_call(["chown", user, home], cwd="/", user="root") def external_path(self, path: str) -> str: + if os.path.isabs(path): + return os.path.join(self.location, path.lstrip("/")) if self._cwd is None: raise ValueError("no cwd set") return os.path.join(self.location, os.path.join(self._cwd, path).lstrip("/")) From 10448e36d99df8e4faa5e183dc290935236fa93a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 00:18:41 +0100 Subject: [PATCH 14/65] Don't run 'gbp dch' when not at root of the tree. --- ognibuild/debian/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index deafe39..4da2fc0 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -239,7 +239,7 @@ def attempt_build( source_date_epoch: Source date epoch to set Returns: Tuple with (changes_name, cl_version) """ - if run_gbp_dch: + if run_gbp_dch and not subpath: gbp_dch(local_tree.abspath(subpath)) if build_changelog_entry is not None: add_dummy_changelog_entry( From 9ec80081e0a519282a8117c94e89ee1aa1c8d33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 01:21:27 +0100 Subject: [PATCH 15/65] Fix packages. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d3676b2..7382e99 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup(name="ognibuild", maintainer_email="jelmer@jelmer.uk", license="GNU GPLv2 or later", url="https://jelmer.uk/code/ognibuild", - packages=['ognibuild'], + packages=['ognibuild', 'ognibuild.tests', 'ognibuild.debian', 'ognibuild.resolver', 'ognibuild.session'], classifiers=[ 'Development Status :: 4 - Beta', 'License :: OSI Approved :: ' From 3dd8e6d8145bef8d12aae7211c70f87cac00a770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 03:38:24 +0100 Subject: [PATCH 16/65] Fix haskell package finding. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 80dd090..375f161 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -486,7 +486,7 @@ def resolve_cmake_file_req(apt_mgr, req): def resolve_haskell_package_req(apt_mgr, req): - path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) + path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.package) return find_reqs_simple(apt_mgr, [path], regex=True) From 19ca52908eba04bbfb727a09bcafddf913ce577b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 17:32:14 +0100 Subject: [PATCH 17/65] Prefer matching dev packages. --- ognibuild/debian/fix_build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 4de6403..6b76b70 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -22,6 +22,7 @@ __all__ = [ from functools import partial import logging import os +import re import shutil import sys from typing import List, Set, Optional, Type @@ -304,6 +305,8 @@ def python_tie_breaker(tree, subpath, reqs): return True if pkg.startswith("lib%s-" % python_version): return True + if re.match('lib%s\.[0-9]-dev' % python_version, pkg): + return True return False for python_version in targeted: From dc9a521a73e81a63e5bcfe0abb2622391f066f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 18:23:31 +0100 Subject: [PATCH 18/65] R: Handle minimum versions and LinkingTo field. --- ognibuild/buildsystem.py | 3 +++ ognibuild/resolver/apt.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index f33dd59..a0352a2 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -709,6 +709,9 @@ class R(BuildSystem): if "Imports" in description: for s in parse_list(description["Imports"]): yield "build", RPackageRequirement.from_str(s) + if "LinkingTo" in description: + for s in parse_list(description["LinkingTo"]): + yield "build", RPackageRequirement.from_str(s) def get_declared_outputs(self, session, fixers=None): description = self._read_description() diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 375f161..c290273 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -396,7 +396,7 @@ def resolve_r_package_req(apt_mgr, req): paths = [ posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package)) ] - return find_reqs_simple(apt_mgr, paths, regex=True) + return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version) def resolve_node_module_req(apt_mgr, req): From 4d052ebb44e0379e01c85e3a17bc9ceeabe40ed1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 19:15:20 +0100 Subject: [PATCH 19/65] Fix R package finding. --- ognibuild/resolver/apt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index c290273..ed67b89 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -394,9 +394,9 @@ def resolve_php_package_req(apt_mgr, req): def resolve_r_package_req(apt_mgr, req): paths = [ - posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package)) + posixpath.join("/usr/lib/R/site-library", req.package) ] - return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version) + return find_reqs_simple(apt_mgr, paths, minimum_version=req.minimum_version) def resolve_node_module_req(apt_mgr, req): From 7629a1932eb6c701ea231656367372a81b6b4d0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 19:17:41 +0100 Subject: [PATCH 20/65] Make max iterations a flag. --- ognibuild/debian/fix_build.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 6b76b70..1a63760 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -607,6 +607,11 @@ def main(argv=None): dest="update_changelog", help="do not update the changelog", ) + parser.add_argument( + '--max-iterations', + type=int, + default=DEFAULT_MAX_ITERATIONS, + help='Maximum number of issues to attempt to fix before giving up.') parser.add_argument( "--update-changelog", action="store_true", @@ -660,6 +665,7 @@ def main(argv=None): None, committer=args.committer, update_changelog=args.update_changelog, + max_iterations=args.max_iterations, ) except SbuildFailure as e: if e.phase is None: @@ -676,7 +682,7 @@ def main(argv=None): logging.info( 'Built %s - changes file at %r.', - os.path.join(output_directory, changes_files)) + os.path.join(output_directory, changes_filenames)) if __name__ == "__main__": From d906ec1d1c0cb79cb44c96c9de3562c1e9655365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 19:33:54 +0100 Subject: [PATCH 21/65] Handle perl predeclared functions. --- ognibuild/buildlog.py | 4 ++++ ognibuild/requirements.py | 26 ++++++++++++++++++++++++++ ognibuild/resolver/apt.py | 12 ++++++++++++ 3 files changed, 42 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index a851e14..ec666d5 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -61,6 +61,7 @@ from buildlog_consultant.common import ( MissingLibtool, MissingQt, MissingX11, + MissingPerlPredeclared, ) from .fix_build import BuildFixer @@ -101,6 +102,7 @@ from .requirements import ( LibtoolRequirement, VagueDependencyRequirement, IntrospectionTypelibRequirement, + PerlPreDeclaredRequirement, ) from .resolver import UnsatisfiedRequirements @@ -173,6 +175,8 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return LibtoolRequirement() elif isinstance(problem, UnknownCertificateAuthority): return CertificateAuthorityRequirement(problem.url) + elif isinstance(problem, MissingPerlPredeclared): + return PerlPreDeclaredRequirement(problem.name) elif isinstance(problem, MissingSetupPyCommand): if problem.command == "test": return PythonPackageRequirement("setuptools") diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 71219df..7c42fa6 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -192,6 +192,32 @@ class NodePackageRequirement(Requirement): return "%s(%r)" % (type(self).__name__, self.package) +class PerlPreDeclaredRequirement(Requirement): + + name: str + + # TODO(jelmer): Can we obtain this information elsewhere? + KNOWN_MODULES = { + 'auto_set_repository': 'Module::Install::Repository', + 'author_tests': 'Module::Install::AuthorTests', + 'readme_from': 'Module::Install::ReadmeFromPod', + 'catalyst': 'Module::Install::Catalyst', + 'githubmeta': 'Module::Install::GithubMeta', + 'use_ppport': 'Module::Install::XSUtil', + } + + def __init__(self, name): + super(PerlPreDeclaredRequirement, self).__init__("perl-predeclared") + self.name = name + + def lookup_module(self): + module = self.KNOWN_MODULES[self.name] + return PerlModuleRequirement(module=module) + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.name) + + class NodeModuleRequirement(Requirement): module: str diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index ed67b89..393194c 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -68,6 +68,7 @@ from ..requirements import ( CertificateAuthorityRequirement, LibtoolRequirement, VagueDependencyRequirement, + PerlPreDeclaredRequirement, IntrospectionTypelibRequirement, ) @@ -117,6 +118,16 @@ class AptRequirement(Requirement): return False +def resolve_perl_predeclared_req(apt_mgr, req): + try: + req = req.lookup_module() + except KeyError: + logging.warning( + 'Unable to map predeclared function %s to a perl module', req.name) + return None + return resolve_perl_module_req(apt_mgr, req) + + def find_package_names( apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False ) -> List[str]: @@ -639,6 +650,7 @@ APT_REQUIREMENT_RESOLVERS = [ (AptRequirement, resolve_apt_req), (BinaryRequirement, resolve_binary_req), (VagueDependencyRequirement, resolve_vague_dep_req), + (PerlPreDeclaredRequirement, resolve_perl_predeclared_req), (PkgConfigRequirement, resolve_pkg_config_req), (PathRequirement, resolve_path_req), (CHeaderRequirement, resolve_c_header_req), From 7d1b7ed4ed0911b907440d6616648153bcd4a3f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 19:39:35 +0100 Subject: [PATCH 22/65] Some more logging. --- ognibuild/debian/apt.py | 1 + ognibuild/resolver/__init__.py | 32 ++++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index 2451a2b..c7926ae 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -41,6 +41,7 @@ def run_apt( if prefix is None: prefix = [] args = prefix = ["apt", "-y"] + args + logging.info('apt: running %r', args) retcode, lines = run_with_tee(session, args, cwd="/", user="root") if retcode == 0: return diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 5460fec..4996929 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -16,6 +16,7 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import logging import subprocess from ..fix_build import run_detecting_problems @@ -88,9 +89,11 @@ class CPANResolver(Resolver): if not isinstance(requirement, PerlModuleRequirement): missing.append(requirement) continue + cmd = self._cmd([requirement]) + logging.info("CPAN: running %r", cmd) run_detecting_problems( self.session, - self._cmd([requirement]), + cmd, env=env, user=user, ) @@ -142,7 +145,9 @@ class RResolver(Resolver): if not isinstance(requirement, RPackageRequirement): missing.append(requirement) continue - run_detecting_problems(self.session, self._cmd(requirement), user=user) + cmd = self._cmd(requirement) + logging.info("RResolver(%r): running %r", self.repos, cmd) + run_detecting_problems(self.session, cmd, user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -186,7 +191,9 @@ class OctaveForgeResolver(Resolver): if not isinstance(requirement, OctavePackageRequirement): missing.append(requirement) continue - run_detecting_problems(self.session, self._cmd(requirement), user=user) + cmd = self._cmd(requirement) + logging.info("Octave: running %r", cmd) + run_detecting_problems(self.session, cmd, user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -235,7 +242,9 @@ class HackageResolver(Resolver): if not isinstance(requirement, HaskellPackageRequirement): missing.append(requirement) continue - run_detecting_problems(self.session, self._cmd([requirement]), user=user) + cmd = self._cmd([requirement]) + logging.info("Hackage: running %r", cmd) + run_detecting_problems(self.session, cmd, user=user) if missing: raise UnsatisfiedRequirements(missing) @@ -281,8 +290,10 @@ class PypiResolver(Resolver): if not isinstance(requirement, PythonPackageRequirement): missing.append(requirement) continue + cmd = self._cmd([requirement]) + logging.info("pip: running %r", cmd) try: - run_detecting_problems(self.session, self._cmd([requirement]), user=user) + run_detecting_problems(self.session, cmd, user=user) except subprocess.CalledProcessError: missing.append(requirement) if missing: @@ -325,7 +336,9 @@ class GoResolver(Resolver): if not isinstance(requirement, GoPackageRequirement): missing.append(requirement) continue - run_detecting_problems(self.session, ["go", "get", requirement.package], env=env) + cmd = ["go", "get", requirement.package] + logging.info("go: running %r", cmd) + run_detecting_problems(self.session, cmd, env=env) if missing: raise UnsatisfiedRequirements(missing) @@ -387,10 +400,9 @@ class NpmResolver(Resolver): if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue - run_detecting_problems( - self.session, - ["npm", "-g", "install", requirement.package], user=user - ) + cmd = ["npm", "-g", "install", requirement.package] + logging.info("npm: running %r", cmd) + run_detecting_problems(self.session, cmd, user=user) if missing: raise UnsatisfiedRequirements(missing) From 814960809cf6810d918c0c0f84481577363a2afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 19:43:23 +0100 Subject: [PATCH 23/65] Add more vague dep mappings. --- ognibuild/resolver/apt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 393194c..27ad5cf 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -294,6 +294,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): vague_map = { "the Gnu Scientific Library": "libgsl-dev", "the required FreeType library": "libfreetype-dev", + "PythonLibs": "libpython3-dev", + "ZLIB": "libz3-dev", } From 2231a788167ed68a9ba4db899857e13883c4ee0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 20:17:29 +0100 Subject: [PATCH 24/65] More perl support. --- ognibuild/buildsystem.py | 58 ++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index a0352a2..56d5b0e 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -985,6 +985,26 @@ def _declared_deps_from_cpanfile(session, fixers): yield from _read_cpanfile(session, ["--test"], "test", fixers) +def _declared_deps_from_meta_yml(f): + # See http://module-build.sourceforge.net/META-spec-v1.4.html for + # the specification of the format. + import ruamel.yaml + import ruamel.yaml.reader + + try: + data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) + except ruamel.yaml.reader.ReaderError as e: + warnings.warn("Unable to parse META.yml: %s" % e) + return + for require in data.get("requires", []): + yield "core", PerlModuleRequirement(require) + for require in data.get("build_requires", []): + yield "build", PerlModuleRequirement(require) + for require in data.get("configure_requires", []): + yield "build", PerlModuleRequirement(require) + # TODO(jelmer): recommends + + class Make(BuildSystem): name = "make" @@ -1105,24 +1125,9 @@ class Make(BuildSystem): something = False # TODO(jelmer): Split out the perl-specific stuff? if os.path.exists(os.path.join(self.path, "META.yml")): - # See http://module-build.sourceforge.net/META-spec-v1.4.html for - # the specification of the format. - import ruamel.yaml - import ruamel.yaml.reader - with open(os.path.join(self.path, "META.yml"), "rb") as f: - try: - data = ruamel.yaml.load(f, ruamel.yaml.SafeLoader) - except ruamel.yaml.reader.ReaderError as e: - warnings.warn("Unable to parse META.yml: %s" % e) - return - for require in data.get("requires", []): - yield "core", PerlModuleRequirement(require) - for require in data.get("build_requires", []): - yield "build", PerlModuleRequirement(require) - for require in data.get("configure_requires", []): - yield "build", PerlModuleRequirement(require) - something = True + yield from _declared_deps_from_meta_yml(f) + something = True if os.path.exists(os.path.join(self.path, "cpanfile")): yield from _declared_deps_from_cpanfile(session, fixers) something = True @@ -1415,10 +1420,29 @@ class PerlBuildTiny(BuildSystem): self.setup(session, fixers) run_with_build_fixers(session, ["./Build", "clean"], fixers) + def dist(self, session, resolver, fixers, target_directory, quiet=False): + self.setup(session, fixers) + with DistCatcher([session.external_path('.')]) as dc: + try: + run_with_build_fixers(session, ["./Build", "dist"], fixers) + except UnidentifiedError as e: + if "Can't find dist packages without a MANIFEST file" in e.lines: + run_with_build_fixers(session, ["./Build", "manifest"], fixers) + run_with_build_fixers(session, ["./Build", "dist"], fixers) + else: + raise + return dc.copy_single(target_directory) + def install(self, session, resolver, fixers, install_target): self.setup(session, fixers) run_with_build_fixers(session, ["./Build", "install"], fixers) + def get_declared_dependencies(self, session, fixers=None): + self.setup(session, fixers) + run_with_build_fixers(session, ["./Build", "distmeta"], fixers) + with open(os.path.join(self.path, 'META.yml'), 'r') as f: + yield from _declared_deps_from_meta_yml(f) + @classmethod def probe(cls, path): if os.path.exists(os.path.join(path, "Build.PL")): From 86cf658a81dacbff034d7fc239af4e55dfa58676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 30 Mar 2021 22:16:00 +0100 Subject: [PATCH 25/65] Add vague dependecy. --- ognibuild/buildsystem.py | 1 + ognibuild/resolver/apt.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 56d5b0e..825e6fa 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -712,6 +712,7 @@ class R(BuildSystem): if "LinkingTo" in description: for s in parse_list(description["LinkingTo"]): yield "build", RPackageRequirement.from_str(s) + # TODO(jelmer): Suggests def get_declared_outputs(self, session, fixers=None): description = self._read_description() diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 27ad5cf..13f6115 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -294,6 +294,7 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs): vague_map = { "the Gnu Scientific Library": "libgsl-dev", "the required FreeType library": "libfreetype-dev", + "the Boost C++ libraries": "libboost-dev", "PythonLibs": "libpython3-dev", "ZLIB": "libz3-dev", } From 7326002a37c5f6c4b1d7f7e56d75e52d1730a34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 31 Mar 2021 00:19:27 +0100 Subject: [PATCH 26/65] Add another vague dependency. --- ognibuild/resolver/apt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 13f6115..f68377c 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -295,8 +295,11 @@ vague_map = { "the Gnu Scientific Library": "libgsl-dev", "the required FreeType library": "libfreetype-dev", "the Boost C++ libraries": "libboost-dev", + + # TODO(jelmer): Support resolving virtual packages "PythonLibs": "libpython3-dev", "ZLIB": "libz3-dev", + "Osmium": "libosmium2-dev", } From add0ae783c0db0a6573079b41aa3528c8266597e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 31 Mar 2021 01:50:06 +0100 Subject: [PATCH 27/65] Try to resolve 'Python'. --- ognibuild/resolver/apt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index f68377c..dae2554 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -300,6 +300,9 @@ vague_map = { "PythonLibs": "libpython3-dev", "ZLIB": "libz3-dev", "Osmium": "libosmium2-dev", + + # TODO(jelmer): For Python, check minimum_version and map to python 2 or python 3 + "Python": "libpython3-dev", } @@ -307,7 +310,7 @@ def resolve_vague_dep_req(apt_mgr, req): name = req.name options = [] if name in vague_map: - options.append(AptRequirement.simple(vague_map[name])) + options.append(AptRequirement.simple(vague_map[name], minimum_version=req.minimum_version)) for x in req.expand(): options.extend(resolve_requirement_apt(apt_mgr, x)) return options From 4f2a27dfba4f6f5a4b2b0f44942d72575c12bd74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 31 Mar 2021 02:37:22 +0100 Subject: [PATCH 28/65] Add another vague dependency. --- ognibuild/resolver/apt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index dae2554..8c52202 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -300,6 +300,7 @@ vague_map = { "PythonLibs": "libpython3-dev", "ZLIB": "libz3-dev", "Osmium": "libosmium2-dev", + "glib": "libglib2.0-dev", # TODO(jelmer): For Python, check minimum_version and map to python 2 or python 3 "Python": "libpython3-dev", From 41995f9bc1613b7e6e0690f47536f37d2d28fa60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Wed, 31 Mar 2021 02:42:05 +0100 Subject: [PATCH 29/65] Add another alias. --- ognibuild/resolver/apt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 8c52202..4e7aa89 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -304,6 +304,7 @@ vague_map = { # TODO(jelmer): For Python, check minimum_version and map to python 2 or python 3 "Python": "libpython3-dev", + "Lua": "liblua5.4-dev", } From 6712cadc1958117a8deb9d6e09d3315e975c9c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 15:51:44 +0100 Subject: [PATCH 30/65] Fix style. --- ognibuild/debian/build.py | 2 +- ognibuild/debian/fix_build.py | 2 +- ognibuild/resolver/apt.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 4da2fc0..8c35f89 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -31,7 +31,7 @@ import subprocess import sys from debian.changelog import Changelog -from debmutate.changelog import get_maintainer, format_datetime +from debmutate.changelog import get_maintainer from breezy.mutabletree import MutableTree from breezy.plugins.debian.builder import BuildFailedError diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 1a63760..df42eff 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -305,7 +305,7 @@ def python_tie_breaker(tree, subpath, reqs): return True if pkg.startswith("lib%s-" % python_version): return True - if re.match('lib%s\.[0-9]-dev' % python_version, pkg): + if re.match(r'lib%s\.[0-9]-dev' % python_version, pkg): return True return False diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 4e7aa89..77ee9c3 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -645,7 +645,7 @@ def resolve_ca_req(apt_mgr, req): def resolve_introspection_typelib_req(apt_mgr, req): return find_reqs_simple( - apt_mgr, ['/usr/lib/.*/girepository-.*/%s-.*\.typelib' % re.escape(req.library)], + apt_mgr, [r'/usr/lib/.*/girepository-.*/%s-.*\.typelib' % re.escape(req.library)], regex=True) From c748bca228b44f0b2f8eb8b2db9fbdcd4782635a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 16:22:37 +0100 Subject: [PATCH 31/65] Install testtools. --- .github/workflows/pythonpackage.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 9c0a80c..0725c4c 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -28,6 +28,7 @@ jobs: python -m pip install wheel python -m pip install git+https://salsa.debian.org/apt-team/python-apt python -m pip install -e ".[debian]" + python -m pip install testtools mkdir -p ~/.config/breezy/plugins brz branch lp:brz-debian ~/.config/breezy/plugins/debian if: "matrix.python-version != 'pypy3' && matrix.os == 'ubuntu-latest'" From e52e90dd894ef88949209878e7cc645652ef24f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 17:50:16 +0100 Subject: [PATCH 32/65] break endless loop --- ognibuild/buildsystem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 825e6fa..9ec082d 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1213,6 +1213,7 @@ def _parse_go_mod(f): while line: parts = line.strip().split(" ") if not parts or parts == [""]: + line = readline() continue if len(parts) == 2 and parts[1] == "(": line = readline() From b807d6cd66f3f1752a86bc87382a89940201dd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 18:27:41 +0100 Subject: [PATCH 33/65] Fix npm install. --- ognibuild/resolver/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 4996929..1fda055 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -396,7 +396,11 @@ class NpmResolver(Resolver): requirement = NodePackageRequirement(package) if isinstance(requirement, NodeModuleRequirement): # TODO: Is this legit? - requirement = NodePackageRequirement(requirement.module.split("/")[0]) + parts = requirement.module.split("/")[0] + if parts[0].startswith('@'): + requirement = NodePackageRequirement('/'.join(parts:2])) + else: + requirement = NodePackageRequirement(parts[0]) if not isinstance(requirement, NodePackageRequirement): missing.append(requirement) continue From f9e2d16131c0e8e398ff550d0f8f4f377156aee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 18:41:34 +0100 Subject: [PATCH 34/65] handle lack of dist. --- ognibuild/buildsystem.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9ec082d..9478ef3 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1431,6 +1431,8 @@ class PerlBuildTiny(BuildSystem): if "Can't find dist packages without a MANIFEST file" in e.lines: run_with_build_fixers(session, ["./Build", "manifest"], fixers) run_with_build_fixers(session, ["./Build", "dist"], fixers) + elif "No such action 'dist'" in e.lines: + raise NotImplementedError else: raise return dc.copy_single(target_directory) From f5433db674482135baee9c092df734e43558ca5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 18:47:15 +0100 Subject: [PATCH 35/65] Add basic minilla support. --- ognibuild/buildsystem.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9478ef3..9668026 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1403,6 +1403,7 @@ class PerlBuildTiny(BuildSystem): def __init__(self, path): self.path = path + self.minilla = os.path.exists(os.path.join(self.path, "minil.toml")) def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) @@ -1412,7 +1413,10 @@ class PerlBuildTiny(BuildSystem): def test(self, session, resolver, fixers): self.setup(session, fixers) - run_with_build_fixers(session, ["./Build", "test"], fixers) + if self.minilla: + run_with_build_fixers(session, ["minil", "test"], fixers) + else: + run_with_build_fixers(session, ["./Build", "test"], fixers) def build(self, session, resolver, fixers): self.setup(session, fixers) @@ -1425,21 +1429,27 @@ class PerlBuildTiny(BuildSystem): def dist(self, session, resolver, fixers, target_directory, quiet=False): self.setup(session, fixers) with DistCatcher([session.external_path('.')]) as dc: - try: - run_with_build_fixers(session, ["./Build", "dist"], fixers) - except UnidentifiedError as e: - if "Can't find dist packages without a MANIFEST file" in e.lines: - run_with_build_fixers(session, ["./Build", "manifest"], fixers) + if self.minilla: + run_with_build_fixers(session, ["minil", "dist"], fixers) + else: + try: run_with_build_fixers(session, ["./Build", "dist"], fixers) - elif "No such action 'dist'" in e.lines: - raise NotImplementedError - else: - raise + except UnidentifiedError as e: + if "Can't find dist packages without a MANIFEST file" in e.lines: + run_with_build_fixers(session, ["./Build", "manifest"], fixers) + run_with_build_fixers(session, ["./Build", "dist"], fixers) + elif "No such action 'dist'" in e.lines: + raise NotImplementedError + else: + raise return dc.copy_single(target_directory) def install(self, session, resolver, fixers, install_target): self.setup(session, fixers) - run_with_build_fixers(session, ["./Build", "install"], fixers) + if self.minilla: + run_with_build_fixers(session, ["minil", "install"], fixers) + else: + run_with_build_fixers(session, ["./Build", "install"], fixers) def get_declared_dependencies(self, session, fixers=None): self.setup(session, fixers) From edbac0513f5fc96d8c7025dfdc4013fad80a2da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 18:53:54 +0100 Subject: [PATCH 36/65] Update README. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2eb483d..24951b2 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ issues (or lack of support for a particular ecosystem), please file a bug. - Octave - Perl - Module::Build::Tiny + - Dist::Zilla + - Minilla - PHP Pear - Python - setup.py/setup.cfg/pyproject.toml - R From 697b279953b4349dbdc523abfef5a18cd51da720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 19:05:29 +0100 Subject: [PATCH 37/65] Fix syntax. --- ognibuild/resolver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 1fda055..1629894 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -398,7 +398,7 @@ class NpmResolver(Resolver): # TODO: Is this legit? parts = requirement.module.split("/")[0] if parts[0].startswith('@'): - requirement = NodePackageRequirement('/'.join(parts:2])) + requirement = NodePackageRequirement('/'.join(parts[:2])) else: requirement = NodePackageRequirement(parts[0]) if not isinstance(requirement, NodePackageRequirement): From 62b5af617f1fdc035c52027ffaeb3ca475671fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 21:56:28 +0100 Subject: [PATCH 38/65] Allow passing in fixers. --- ognibuild/debian/fix_build.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index df42eff..9db0774 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -500,6 +500,15 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]: ] +def default_fixers(local_tree, subpath, apt, committer=None, update_changelog=None): + packaging_context = DebianPackagingContext( + local_tree, subpath, committer, update_changelog + ) + return versioned_package_fixers(apt.session, packaging_context, apt) + apt_fixers( + apt, packaging_context + ) + + def build_incrementally( local_tree, apt, @@ -513,14 +522,13 @@ def build_incrementally( subpath="", source_date_epoch=None, update_changelog=True, + fixers=None ): fixed_errors = [] - packaging_context = DebianPackagingContext( - local_tree, subpath, committer, update_changelog - ) - fixers = versioned_package_fixers(apt.session, packaging_context, apt) + apt_fixers( - apt, packaging_context - ) + if fixers is None: + fixers = default_fixers( + local_tree, subpath, apt, committer=committer, + update_changelog=update_changelog) logging.info("Using fixers: %r", fixers) while True: try: From d4d1d2cf4c57ec90ce4d00aa83bb7d7806b97d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Thu, 1 Apr 2021 23:22:12 +0100 Subject: [PATCH 39/65] search for R descriptin file. --- ognibuild/resolver/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 77ee9c3..65def68 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -416,7 +416,7 @@ def resolve_php_package_req(apt_mgr, req): def resolve_r_package_req(apt_mgr, req): paths = [ - posixpath.join("/usr/lib/R/site-library", req.package) + posixpath.join("/usr/lib/R/site-library", req.package, "DESCRIPTION") ] return find_reqs_simple(apt_mgr, paths, minimum_version=req.minimum_version) From 0a4c6f9608a58cbdb421f007e6b13ba6a2895a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 00:13:50 +0100 Subject: [PATCH 40/65] Fix pear. --- ognibuild/buildsystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9668026..b790c54 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -175,7 +175,7 @@ class Pear(BuildSystem): def probe(cls, path): if os.path.exists(os.path.join(path, "package.xml")): logging.debug("Found package.xml, assuming pear package.") - return cls(os.path.join(path, "package.xml")) + return cls(path) # run_setup, but setting __name__ From b4481b0a31c0ac78b78196d000284cd2cdd57490 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 01:51:10 +0100 Subject: [PATCH 41/65] Add basic ctan support. --- ognibuild/requirements.py | 9 +++++ ognibuild/resolver/__init__.py | 71 +++++++++++++++++++++++++++++++++- 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 7c42fa6..8b74c69 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -82,6 +82,15 @@ class PythonPackageRequirement(Requirement): return p.returncode == 0 +class LatexPackageRequirement(Requirement): + + def __init__(self, package: str): + self.package = package + + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.package) + + class PhpPackageRequirement(Requirement): def __init__( self, diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 1629894..0bf21e0 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -18,6 +18,7 @@ import logging import subprocess +from .. import UnidentifiedError from ..fix_build import run_detecting_problems @@ -68,7 +69,7 @@ class CPANResolver(Resolver): continue perlreqs.append(requirement) if perlreqs: - yield (self._cmd(perlreqs), [perlreqs]) + yield (self._cmd(perlreqs), perlreqs) def install(self, requirements): from ..requirements import PerlModuleRequirement @@ -101,6 +102,73 @@ class CPANResolver(Resolver): raise UnsatisfiedRequirements(missing) +class TlmgrResolver(Resolver): + def __init__(self, session, repository: str, user_local=False): + self.session = session + self.user_local = user_local + self.repository = repository + + def __str__(self): + if self.repository.startswith('http://') or self.repository.startswith('https://'): + return 'tlmgr(%r)' % self.repository + else: + return self.repository + + def __repr__(self): + return "%s(%r, %r)" % ( + type(self).__name__, self.session, self.repository) + + def _cmd(self, reqs): + ret = ["tlmgr", "--repository=%s" % self.repository, "install"] + if self.user_local: + ret.append("--usermode") + ret.extend([req.package for req in reqs]) + return ret + + def explain(self, requirements): + from ..requirements import LatexPackageRequirement + + latexreqs = [] + for requirement in requirements: + if not isinstance(requirement, LatexPackageRequirement): + continue + latexreqs.append(requirement) + if latexreqs: + yield (self._cmd(latexreqs), latexreqs) + + def install(self, requirements): + from ..requirements import LatexPackageRequirement + + if not self.user_local: + user = "root" + else: + user = None + + missing = [] + for requirement in requirements: + if not isinstance(requirement, LatexPackageRequirement): + missing.append(requirement) + continue + cmd = self._cmd([requirement]) + logging.info("tlmgr: running %r", cmd) + try: + run_detecting_problems(self.session, cmd, user=user) + except UnidentifiedError as e: + if "tlmgr: user mode not initialized, please read the documentation!" in e.lines: + self.session.check_call(['tlmgr', 'init-usertree']) + else: + raise + if missing: + raise UnsatisfiedRequirements(missing) + + +class CTANResolver(TlmgrResolver): + + def __init__(self, session, user_local=False): + super(CTANResolver, self).__init__( + session, "ctan", user_local=user_local) + + class RResolver(Resolver): def __init__(self, session, repos, user_local=False): self.session = session @@ -463,6 +531,7 @@ class StackedResolver(Resolver): NATIVE_RESOLVER_CLS = [ CPANResolver, + CTANResolver, PypiResolver, NpmResolver, GoResolver, From 4145f004a412e596932021efc84a0f540d8d541a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 01:52:41 +0100 Subject: [PATCH 42/65] Some initial latex package support. --- ognibuild/buildlog.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index ec666d5..81eec5d 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -62,6 +62,7 @@ from buildlog_consultant.common import ( MissingQt, MissingX11, MissingPerlPredeclared, + MissingLaTeXFile, ) from .fix_build import BuildFixer @@ -103,6 +104,7 @@ from .requirements import ( VagueDependencyRequirement, IntrospectionTypelibRequirement, PerlPreDeclaredRequirement, + LatexPackageRequirement, ) from .resolver import UnsatisfiedRequirements @@ -136,6 +138,10 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return NodeModuleRequirement(problem.module) elif isinstance(problem, MissingNodePackage): return NodePackageRequirement(problem.package) + elif isinstance(problem, MissingLaTeXFile): + if problem.filename.endswith('.sty'): + return LatexPackageRequirement(problem.filename[:-4]) + return None elif isinstance(problem, MissingVagueDependency): return VagueDependencyRequirement(problem.name, minimum_version=problem.minimum_version) elif isinstance(problem, MissingLibrary): From ad6c37f0c4739838d32347fc8214f5285f7a3f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 02:02:09 +0100 Subject: [PATCH 43/65] Capitalisation. --- ognibuild/buildlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 81eec5d..49164bf 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -62,7 +62,7 @@ from buildlog_consultant.common import ( MissingQt, MissingX11, MissingPerlPredeclared, - MissingLaTeXFile, + MissingLatexFile, ) from .fix_build import BuildFixer From 6bfccd9ab0d1fde4ccb12f67c938506070ad8283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 02:07:58 +0100 Subject: [PATCH 44/65] Fix casing. --- ognibuild/buildlog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 49164bf..3805b58 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -138,7 +138,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return NodeModuleRequirement(problem.module) elif isinstance(problem, MissingNodePackage): return NodePackageRequirement(problem.package) - elif isinstance(problem, MissingLaTeXFile): + elif isinstance(problem, MissingLatexFile): if problem.filename.endswith('.sty'): return LatexPackageRequirement(problem.filename[:-4]) return None From 9653795e260285ee889b1536e3c3088353c09f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 03:49:46 +0100 Subject: [PATCH 45/65] Fix npm package handling. --- ognibuild/resolver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/resolver/__init__.py b/ognibuild/resolver/__init__.py index 0bf21e0..5402454 100644 --- a/ognibuild/resolver/__init__.py +++ b/ognibuild/resolver/__init__.py @@ -464,7 +464,7 @@ class NpmResolver(Resolver): requirement = NodePackageRequirement(package) if isinstance(requirement, NodeModuleRequirement): # TODO: Is this legit? - parts = requirement.module.split("/")[0] + parts = requirement.module.split("/") if parts[0].startswith('@'): requirement = NodePackageRequirement('/'.join(parts[:2])) else: From 9ba2cd7bfda03a7beaaecbe907082f0fc107efdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 05:45:33 +0100 Subject: [PATCH 46/65] Pass fixers along, strip whitespace. --- ognibuild/__main__.py | 6 +++++- ognibuild/buildsystem.py | 2 +- ognibuild/fix_build.py | 6 ++++-- ognibuild/info.py | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index 8f82ca2..0de3307 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -150,6 +150,10 @@ def main(): # noqa: C901 external_dir, internal_dir = session.setup_from_directory(args.directory) session.chdir(internal_dir) os.chdir(external_dir) + + if not session.is_temporary and args.subcommand == 'info': + args.explain = True + if args.resolve == "apt": resolver = AptResolver.from_session(session) elif args.resolve == "native": @@ -211,7 +215,7 @@ def main(): # noqa: C901 if args.subcommand == "info": from .info import run_info - run_info(session, buildsystems=bss) + run_info(session, buildsystems=bss, fixers=fixers) except ExplainInstall as e: display_explain_commands(e.commands) except (UnidentifiedError, DetailedFailure): diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index b790c54..615d543 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -978,7 +978,7 @@ class RunTests(BuildSystem): def _read_cpanfile(session, args, kind, fixers): for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers): - yield kind, PerlModuleRequirement(line) + yield kind, PerlModuleRequirement(line.strip()) def _declared_deps_from_cpanfile(session, fixers): diff --git a/ognibuild/fix_build.py b/ognibuild/fix_build.py index e83b84b..90d67a2 100644 --- a/ognibuild/fix_build.py +++ b/ognibuild/fix_build.py @@ -17,7 +17,7 @@ from functools import partial import logging -from typing import List, Tuple, Callable, Any +from typing import List, Tuple, Callable, Any, Optional from buildlog_consultant import Problem from buildlog_consultant.common import ( @@ -104,8 +104,10 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]): def run_with_build_fixers( - session: Session, args: List[str], fixers: List[BuildFixer], **kwargs + session: Session, args: List[str], fixers: Optional[List[BuildFixer]], **kwargs ): + if fixers is None: + fixers = [] return iterate_with_build_fixers( fixers, partial(run_detecting_problems, session, args, **kwargs) ) diff --git a/ognibuild/info.py b/ognibuild/info.py index 0ee1cfd..c0bc425 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -16,12 +16,12 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -def run_info(session, buildsystems): +def run_info(session, buildsystems, fixers=None): for buildsystem in buildsystems: print("%r:" % buildsystem) deps = {} try: - for kind, dep in buildsystem.get_declared_dependencies(session): + for kind, dep in buildsystem.get_declared_dependencies(session, fixers=fixers): deps.setdefault(kind, []).append(dep) except NotImplementedError: print( @@ -35,7 +35,7 @@ def run_info(session, buildsystems): print("\t\t\t%s" % dep) print("") try: - outputs = list(buildsystem.get_declared_outputs(session)) + outputs = list(buildsystem.get_declared_outputs(session, fixers=fixers)) except NotImplementedError: print("\tUnable to detect declared outputs for this type of build system") outputs = [] From a56224a3a99aae7bacb99a77881804b24b209094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 05:46:02 +0100 Subject: [PATCH 47/65] Strip empty lines. --- ognibuild/buildsystem.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 615d543..9ef04d7 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -978,7 +978,9 @@ class RunTests(BuildSystem): def _read_cpanfile(session, args, kind, fixers): for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers): - yield kind, PerlModuleRequirement(line.strip()) + line = line.strip() + if line: + yield kind, PerlModuleRequirement(line) def _declared_deps_from_cpanfile(session, fixers): From 43937c83268face69caf7883a5b63e6c93776e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 13:05:33 +0100 Subject: [PATCH 48/65] Handle missing META.yml file. --- ognibuild/buildsystem.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 9ef04d7..ac2e9ce 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1456,8 +1456,11 @@ class PerlBuildTiny(BuildSystem): def get_declared_dependencies(self, session, fixers=None): self.setup(session, fixers) run_with_build_fixers(session, ["./Build", "distmeta"], fixers) - with open(os.path.join(self.path, 'META.yml'), 'r') as f: - yield from _declared_deps_from_meta_yml(f) + try: + with open(os.path.join(self.path, 'META.yml'), 'r') as f: + yield from _declared_deps_from_meta_yml(f) + except FileNotFoundError: + pass @classmethod def probe(cls, path): From 52b8694b5c0e86fbc5e3d0cc470cee286656b4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 16:20:59 +0100 Subject: [PATCH 49/65] Stop returning SbuildFailure. --- ognibuild/debian/build.py | 44 +++++++++++++++++++++++++++++++---- ognibuild/debian/fix_build.py | 21 +++++++++++------ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 8c35f89..c25b3d5 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -19,7 +19,8 @@ __all__ = [ "get_build_architecture", "add_dummy_changelog_entry", "build", - "SbuildFailure", + "DetailedDebianBuildFailure", + "UnidentifiedDebianBuildError", ] from datetime import datetime @@ -27,6 +28,7 @@ from debmutate.changelog import ChangelogEditor import logging import os import re +import shlex import subprocess import sys @@ -39,13 +41,33 @@ from breezy.tree import Tree from buildlog_consultant.sbuild import ( worker_failure_from_sbuild_log, - SbuildFailure, ) +from .. import DetailedFailure as DetailedFailure, UnidentifiedError + DEFAULT_BUILDER = "sbuild --no-clean-source" +class DetailedDebianBuildFailure(DetailedFailure): + + def __init__(self, stage, phase, retcode, argv, error, description): + super(DetailedDebianBuildFailure, self).__init__(retcode, argv, error) + self.stage = stage + self.phase = phase + self.description = description + + +class UnidentifiedDebianBuildError(UnidentifiedError): + + def __init__(self, stage, phase, retcode, argv, lines, description, secondary=None): + super(UnidentifiedDebianBuildError, self).__init__( + retcode, argv, lines, secondary) + self.stage = stage + self.phase = phase + self.description = description + + class MissingChangesFile(Exception): """Expected changes file was not written.""" @@ -200,9 +222,23 @@ def build_once( subpath=subpath, source_date_epoch=source_date_epoch, ) - except BuildFailedError: + except BuildFailedError as e: with open(build_log_path, "rb") as f: - raise worker_failure_from_sbuild_log(f) + sbuild_failure = worker_failure_from_sbuild_log(f) + retcode = getattr(e, 'returncode', None) + if sbuild_failure.error: + raise DetailedDebianBuildFailure( + sbuild_failure.stage, + sbuild_failure.phase, retcode, + shlex.split(build_command), + sbuild_failure.error, + sbuild_failure.description) + else: + raise UnidentifiedDebianBuildError( + sbuild_failure.stage, + sbuild_failure.phase, + retcode, shlex.split(build_command), + [], sbuild_failure.description) (cl_package, cl_version) = get_latest_changelog_version(local_tree, subpath) changes_names = [] diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 9db0774..4a2a112 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -111,10 +111,11 @@ from buildlog_consultant.common import ( NeedPgBuildExtUpdateControl, MissingPerlFile, ) -from buildlog_consultant.sbuild import ( - SbuildFailure, -) +from . import ( + DetailedDebianBuildFailure, + UnidentifiedDebianBuildError, + ) from ..buildlog import problem_to_upstream_requirement from ..fix_build import BuildFixer, resolve_error from ..resolver.apt import ( @@ -675,17 +676,23 @@ def main(argv=None): update_changelog=args.update_changelog, max_iterations=args.max_iterations, ) - except SbuildFailure as e: + except DetailedDebianBuildFailure as e: if e.phase is None: phase = "unknown phase" elif len(e.phase) == 1: phase = e.phase[0] else: phase = "%s (%s)" % (e.phase[0], e.phase[1]) - if e.error: - logging.fatal("Error during %s: %s", phase, e.error) + logging.fatal("Error during %s: %s", phase, e.error) + return 1 + except UnidentifiedDebianBuildError as e: + if e.phase is None: + phase = "unknown phase" + elif len(e.phase) == 1: + phase = e.phase[0] else: - logging.fatal("Error during %s: %s", phase, e.description) + phase = "%s (%s)" % (e.phase[0], e.phase[1]) + logging.fatal("Error during %s: %s", phase, e.description) return 1 logging.info( From 0ee47771dbabd48ac147b35d34e5cfbfe3ec6fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 17:33:29 +0100 Subject: [PATCH 50/65] Fix import --- ognibuild/debian/fix_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 4a2a112..078e396 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -112,7 +112,7 @@ from buildlog_consultant.common import ( MissingPerlFile, ) -from . import ( +from .build import ( DetailedDebianBuildFailure, UnidentifiedDebianBuildError, ) From f9e757760b4bc3d310f5afe2d2d227e4728594b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Fri, 2 Apr 2021 17:57:25 +0100 Subject: [PATCH 51/65] Fix build. --- ognibuild/debian/fix_build.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 078e396..c125220 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -544,10 +544,10 @@ def build_incrementally( source_date_epoch=source_date_epoch, run_gbp_dch=(update_changelog is False) ) - except SbuildFailure as e: - if e.error is None: - logging.warning("Build failed with unidentified error. Giving up.") - raise + except UnidentifiedDebianBuildError: + logging.warning("Build failed with unidentified error. Giving up.") + raise + except DetailedDebianBuildFailure as e: if e.phase is None: logging.info("No relevant context, not making any changes.") raise From 90f740fd4a7173717ce20df78c28b12c91fffa57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 00:38:17 +0100 Subject: [PATCH 52/65] Handle distmeta not being supported. --- ognibuild/buildsystem.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index ac2e9ce..76604e8 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -1455,7 +1455,13 @@ class PerlBuildTiny(BuildSystem): def get_declared_dependencies(self, session, fixers=None): self.setup(session, fixers) - run_with_build_fixers(session, ["./Build", "distmeta"], fixers) + try: + run_with_build_fixers(session, ["./Build", "distmeta"], fixers) + except UnidentifiedError as e: + if "No such action 'distmeta'" in e.lines: + pass + else: + raise try: with open(os.path.join(self.path, 'META.yml'), 'r') as f: yield from _declared_deps_from_meta_yml(f) From c20a26999a7663454ce77d02d2cbf8bf2b61dc6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 00:38:37 +0100 Subject: [PATCH 53/65] Support extra repositories flag. --- ognibuild/debian/build.py | 9 ++++++++- ognibuild/debian/fix_build.py | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index c25b3d5..9071cb1 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -175,6 +175,7 @@ def build( distribution=None, subpath="", source_date_epoch=None, + extra_repositories=None, ): args = [ sys.executable, @@ -193,6 +194,8 @@ def build( env["DISTRIBUTION"] = distribution if source_date_epoch is not None: env["SOURCE_DATE_EPOCH"] = "%d" % source_date_epoch + for repo in extra_repositories or []: + build_command += " --extra-repository=" + shlex.quote(repo) logging.info("Building debian packages, running %r.", build_command) try: subprocess.check_call( @@ -209,6 +212,7 @@ def build_once( build_command, subpath="", source_date_epoch=None, + extra_repositories=None ): build_log_path = os.path.join(output_directory, "build.log") try: @@ -221,6 +225,7 @@ def build_once( distribution=build_suite, subpath=subpath, source_date_epoch=source_date_epoch, + extra_repositories=extra_repositories, ) except BuildFailedError as e: with open(build_log_path, "rb") as f: @@ -260,7 +265,8 @@ def attempt_build( build_changelog_entry=None, subpath="", source_date_epoch=None, - run_gbp_dch=False + run_gbp_dch=False, + extra_repositories=None ): """Attempt a build, with a custom distribution set. @@ -288,4 +294,5 @@ def attempt_build( build_command, subpath, source_date_epoch=source_date_epoch, + extra_repositories=extra_repositories, ) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index c125220..1b7a926 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -523,6 +523,7 @@ def build_incrementally( subpath="", source_date_epoch=None, update_changelog=True, + extra_repositories=None, fixers=None ): fixed_errors = [] @@ -542,7 +543,8 @@ def build_incrementally( build_changelog_entry, subpath=subpath, source_date_epoch=source_date_epoch, - run_gbp_dch=(update_changelog is False) + run_gbp_dch=(update_changelog is False), + extra_repositories=extra_repositories, ) except UnidentifiedDebianBuildError: logging.warning("Build failed with unidentified error. Giving up.") From 8310d973ad4584f0b75b53a785029eab13613a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 05:09:47 +0100 Subject: [PATCH 54/65] Fix argument order. --- ognibuild/debian/build.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 9071cb1..427b8b7 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -177,6 +177,8 @@ def build( source_date_epoch=None, extra_repositories=None, ): + for repo in extra_repositories or []: + build_command += " --extra-repository=" + shlex.quote(repo) args = [ sys.executable, "-m", @@ -194,8 +196,6 @@ def build( env["DISTRIBUTION"] = distribution if source_date_epoch is not None: env["SOURCE_DATE_EPOCH"] = "%d" % source_date_epoch - for repo in extra_repositories or []: - build_command += " --extra-repository=" + shlex.quote(repo) logging.info("Building debian packages, running %r.", build_command) try: subprocess.check_call( @@ -214,7 +214,8 @@ def build_once( source_date_epoch=None, extra_repositories=None ): - build_log_path = os.path.join(output_directory, "build.log") + pkg, version = get_latest_changelog_version(local_tree, subpath) + build_log_path = os.path.join(output_directory, pkg + ".build.log") try: with open(build_log_path, "w") as f: build( From c69823be6dd2c50b53016ebccbc4c9ad2461e070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 15:36:31 +0100 Subject: [PATCH 55/65] Crate support. --- ognibuild/buildlog.py | 9 +++++++++ ognibuild/debian/build.py | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index 3805b58..cf487d3 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -63,7 +63,9 @@ from buildlog_consultant.common import ( MissingX11, MissingPerlPredeclared, MissingLatexFile, + MissingCargoCrate, ) +from buildlog_consultant.apt import UnsatisfiedDependencies from .fix_build import BuildFixer from .requirements import ( @@ -105,6 +107,7 @@ from .requirements import ( IntrospectionTypelibRequirement, PerlPreDeclaredRequirement, LatexPackageRequirement, + CargoCrateRequirement, ) from .resolver import UnsatisfiedRequirements @@ -183,6 +186,9 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return CertificateAuthorityRequirement(problem.url) elif isinstance(problem, MissingPerlPredeclared): return PerlPreDeclaredRequirement(problem.name) + elif isinstance(problem, MissingCargoCrate): + # TODO(jelmer): handle problem.requirements + return CargoCrateRequirement(problem.name) elif isinstance(problem, MissingSetupPyCommand): if problem.command == "test": return PythonPackageRequirement("setuptools") @@ -221,6 +227,9 @@ def problem_to_upstream_requirement(problem): # noqa: C901 python_version=problem.python_version, minimum_version=problem.minimum_version, ) + elif isinstance(problem, UnsatisfiedDependencies): + from .resolver.apt import AptRequirement + return AptRequirement(problem.relations) else: return None diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index 427b8b7..c74debc 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -214,8 +214,7 @@ def build_once( source_date_epoch=None, extra_repositories=None ): - pkg, version = get_latest_changelog_version(local_tree, subpath) - build_log_path = os.path.join(output_directory, pkg + ".build.log") + build_log_path = os.path.join(output_directory, "build.log") try: with open(build_log_path, "w") as f: build( From f08ebc950b024e6f1edd4d18ad52147793c7b7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 16:23:13 +0100 Subject: [PATCH 56/65] Update to newer buildlog-consultant. --- ognibuild/buildlog.py | 4 ++-- ognibuild/debian/build.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index cf487d3..f0a2fba 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -65,7 +65,7 @@ from buildlog_consultant.common import ( MissingLatexFile, MissingCargoCrate, ) -from buildlog_consultant.apt import UnsatisfiedDependencies +from buildlog_consultant.apt import UnsatisfiedAptDependencies from .fix_build import BuildFixer from .requirements import ( @@ -227,7 +227,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 python_version=problem.python_version, minimum_version=problem.minimum_version, ) - elif isinstance(problem, UnsatisfiedDependencies): + elif isinstance(problem, UnsatisfiedAptDependencies): from .resolver.apt import AptRequirement return AptRequirement(problem.relations) else: diff --git a/ognibuild/debian/build.py b/ognibuild/debian/build.py index c74debc..189b0ae 100644 --- a/ognibuild/debian/build.py +++ b/ognibuild/debian/build.py @@ -215,6 +215,7 @@ def build_once( extra_repositories=None ): build_log_path = os.path.join(output_directory, "build.log") + logging.debug("Writing build log to %s", build_log_path) try: with open(build_log_path, "w") as f: build( From 3f27ddb97c265f0ff5478d2eece39679f6a5990a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 18:17:31 +0100 Subject: [PATCH 57/65] Add AptRequirement.satisfied_by. --- ognibuild/resolver/apt.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ognibuild/resolver/apt.py b/ognibuild/resolver/apt.py index 65def68..cc2177b 100644 --- a/ognibuild/resolver/apt.py +++ b/ognibuild/resolver/apt.py @@ -117,6 +117,26 @@ class AptRequirement(Requirement): return True return False + def satisfied_by(self, binaries, version): + def binary_pkg_matches(entry, binary): + # TODO(jelmer): check versions + if entry['name'] == binary['Package']: + return True + for provides_top in PkgRelation.parse_relations( + binary.get('Provides', '')): + for provides in provides_top: + if entry['name'] == provides['name']: + return True + return False + + for rel in self.relations: + for entry in rel: + if any(binary_pkg_matches(entry, binary) for binary in binaries): + break + else: + return False + return True + def resolve_perl_predeclared_req(apt_mgr, req): try: From 3daeee51b50c5b99291ad2fb53bcf9592cde0272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Sat, 3 Apr 2021 18:38:41 +0100 Subject: [PATCH 58/65] Add debcargo unacceptable predicate fixer. --- ognibuild/debian/fix_build.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index 1b7a926..faadc30 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -111,6 +111,9 @@ from buildlog_consultant.common import ( NeedPgBuildExtUpdateControl, MissingPerlFile, ) +from buildlog_consultant.sbuild import ( + DebcargoUnacceptablePredicate, + ) from .build import ( DetailedDebianBuildFailure, @@ -427,6 +430,13 @@ def fix_missing_makefile_pl(error, phase, context): return False +def coerce_unaccpetable_predicate(error, phase, context): + from debmutate.debcargo import DebcargoEditor + with DebcargoEditor(context.abspath('debian/debcargo.toml')) as editor: + editor['allow_prerelease_deps'] = True + return context.commit('Enable allow_prerelease_deps.') + + class SimpleBuildFixer(BuildFixer): def __init__(self, packaging_context, problem_cls: Type[Problem], fn): self.context = packaging_context @@ -479,6 +489,7 @@ def versioned_package_fixers(session, packaging_context, apt): packaging_context, MissingConfigStatusInput, fix_missing_config_status_input ), SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl), + SimpleBuildFixer(packaging_context, DebcargoUnacceptablePredicate, coerce_unaccpetable_predicate), ] From d282a125b82fa77b147da51c3b1bc379270dad93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 5 Apr 2021 21:26:22 +0100 Subject: [PATCH 59/65] Return at least one tarball when there are multiple. --- ognibuild/dist_catcher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ognibuild/dist_catcher.py b/ognibuild/dist_catcher.py index 2739773..b2546a1 100644 --- a/ognibuild/dist_catcher.py +++ b/ognibuild/dist_catcher.py @@ -94,7 +94,8 @@ class DistCatcher(object): logging.warning( "Found multiple tarballs %r in %s.", possible_new, directory ) - return + self.files.extend([entry.path for entry in possible_new]) + return possible_new[0].name if len(possible_updated) == 1: entry = possible_updated[0] From b2b2244926f05f3bdf45372672eb2c78b2b824cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 5 Apr 2021 21:26:40 +0100 Subject: [PATCH 60/65] Add commit reporter. --- ognibuild/debian/fix_build.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ognibuild/debian/fix_build.py b/ognibuild/debian/fix_build.py index faadc30..2b998df 100644 --- a/ognibuild/debian/fix_build.py +++ b/ognibuild/debian/fix_build.py @@ -32,7 +32,7 @@ from debian.deb822 import ( PkgRelation, ) -from breezy.commit import PointlessCommit +from breezy.commit import PointlessCommit, NullCommitReporter from breezy.tree import Tree from debmutate.changelog import ChangelogEditor from debmutate.control import ( @@ -159,7 +159,10 @@ class DebianPackagingContext(object): cl_path = self.abspath("debian/changelog") with ChangelogEditor(cl_path) as editor: editor.add_entry([summary]) - debcommit(self.tree, committer=self.committer, subpath=self.subpath) + debcommit( + self.tree, committer=self.committer, + subpath=self.subpath, + reporter=self.commit_reporter) else: self.tree.commit( message=summary, @@ -514,7 +517,8 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]: def default_fixers(local_tree, subpath, apt, committer=None, update_changelog=None): packaging_context = DebianPackagingContext( - local_tree, subpath, committer, update_changelog + local_tree, subpath, committer, update_changelog, + commit_reporter=NullCommitReporter() ) return versioned_package_fixers(apt.session, packaging_context, apt) + apt_fixers( apt, packaging_context @@ -710,7 +714,7 @@ def main(argv=None): logging.info( 'Built %s - changes file at %r.', - os.path.join(output_directory, changes_filenames)) + cl_version, changes_filenames) if __name__ == "__main__": From 72b9fe6f5448db2d7afd8385648c8504710308d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Mon, 5 Apr 2021 21:34:57 +0100 Subject: [PATCH 61/65] Check pear namespace. --- ognibuild/buildsystem.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 76604e8..bc05873 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -118,6 +118,11 @@ class Pear(BuildSystem): name = "pear" + PEAR_NAMESPACES = [ + "http://pear.php.net/dtd/package-2.0", + "http://pear.php.net/dtd/package-2.1", + ] + def __init__(self, path): self.path = path @@ -146,10 +151,7 @@ class Pear(BuildSystem): try: root = xmlparse_simplify_namespaces( path, - [ - "http://pear.php.net/dtd/package-2.0", - "http://pear.php.net/dtd/package-2.1", - ], + self.PEAR_NAMESPACES ) except ET.ParseError as e: logging.warning("Unable to parse package.xml: %s", e) @@ -173,9 +175,20 @@ class Pear(BuildSystem): @classmethod def probe(cls, path): - if os.path.exists(os.path.join(path, "package.xml")): - logging.debug("Found package.xml, assuming pear package.") - return cls(path) + if not os.path.exists(os.path.join(path, "package.xml")): + return + + import xml.etree.ElementTree as ET + try: + tree = ET.iterparse(path) + except ET.ParseError as e: + logging.warning("Unable to parse package.xml: %s", e) + return + + for ns in cls.PEAR_NAMESPACES: + if tree.root.tag == '{%s}package' % ns: + logging.debug("Found package.xml with namespace %s, assuming pear package.") + return cls(path) # run_setup, but setting __name__ From ac553e1ae9359deaddf9c9cc948cd0f03ff8ea70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 6 Apr 2021 00:13:57 +0100 Subject: [PATCH 62/65] Open package.xml, not the directory. --- ognibuild/buildsystem.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index bc05873..5c54eab 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -175,12 +175,13 @@ class Pear(BuildSystem): @classmethod def probe(cls, path): - if not os.path.exists(os.path.join(path, "package.xml")): + package_xml_path = os.path.join(path, "package.xml") + if not os.path.exists(package_xml_path): return import xml.etree.ElementTree as ET try: - tree = ET.iterparse(path) + tree = ET.iterparse(package_xml_path) except ET.ParseError as e: logging.warning("Unable to parse package.xml: %s", e) return From 0f08d14d81598e88631c98909ed0dcce14cdcbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 6 Apr 2021 12:52:29 +0100 Subject: [PATCH 63/65] Cope with missing root. --- ognibuild/buildsystem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index 5c54eab..07562df 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -186,6 +186,10 @@ class Pear(BuildSystem): logging.warning("Unable to parse package.xml: %s", e) return + if not tree.root: + # No root? + return + for ns in cls.PEAR_NAMESPACES: if tree.root.tag == '{%s}package' % ns: logging.debug("Found package.xml with namespace %s, assuming pear package.") From 42e5e4faa76032f0d3738bc4483415580ee36985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 6 Apr 2021 17:09:22 +0100 Subject: [PATCH 64/65] add some reprs. --- ognibuild/buildlog.py | 2 +- ognibuild/debian/apt.py | 11 +++++++++-- ognibuild/requirements.py | 15 +++++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/ognibuild/buildlog.py b/ognibuild/buildlog.py index f0a2fba..e3110ee 100644 --- a/ognibuild/buildlog.py +++ b/ognibuild/buildlog.py @@ -188,7 +188,7 @@ def problem_to_upstream_requirement(problem): # noqa: C901 return PerlPreDeclaredRequirement(problem.name) elif isinstance(problem, MissingCargoCrate): # TODO(jelmer): handle problem.requirements - return CargoCrateRequirement(problem.name) + return CargoCrateRequirement(problem.crate) elif isinstance(problem, MissingSetupPyCommand): if problem.command == "test": return PythonPackageRequirement("setuptools") diff --git a/ognibuild/debian/apt.py b/ognibuild/debian/apt.py index c7926ae..1e704b0 100644 --- a/ognibuild/debian/apt.py +++ b/ognibuild/debian/apt.py @@ -82,12 +82,19 @@ class AptManager(object): ] return self._searchers - def package_exists(self, package): + @property + def apt_cache(self): if self._apt_cache is None: import apt self._apt_cache = apt.Cache(rootdir=self.session.location) - return package in self._apt_cache + return self._apt_cache + + def package_exists(self, package): + return package in self.apt_cache + + def package_versions(self, package): + return list(self.apt_cache[package].versions) def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): logging.debug("Searching for packages containing %r", paths) diff --git a/ognibuild/requirements.py b/ognibuild/requirements.py index 8b74c69..858ad09 100644 --- a/ognibuild/requirements.py +++ b/ognibuild/requirements.py @@ -281,6 +281,10 @@ class PkgConfigRequirement(Requirement): self.module = module self.minimum_version = minimum_version + def __repr__(self): + return "%s(%r, minimum_version=%r)" % ( + type(self).__name__, self.module, self.minimum_version) + class PathRequirement(Requirement): @@ -290,6 +294,9 @@ class PathRequirement(Requirement): super(PathRequirement, self).__init__("path") self.path = path + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.path) + class CHeaderRequirement(Requirement): @@ -299,6 +306,9 @@ class CHeaderRequirement(Requirement): super(CHeaderRequirement, self).__init__("c-header") self.header = header + def __repr__(self): + return "%s(%r)" % (type(self).__name__, self.header) + class JavaScriptRuntimeRequirement(Requirement): def __init__(self): @@ -659,3 +669,8 @@ class PythonModuleRequirement(Requirement): ) p.communicate() return p.returncode == 0 + + def __repr__(self): + return "%s(%r, python_version=%r, minimum_version=%r)" % ( + type(self).__name__, self.module, self.python_version, + self.minimum_version) From dc9909553aacff024eed3dc656a8346e56f5d879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelmer=20Vernoo=C4=B3?= Date: Tue, 6 Apr 2021 23:47:02 +0100 Subject: [PATCH 65/65] Release 0.0.4. --- ognibuild/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ognibuild/__init__.py b/ognibuild/__init__.py index ab42dbb..02b30ed 100644 --- a/ognibuild/__init__.py +++ b/ognibuild/__init__.py @@ -20,7 +20,7 @@ import os import stat -__version__ = (0, 0, 3) +__version__ = (0, 0, 4) USER_AGENT = "Ognibuild" diff --git a/setup.py b/setup.py index 7382e99..86ecf2e 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools import setup setup(name="ognibuild", description="Detect and run any build system", - version="0.0.3", + version="0.0.4", maintainer="Jelmer Vernooij", maintainer_email="jelmer@jelmer.uk", license="GNU GPLv2 or later",