billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support passing in tie breaking functions to apt resolver.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-03-23 15:29:31 +00:00
parent 94e0b4f99d
commit 0b6cc8d8cc
No known key found for this signature in database
GPG key ID: 579C160D4C9E23E8
5 changed files with 154 additions and 290 deletions
Download patch file Download diff file Expand all files Collapse all files

6
ognibuild/debian/apt.py
View file

@ -26,7 +26,7 @@ from buildlog_consultant.apt import (
from .. import DetailedFailure, UnidentifiedError from .. import DetailedFailure, UnidentifiedError
from ..session import Session, run_with_tee, get_user from ..session import Session, run_with_tee, get_user
from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_package_for_paths from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_packages_for_paths
def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None: def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None:
@ -81,10 +81,10 @@ class AptManager(object):
self._apt_cache = apt.Cache(rootdir=self.session.location) self._apt_cache = apt.Cache(rootdir=self.session.location)
return package in self._apt_cache return package in self._apt_cache
def get_package_for_paths(self, paths, regex=False): def get_packages_for_paths(self, paths, regex=False):
logging.debug("Searching for packages containing %r", paths) logging.debug("Searching for packages containing %r", paths)
# TODO(jelmer): Make sure we use whatever is configured in self.session # TODO(jelmer): Make sure we use whatever is configured in self.session
return get_package_for_paths(paths, self.searchers(), regex=regex) return get_packages_for_paths(paths, self.searchers(), regex=regex)
def missing(self, packages): def missing(self, packages):
root = getattr(self.session, "location", "/") root = getattr(self.session, "location", "/")

39
ognibuild/debian/file_search.py
View file

@ -264,40 +264,14 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher(
) )
def get_package_for_paths( def get_packages_for_paths(
paths: List[str], searchers: List[FileSearcher], regex: bool = False paths: List[str], searchers: List[FileSearcher], regex: bool = False
) -> Optional[str]: ) -> List[str]:
candidates: Set[str] = set() candidates: List[str] = list()
for path in paths: for path in paths:
for searcher in searchers: for searcher in searchers:
candidates.update(searcher.search_files(path, regex=regex)) candidates.extend(searcher.search_files(path, regex=regex))
if candidates: return candidates
break
if len(candidates) == 0:
logging.debug("No packages found that contain %r", paths)
return None
if len(candidates) > 1:
logging.warning(
"More than 1 packages found that contain %r: %r", path, candidates
)
# TODO(jelmer): Pick package based on what appears most commonly in
# build-depends{-indep,-arch}
try:
from .udd import UDD
except ModuleNotFoundError:
logging.warning('Unable to import UDD, not ranking by popcon')
return sorted(candidates, key=len)[0]
udd = UDD()
udd.connect()
winner = udd.get_most_popular(candidates)
if winner is None:
logging.warning(
'No relevant popcon information found, not ranking by popcon')
return sorted(candidates, key=len)[0]
logging.info('Picked winner using popcon')
return winner
else:
return candidates.pop()
def main(argv): def main(argv):
@ -317,7 +291,8 @@ def main(argv):
main_searcher.load_local() main_searcher.load_local()
searchers = [main_searcher, GENERATED_FILE_SEARCHER] searchers = [main_searcher, GENERATED_FILE_SEARCHER]
package = get_package_for_paths(args.path, searchers=searchers, regex=args.regex) packages = get_packages_for_paths(args.path, searchers=searchers, regex=args.regex)
for package in packages:
print(package) print(package)

173
ognibuild/debian/fix_build.py
View file

@ -20,6 +20,7 @@ __all__ = [
] ]
from datetime import datetime from datetime import datetime
from functools import partial
import logging import logging
import os import os
import shutil import shutil
@ -131,11 +132,12 @@ class CircularDependency(Exception):
class DebianPackagingContext(object): class DebianPackagingContext(object):
def __init__(self, tree, subpath, committer, update_changelog): def __init__(self, tree, subpath, committer, update_changelog, commit_reporter=None):
self.tree = tree self.tree = tree
self.subpath = subpath self.subpath = subpath
self.committer = committer self.committer = committer
self.update_changelog = update_changelog self.update_changelog = update_changelog
self.commit_reporter = commit_reporter
def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool: def commit(self, summary: str, update_changelog: Optional[bool] = None) -> bool:
if update_changelog is None: if update_changelog is None:
@ -149,7 +151,8 @@ class DebianPackagingContext(object):
debcommit(self.tree, committer=self.committer, subpath=self.subpath) debcommit(self.tree, committer=self.committer, subpath=self.subpath)
else: else:
self.tree.commit( self.tree.commit(
message=summary, committer=self.committer, specific_files=[self.subpath] message=summary, committer=self.committer, specific_files=[self.subpath],
reporter=self.commit_reporter
) )
except PointlessCommit: except PointlessCommit:
return False return False
@ -268,134 +271,36 @@ def add_test_dependency(context, testname, requirement):
) )
def targeted_python_versions(tree: Tree, subpath: str) -> Set[str]: def targeted_python_versions(tree: Tree, subpath: str) -> List[str]:
with tree.get_file(os.path.join(subpath, "debian/control")) as f: with tree.get_file(os.path.join(subpath, "debian/control")) as f:
control = Deb822(f) control = Deb822(f)
build_depends = PkgRelation.parse_relations(control.get("Build-Depends", "")) build_depends = PkgRelation.parse_relations(control.get("Build-Depends", ""))
all_build_deps: Set[str] = set() all_build_deps: Set[str] = set()
for or_deps in build_depends: for or_deps in build_depends:
all_build_deps.update(or_dep["name"] for or_dep in or_deps) all_build_deps.update(or_dep["name"] for or_dep in or_deps)
targeted = set() targeted = []
if any(x.startswith("pypy") for x in all_build_deps):
targeted.add("pypy")
if any(x.startswith("python-") for x in all_build_deps):
targeted.add("cpython2")
if any(x.startswith("python3-") for x in all_build_deps): if any(x.startswith("python3-") for x in all_build_deps):
targeted.add("cpython3") targeted.append("python3")
if any(x.startswith("pypy") for x in all_build_deps):
targeted.append("pypy")
if any(x.startswith("python-") for x in all_build_deps):
targeted.append("python")
return targeted return targeted
def fix_missing_python_distribution(error, phase, apt, context): # noqa: C901 def python_tie_breaker(tree, subpath, reqs):
targeted = targeted_python_versions(context.tree, context.subpath) targeted = targeted_python_versions(tree, subpath)
default = not targeted if not targeted:
return None
pypy_pkg = apt.get_package_for_paths( for prefix in targeted:
["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True for req in reqs:
) if any(name.startswith(prefix + '-') for name in req.package_names()):
if pypy_pkg is None: logging.info(
pypy_pkg = "pypy-%s" % error.distribution 'Breaking tie between %r to %r, since package already '
if not apt.package_exists(pypy_pkg): 'has %r build-dependencies', [str(req) for req in reqs],
pypy_pkg = None str(req), prefix)
return req
py2_pkg = apt.get_package_for_paths( return None
["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution],
regex=True,
)
if py2_pkg is None:
py2_pkg = "python-%s" % error.distribution
if not apt.package_exists(py2_pkg):
py2_pkg = None
py3_pkg = apt.get_package_for_paths(
["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution],
regex=True,
)
if py3_pkg is None:
py3_pkg = "python3-%s" % error.distribution
if not apt.package_exists(py3_pkg):
py3_pkg = None
extra_build_deps = []
if error.python_version == 2:
if "pypy" in targeted:
if not pypy_pkg:
logging.warning("no pypy package found for %s", error.module)
else:
extra_build_deps.append(pypy_pkg)
if "cpython2" in targeted or default:
if not py2_pkg:
logging.warning("no python 2 package found for %s", error.module)
return False
extra_build_deps.append(py2_pkg)
elif error.python_version == 3:
if not py3_pkg:
logging.warning("no python 3 package found for %s", error.module)
return False
extra_build_deps.append(py3_pkg)
else:
if py3_pkg and ("cpython3" in targeted or default):
extra_build_deps.append(py3_pkg)
if py2_pkg and ("cpython2" in targeted or default):
extra_build_deps.append(py2_pkg)
if pypy_pkg and "pypy" in targeted:
extra_build_deps.append(pypy_pkg)
if not extra_build_deps:
return False
for dep_pkg in extra_build_deps:
assert dep_pkg is not None
if not add_dependency(context, phase, dep_pkg):
return False
return True
def fix_missing_python_module(error, phase, apt, context):
targeted = targeted_python_versions(context.tree, context.subpath)
default = not targeted
if error.minimum_version:
specs = [(">=", error.minimum_version)]
else:
specs = []
pypy_pkg = get_package_for_python_module(apt, error.module, "pypy", specs)
py2_pkg = get_package_for_python_module(apt, error.module, "cpython2", specs)
py3_pkg = get_package_for_python_module(apt, error.module, "cpython3", specs)
extra_build_deps = []
if error.python_version == 2:
if "pypy" in targeted:
if not pypy_pkg:
logging.warning("no pypy package found for %s", error.module)
else:
extra_build_deps.append(pypy_pkg)
if "cpython2" in targeted or default:
if not py2_pkg:
logging.warning("no python 2 package found for %s", error.module)
return False
extra_build_deps.append(py2_pkg)
elif error.python_version == 3:
if not py3_pkg:
logging.warning("no python 3 package found for %s", error.module)
return False
extra_build_deps.append(py3_pkg)
else:
if py3_pkg and ("cpython3" in targeted or default):
extra_build_deps.append(py3_pkg)
if py2_pkg and ("cpython2" in targeted or default):
extra_build_deps.append(py2_pkg)
if pypy_pkg and "pypy" in targeted:
extra_build_deps.append(pypy_pkg)
if not extra_build_deps:
return False
for dep_pkg in extra_build_deps:
assert dep_pkg is not None
if not add_dependency(context, phase, dep_pkg):
return False
return True
def retry_apt_failure(error, phase, apt, context): def retry_apt_failure(error, phase, apt, context):
@ -536,12 +441,34 @@ def versioned_package_fixers(session, packaging_context):
] ]
def udd_tie_breaker(candidates):
# TODO(jelmer): Pick package based on what appears most commonly in
# build-depends{-indep,-arch}
try:
from .udd import UDD
except ModuleNotFoundError:
logging.warning('Unable to import UDD, not ranking by popcon')
return sorted(candidates, key=len)[0]
udd = UDD()
udd.connect()
names = {list(c.package_names())[0]: c for c in candidates}
winner = udd.get_most_popular(list(names.keys()))
if winner is None:
logging.warning(
'No relevant popcon information found, not ranking by popcon')
return None
logging.info('Picked winner using popcon')
return names[winner]
def apt_fixers(apt, packaging_context) -> List[BuildFixer]: def apt_fixers(apt, packaging_context) -> List[BuildFixer]:
from ..resolver.apt import AptResolver from ..resolver.apt import AptResolver
resolver = AptResolver(apt) apt_tie_breakers = [
partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath),
udd_tie_breaker,
]
resolver = AptResolver(apt, apt_tie_breakers)
return [ return [
DependencyBuildFixer(packaging_context, apt, MissingPythonModule, fix_missing_python_module),
DependencyBuildFixer(packaging_context, apt, MissingPythonDistribution, fix_missing_python_distribution),
DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure), DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure),
PackageDependencyFixer(packaging_context, resolver), PackageDependencyFixer(packaging_context, resolver),
] ]

216
ognibuild/resolver/apt.py
View file

@ -20,6 +20,7 @@ import logging
import os import os
import posixpath import posixpath
import re import re
from typing import Optional, List
from debian.changelog import Version from debian.changelog import Version
from debian.deb822 import PkgRelation from debian.deb822 import PkgRelation
@ -84,14 +85,30 @@ class AptRequirement(Requirement):
def __str__(self): def __str__(self):
return "apt requirement: %s" % self.pkg_relation_str() return "apt requirement: %s" % self.pkg_relation_str()
def touches_package(self, package): def package_names(self):
for rel in self.relations: for rel in self.relations:
for entry in rel: for entry in rel:
if entry["name"] == package: yield entry["name"]
def touches_package(self, package):
for name in self.package_names():
if name == package:
return True return True
return False return False
def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False) -> List[str]:
if not isinstance(paths, list):
raise TypeError(paths)
return apt_mgr.get_packages_for_paths(paths, regex)
def find_reqs_simple(apt_mgr: AptManager, paths: List[str], regex: bool = False, minimum_version=None) -> List[str]:
if not isinstance(paths, list):
raise TypeError(paths)
return [AptRequirement.simple(package, minimum_version=minimum_version) for package in find_package_names(apt_mgr, paths, regex)]
def python_spec_to_apt_rels(pkg_name, specs): def python_spec_to_apt_rels(pkg_name, specs):
# TODO(jelmer): Dealing with epoch, etc? # TODO(jelmer): Dealing with epoch, etc?
if not specs: if not specs:
@ -133,11 +150,8 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st
paths = [cpython3_regex, cpython2_regex, pypy_regex] paths = [cpython3_regex, cpython2_regex, pypy_regex]
else: else:
raise NotImplementedError('unsupported python version %d' % python_version) raise NotImplementedError('unsupported python version %d' % python_version)
pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) names = find_package_names(apt_mgr, paths, regex=True)
if pkg_name is None: return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names]
return None
rels = python_spec_to_apt_rels(pkg_name, specs)
return AptRequirement(rels)
def get_package_for_python_module(apt_mgr, module, python_version, specs): def get_package_for_python_module(apt_mgr, module, python_version, specs):
@ -198,11 +212,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
paths = cpython3_regexes + cpython2_regexes + pypy_regexes paths = cpython3_regexes + cpython2_regexes + pypy_regexes
else: else:
raise AssertionError("unknown python version %r" % python_version) raise AssertionError("unknown python version %r" % python_version)
pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) names = find_package_names(apt_mgr, paths, regex=True)
if pkg_name is None: return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names]
return None
rels = python_spec_to_apt_rels(pkg_name, specs)
return AptRequirement(rels)
def resolve_binary_req(apt_mgr, req): def resolve_binary_req(apt_mgr, req):
@ -212,61 +223,42 @@ def resolve_binary_req(apt_mgr, req):
paths = [ paths = [
posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"]
] ]
pkg_name = apt_mgr.get_package_for_paths(paths) return find_reqs_simple(apt_mgr, paths)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_pkg_config_req(apt_mgr, req): def resolve_pkg_config_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths( names = find_package_names(apt_mgr,
[posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")],
)
if package is None:
package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], [posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")],
regex=True, regex=True)
) if not names:
if package is not None: names = find_package_names(
return AptRequirement.simple(package, minimum_version=req.minimum_version) apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")])
return None return [AptRequirement.simple(name, minimum_version=req.minimum_version) for name in names]
def resolve_path_req(apt_mgr, req): def resolve_path_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths([req.path]) return find_reqs_simple(apt_mgr, [req.path])
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_c_header_req(apt_mgr, req): def resolve_c_header_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths( reqs = find_reqs_simple(
[posixpath.join("/usr/include", req.header)], regex=False apt_mgr,
) [posixpath.join("/usr/include", req.header)], regex=False)
if package is None: if not reqs:
package = apt_mgr.get_package_for_paths( reqs = find_package_names(
apt_mgr,
[posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True
) )
if package is None: return reqs
return None
return AptRequirement.simple(package)
def resolve_js_runtime_req(apt_mgr, req): def resolve_js_runtime_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths( return find_reqs_simple(apt_mgr, ["/usr/bin/node", "/usr/bin/duk"])
["/usr/bin/node", "/usr/bin/duk"], regex=False
)
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_vala_package_req(apt_mgr, req): def resolve_vala_package_req(apt_mgr, req):
path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package) path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package)
package = apt_mgr.get_package_for_paths([path], regex=True) return find_reqs_simple(apt_mgr, [path], regex=True)
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_ruby_gem_req(apt_mgr, req): def resolve_ruby_gem_req(apt_mgr, req):
@ -276,43 +268,29 @@ def resolve_ruby_gem_req(apt_mgr, req):
"specifications/%s-.*\\.gemspec" % re.escape(req.gem) "specifications/%s-.*\\.gemspec" % re.escape(req.gem)
) )
] ]
package = apt_mgr.get_package_for_paths(paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version)
if package is not None:
return AptRequirement.simple(package, minimum_version=req.minimum_version)
return None
def resolve_go_package_req(apt_mgr, req): def resolve_go_package_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths( return find_reqs_simple(
apt_mgr,
[posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True
) )
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_dh_addon_req(apt_mgr, req): def resolve_dh_addon_req(apt_mgr, req):
paths = [posixpath.join("/usr/share/perl5", req.path)] paths = [posixpath.join("/usr/share/perl5", req.path)]
package = apt_mgr.get_package_for_paths(paths) return find_reqs_simple(apt_mgr, paths)
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_php_class_req(apt_mgr, req): def resolve_php_class_req(apt_mgr, req):
path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/")
package = apt_mgr.get_package_for_paths([path]) return find_reqs_simple(apt_mgr, [path])
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_r_package_req(apt_mgr, req): def resolve_r_package_req(apt_mgr, req):
paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))]
package = apt_mgr.get_package_for_paths(paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True)
if package is not None:
return AptRequirement.simple(package)
return None
def resolve_node_package_req(apt_mgr, req): def resolve_node_package_req(apt_mgr, req):
@ -321,10 +299,7 @@ def resolve_node_package_req(apt_mgr, req):
"/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package),
"/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package),
] ]
pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_library_req(apt_mgr, req): def resolve_library_req(apt_mgr, req):
@ -334,27 +309,21 @@ def resolve_library_req(apt_mgr, req):
posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)),
posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)),
] ]
pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) return find_reqs_simple(apt_mgr, paths)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_ruby_file_req(apt_mgr, req): def resolve_ruby_file_req(apt_mgr, req):
paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)]
package = apt_mgr.get_package_for_paths(paths) reqs = find_reqs_simple(apt_mgr, paths, regex=False)
if package is not None: if reqs:
return AptRequirement.simple(package) return reqs
paths = [ paths = [
posixpath.join( posixpath.join(
r"/usr/share/rubygems-integration/all/gems/([^/]+)/" r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
"lib/%s\\.rb" % re.escape(req.filename) "lib/%s\\.rb" % re.escape(req.filename)
) )
] ]
pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_xml_entity_req(apt_mgr, req): def resolve_xml_entity_req(apt_mgr, req):
@ -370,10 +339,7 @@ def resolve_xml_entity_req(apt_mgr, req):
else: else:
return None return None
pkg_name = apt_mgr.get_package_for_paths([search_path], regex=False) return find_reqs_simple(apt_mgr, [search_path], regex=False)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_sprockets_file_req(apt_mgr, req): def resolve_sprockets_file_req(apt_mgr, req):
@ -382,10 +348,7 @@ def resolve_sprockets_file_req(apt_mgr, req):
else: else:
logging.warning("unable to handle content type %s", req.content_type) logging.warning("unable to handle content type %s", req.content_type)
return None return None
pkg_name = apt_mgr.get_package_for_paths([path], regex=True) return find_reqs_simple(apt_mgr, [path], regex=True)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_java_class_req(apt_mgr, req): def resolve_java_class_req(apt_mgr, req):
@ -399,19 +362,12 @@ def resolve_java_class_req(apt_mgr, req):
logging.warning("unable to find classpath for %s", req.classname) logging.warning("unable to find classpath for %s", req.classname)
return False return False
logging.info("Classpath for %s: %r", req.classname, classpath) logging.info("Classpath for %s: %r", req.classname, classpath)
package = apt_mgr.get_package_for_paths(classpath) return find_reqs_simple(apt_mgr, [classpath])
if package is None:
logging.warning("no package for files in %r", classpath)
return None
return AptRequirement.simple(package)
def resolve_haskell_package_req(apt_mgr, req): def resolve_haskell_package_req(apt_mgr, req):
path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0])
pkg_name = apt_mgr.get_package_for_paths([path], regex=True) return find_reqs_simple(apt_mgr, [path], regex=True)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_maven_artifact_req(apt_mgr, req): def resolve_maven_artifact_req(apt_mgr, req):
@ -440,10 +396,7 @@ def resolve_maven_artifact_req(apt_mgr, req):
"%s-%s.%s" % (artifact_id, version, kind), "%s-%s.%s" % (artifact_id, version, kind),
) )
] ]
pkg_name = apt_mgr.get_package_for_paths(paths, regex=regex) return find_reqs_simple(apt_mgr, paths, regex=regex)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_gnome_common_req(apt_mgr, req): def resolve_gnome_common_req(apt_mgr, req):
@ -452,10 +405,7 @@ def resolve_gnome_common_req(apt_mgr, req):
def resolve_jdk_file_req(apt_mgr, req): def resolve_jdk_file_req(apt_mgr, req):
path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename) path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename)
pkg_name = apt_mgr.get_package_for_paths([path], regex=True) return find_reqs_simple(apt_mgr, [path], regex=True)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_jdk_req(apt_mgr, req): def resolve_jdk_req(apt_mgr, req):
@ -478,17 +428,11 @@ def resolve_perl_module_req(apt_mgr, req):
paths = [req.filename] paths = [req.filename]
else: else:
paths = [posixpath.join(inc, req.filename) for inc in req.inc] paths = [posixpath.join(inc, req.filename) for inc in req.inc]
pkg_name = apt_mgr.get_package_for_paths(paths, regex=False) return find_reqs_simple(apt_mgr, paths, regex=False)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_perl_file_req(apt_mgr, req): def resolve_perl_file_req(apt_mgr, req):
pkg_name = apt_mgr.get_package_for_paths([req.filename], regex=False) return find_reqs_simple(apt_mgr, [req.filename], regex=False)
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def _find_aclocal_fun(macro): def _find_aclocal_fun(macro):
@ -512,10 +456,7 @@ def resolve_autoconf_macro_req(apt_mgr, req):
except KeyError: except KeyError:
logging.info("No local m4 file found defining %s", req.macro) logging.info("No local m4 file found defining %s", req.macro)
return None return None
pkg_name = apt_mgr.get_package_for_paths([path]) return find_reqs_simple(apt_mgr, [path])
if pkg_name is not None:
return AptRequirement.simple(pkg_name)
return None
def resolve_python_module_req(apt_mgr, req): def resolve_python_module_req(apt_mgr, req):
@ -547,10 +488,7 @@ def resolve_python_package_req(apt_mgr, req):
def resolve_cargo_crate_req(apt_mgr, req): def resolve_cargo_crate_req(apt_mgr, req):
paths = [ paths = [
'/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)] '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)]
pkg_name = apt_mgr.get_package_for_paths(paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True)
if pkg_name is None:
return None
return AptRequirement.simple(pkg_name)
def resolve_ca_req(apt_mgr, req): def resolve_ca_req(apt_mgr, req):
@ -591,16 +529,24 @@ APT_REQUIREMENT_RESOLVERS = [
] ]
def resolve_requirement_apt(apt_mgr, req: Requirement) -> AptRequirement: def resolve_requirement_apt(apt_mgr, req: Requirement) -> List[AptRequirement]:
for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
if isinstance(req, rr_class): if isinstance(req, rr_class):
return rr_fn(apt_mgr, req) ret = rr_fn(apt_mgr, req)
if not ret:
return []
if not isinstance(ret, list):
raise TypeError(ret)
return ret
raise NotImplementedError(type(req)) raise NotImplementedError(type(req))
class AptResolver(Resolver): class AptResolver(Resolver):
def __init__(self, apt): def __init__(self, apt, tie_breakers=None):
self.apt = apt self.apt = apt
if tie_breakers is None:
tie_breakers = []
self.tie_breakers = tie_breakers
def __str__(self): def __str__(self):
return "apt" return "apt"
@ -647,4 +593,18 @@ class AptResolver(Resolver):
yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements]) yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements])
def resolve(self, req: Requirement): def resolve(self, req: Requirement):
return resolve_requirement_apt(self.apt, req) ret = resolve_requirement_apt(self.apt, req)
if not ret:
return None
if len(ret) == 1:
return ret[0]
for tie_breaker in self.tie_breakers:
winner = tie_breaker(ret)
if winner is not None:
if not isinstance(winner, AptRequirement):
raise TypeError(winner)
return winner
logging.info(
'Unable to break tie over %r, picking first: %r',
ret, ret[0])
return ret[0]

4
ognibuild/tests/test_debian_fix_build.py
View file

@ -37,6 +37,7 @@ from ..debian.fix_build import (
apt_fixers, apt_fixers,
DebianPackagingContext, DebianPackagingContext,
) )
from breezy.commit import NullCommitReporter
from breezy.tests import TestCaseWithTransport from breezy.tests import TestCaseWithTransport
@ -99,7 +100,8 @@ blah (0.1) UNRELEASED; urgency=medium
apt._searchers = [DummyAptSearcher(self._apt_files)] apt._searchers = [DummyAptSearcher(self._apt_files)]
context = DebianPackagingContext( context = DebianPackagingContext(
self.tree, subpath="", committer="ognibuild <ognibuild@jelmer.uk>", self.tree, subpath="", committer="ognibuild <ognibuild@jelmer.uk>",
update_changelog=True) update_changelog=True,
commit_reporter=NullCommitReporter())
fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context) fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context)
return resolve_error(error, ("build", ), fixers) return resolve_error(error, ("build", ), fixers)