#!/usr/bin/python3 # Copyright (C) 2020 Jelmer Vernooij # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA from itertools import chain import logging import os import posixpath import re from typing import Optional, List from debian.changelog import Version from debian.deb822 import PkgRelation from ..debian.apt import AptManager from . import Resolver, UnsatisfiedRequirements from ..requirements import ( Requirement, CargoCrateRequirement, BinaryRequirement, CHeaderRequirement, PkgConfigRequirement, PathRequirement, JavaScriptRuntimeRequirement, ValaPackageRequirement, RubyGemRequirement, GoPackageRequirement, GoRequirement, DhAddonRequirement, PhpClassRequirement, PhpPackageRequirement, RPackageRequirement, NodeModuleRequirement, NodePackageRequirement, LibraryRequirement, RubyFileRequirement, XmlEntityRequirement, SprocketsFileRequirement, JavaClassRequirement, HaskellPackageRequirement, MavenArtifactRequirement, GnomeCommonRequirement, JDKFileRequirement, JDKRequirement, JRERequirement, QTRequirement, X11Requirement, PerlModuleRequirement, PerlFileRequirement, AutoconfMacroRequirement, PythonModuleRequirement, PythonPackageRequirement, CertificateAuthorityRequirement, LibtoolRequirement, VagueDependencyRequirement, ) class AptRequirement(Requirement): def __init__(self, relations): super(AptRequirement, self).__init__("apt") if not isinstance(relations, list): raise TypeError(relations) self.relations = relations @classmethod def simple(cls, package, minimum_version=None): rel = {"name": package} if minimum_version is not None: rel["version"] = (">=", minimum_version) return cls([[rel]]) @classmethod def from_str(cls, text): return cls(PkgRelation.parse_relations(text)) def pkg_relation_str(self): return PkgRelation.str(self.relations) def __hash__(self): return hash((type(self), self.pkg_relation_str())) def __eq__(self, other): return isinstance(self, type(other)) and self.relations == other.relations def __str__(self): return "apt requirement: %s" % self.pkg_relation_str() def __repr__(self): return "%s.from_str(%r)" % (type(self).__name__, self.pkg_relation_str()) def package_names(self): for rel in self.relations: for entry in rel: yield entry["name"] def touches_package(self, package): for name in self.package_names(): if name == package: return True return False def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) def find_reqs_simple( apt_mgr: AptManager, paths: List[str], regex: bool = False, minimum_version=None, case_insensitive=False) -> List[str]: if not isinstance(paths, list): raise TypeError(paths) return [AptRequirement.simple(package, minimum_version=minimum_version) for package in find_package_names(apt_mgr, paths, regex, case_insensitive)] def python_spec_to_apt_rels(pkg_name, specs): # TODO(jelmer): Dealing with epoch, etc? if not specs: return [[{"name": pkg_name}]] else: rels = [] for spec in specs: deb_version = Version(spec[1]) if spec[0] == '~=': # PEP 440: For a given release identifier V.N , the compatible # release clause is approximately equivalent to the pair of # comparison clauses: >= V.N, == V.* parts = spec[1].split('.') parts.pop(-1) parts[-1] = str(int(parts[-1])+1) next_maj_deb_version = Version('.'.join(parts)) rels.extend([{"name": pkg_name, "version": ('>=', deb_version)}, {"name": pkg_name, "version": ('<<', next_maj_deb_version)}]) elif spec[0] == '!=': rels.extend([{"name": pkg_name, "version": ('>>', deb_version)}, {"name": pkg_name, "version": ('<<', deb_version)}]) elif spec[1].endswith('.*') and spec[0] == '==': s = spec[1].split('.') s.pop(-1) n = list(s) n[-1] = str(int(n[-1])+1) rels.extend([{"name": pkg_name, "version": ('>=', Version('.'.join(s)))}, {"name": pkg_name, "version": ('<<', Version('.'.join(n)))}]) else: c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] rels.append([{"name": pkg_name, "version": (c, deb_version)}]) return rels def get_package_for_python_package(apt_mgr, package, python_version: Optional[str], specs=None): pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) cpython2_regex = "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) if python_version == "pypy": paths = [pypy_regex] elif python_version == "cpython2": paths = [cpython2_regex] elif python_version == "cpython3": paths = [cpython3_regex] elif python_version is None: paths = [cpython3_regex, cpython2_regex, pypy_regex] else: raise NotImplementedError('unsupported python version %s' % python_version) names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True) return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] def get_package_for_python_module(apt_mgr, module, python_version, specs): cpython3_regexes = [ posixpath.join( "/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")), "__init__.py", ), posixpath.join( "/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")) + ".py" ), posixpath.join( "/usr/lib/python3\\.[0-9]+/lib-dynload", re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", ), posixpath.join( "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" ), posixpath.join( "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")), "__init__.py" ), ] cpython2_regexes = [ posixpath.join( "/usr/lib/python2\\.[0-9]/dist-packages", re.escape(module.replace(".", "/")), "__init__.py", ), posixpath.join( "/usr/lib/python2\\.[0-9]/dist-packages", re.escape(module.replace(".", "/")) + ".py", ), posixpath.join( "/usr/lib/python2.\\.[0-9]/lib-dynload", re.escape(module.replace(".", "/")) + ".so", ), ] pypy_regexes = [ posixpath.join( "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")), "__init__.py" ), posixpath.join( "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" ), posixpath.join( "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", ), ] if python_version == "cpython3": paths = cpython3_regexes elif python_version == "cpython2": paths = cpython2_regexes elif python_version == "pypy": paths = pypy_regexes elif python_version is None: paths = cpython3_regexes + cpython2_regexes + pypy_regexes else: raise AssertionError("unknown python version %r" % python_version) names = find_package_names(apt_mgr, paths, regex=True) return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] vague_map = { 'the Gnu Scientific Library': 'libgsl-dev', 'the required FreeType library': 'libfreetype-dev', } def resolve_vague_dep_req(apt_mgr, req): name = req.name options = [] if name in vague_map: options.append(AptRequirement.simple(vague_map[name])) if name.startswith('gnu '): name = name[4:] for x in req.expand(): options.extend(resolve_requirement_apt(apt_mgr, x)) return options def resolve_binary_req(apt_mgr, req): if posixpath.isabs(req.binary_name): paths = [req.binary_name] else: paths = [ posixpath.join(dirname, req.binary_name) for dirname in ["/usr/bin", "/bin"] ] return find_reqs_simple(apt_mgr, paths) def resolve_pkg_config_req(apt_mgr, req): names = find_package_names(apt_mgr, [posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], regex=True) if not names: names = find_package_names( apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")]) return [AptRequirement.simple(name, minimum_version=req.minimum_version) for name in names] def resolve_path_req(apt_mgr, req): return find_reqs_simple(apt_mgr, [req.path]) def resolve_c_header_req(apt_mgr, req): reqs = find_reqs_simple( apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False) if not reqs: reqs = find_reqs_simple( apt_mgr, [posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True ) return reqs def resolve_js_runtime_req(apt_mgr, req): return find_reqs_simple(apt_mgr, ["/usr/bin/node", "/usr/bin/duk"]) def resolve_vala_package_req(apt_mgr, req): path = "/usr/share/vala-[0-9.]+/vapi/%s\\.vapi" % re.escape(req.package) return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_ruby_gem_req(apt_mgr, req): paths = [ posixpath.join( "/usr/share/rubygems-integration/all/" "specifications/%s-.*\\.gemspec" % re.escape(req.gem) ) ] return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version) def resolve_go_package_req(apt_mgr, req): return find_reqs_simple( apt_mgr, [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True ) def resolve_go_req(apt_mgr, req): return [AptRequirement.simple('golang-go', minimum_version='2:%s' % req.version)] def resolve_dh_addon_req(apt_mgr, req): paths = [posixpath.join("/usr/share/perl5", req.path)] return find_reqs_simple(apt_mgr, paths) def resolve_php_class_req(apt_mgr, req): path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/") return find_reqs_simple(apt_mgr, [path]) def resolve_php_package_req(apt_mgr, req): return [AptRequirement.simple('php-%s' % req.package, minimum_version=req.min_version)] def resolve_r_package_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_node_module_req(apt_mgr, req): paths = [ "/usr/share/nodejs/.*/node_modules/%s/index.js" % re.escape(req.module), "/usr/lib/nodejs/%s/index.js" % re.escape(req.module), "/usr/share/nodejs/%s/index.js" % re.escape(req.module), ] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_node_package_req(apt_mgr, req): paths = [ "/usr/share/nodejs/.*/node_modules/%s/package\\.json" % re.escape(req.package), "/usr/lib/nodejs/%s/package\\.json" % re.escape(req.package), "/usr/share/nodejs/%s/package\\.json" % re.escape(req.package), ] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_library_req(apt_mgr, req): paths = [ posixpath.join("/usr/lib/lib%s.so$" % re.escape(req.library)), posixpath.join("/usr/lib/.*/lib%s.so$" % re.escape(req.library)), posixpath.join("/usr/lib/lib%s.a$" % re.escape(req.library)), posixpath.join("/usr/lib/.*/lib%s.a$" % re.escape(req.library)), ] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_ruby_file_req(apt_mgr, req): paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)] reqs = find_reqs_simple(apt_mgr, paths, regex=False) if reqs: return reqs paths = [ posixpath.join( r"/usr/share/rubygems-integration/all/gems/([^/]+)/" "lib/%s\\.rb" % re.escape(req.filename) ) ] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_xml_entity_req(apt_mgr, req): # Ideally we should be using the XML catalog for this, but hardcoding # a few URLs will do for now.. URL_MAP = { "http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/" } for url, path in URL_MAP.items(): if req.url.startswith(url): search_path = posixpath.join(path, req.url[len(url) :]) break else: return None return find_reqs_simple(apt_mgr, [search_path], regex=False) def resolve_sprockets_file_req(apt_mgr, req): if req.content_type == "application/javascript": path = "/usr/share/.*/app/assets/javascripts/%s\\.js$" % re.escape(req.name) else: logging.warning("unable to handle content type %s", req.content_type) return None return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_java_class_req(apt_mgr, req): # Unfortunately this only finds classes in jars installed on the host # system :( output = apt_mgr.session.check_output( ["java-propose-classpath", "-c" + req.classname] ) classpath = [p for p in output.decode().strip(":").strip().split(":") if p] if not classpath: logging.warning("unable to find classpath for %s", req.classname) return False logging.info("Classpath for %s: %r", req.classname, classpath) return find_reqs_simple(apt_mgr, [classpath]) def resolve_haskell_package_req(apt_mgr, req): path = "/var/lib/ghc/package\\.conf\\.d/%s-.*\\.conf" % re.escape(req.deps[0][0]) return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_maven_artifact_req(apt_mgr, req): if req.version is None: version = ".*" regex = True escape = re.escape else: version = req.version regex = False def escape(x): return x kind = req.kind or 'jar' path = posixpath.join( escape("/usr/share/maven-repo"), escape(req.group_id.replace(".", "/")), escape(req.artifact_id), version, escape("%s-") + version + escape("." + kind) ) return find_reqs_simple(apt_mgr, [path], regex=regex) def resolve_gnome_common_req(apt_mgr, req): return [AptRequirement.simple("gnome-common")] def resolve_jdk_file_req(apt_mgr, req): path = re.escape(req.jdk_path) + ".*/" + re.escape(req.filename) return find_reqs_simple(apt_mgr, [path], regex=True) def resolve_jdk_req(apt_mgr, req): return [AptRequirement.simple('default-jdk')] def resolve_jre_req(apt_mgr, req): return [AptRequirement.simple('default-jre')] def resolve_x11_req(apt_mgr, req): return [AptRequirement.simple('libx11-dev')] def resolve_qt_req(apt_mgr, req): return find_reqs_simple(apt_mgr, ["/usr/lib/.*/qt[0-9]+/bin/qmake"], regex=True) def resolve_libtool_req(apt_mgr, req): return [AptRequirement.simple("libtool")] def resolve_perl_module_req(apt_mgr, req): DEFAULT_PERL_PATHS = ["/usr/share/perl5"] if req.inc is None: if req.filename is None: paths = [posixpath.join(inc, req.relfilename) for inc in DEFAULT_PERL_PATHS] elif not posixpath.isabs(req.filename): return False else: paths = [req.filename] else: paths = [posixpath.join(inc, req.filename) for inc in req.inc] return find_reqs_simple(apt_mgr, paths, regex=False) def resolve_perl_file_req(apt_mgr, req): return find_reqs_simple(apt_mgr, [req.filename], regex=False) def _find_aclocal_fun(macro): # TODO(jelmer): Use the API for codesearch.debian.net instead? defun_prefix = b"AC_DEFUN([%s]," % macro.encode("ascii") au_alias_prefix = b"AU_ALIAS([%s]," % macro.encode("ascii") prefixes = [defun_prefix, au_alias_prefix] for entry in os.scandir("/usr/share/aclocal"): if not entry.is_file(): continue with open(entry.path, "rb") as f: for line in f: if any([line.startswith(prefix) for prefix in prefixes]): return entry.path raise KeyError def resolve_autoconf_macro_req(apt_mgr, req): try: path = _find_aclocal_fun(req.macro) except KeyError: logging.info("No local m4 file found defining %s", req.macro) return None return find_reqs_simple(apt_mgr, [path]) def resolve_python_module_req(apt_mgr, req): if req.minimum_version: specs = [(">=", req.minimum_version)] else: specs = [] if req.python_version == 2: return get_package_for_python_module(apt_mgr, req.module, "cpython2", specs) elif req.python_version in (None, 3): return get_package_for_python_module(apt_mgr, req.module, "cpython3", specs) else: return None def resolve_python_package_req(apt_mgr, req): if req.python_version == 2: return get_package_for_python_package( apt_mgr, req.package, "cpython2", req.specs ) elif req.python_version in (None, 3): return get_package_for_python_package( apt_mgr, req.package, "cpython3", req.specs ) else: return None def resolve_cargo_crate_req(apt_mgr, req): paths = [ '/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)] return find_reqs_simple(apt_mgr, paths, regex=True) def resolve_ca_req(apt_mgr, req): return [AptRequirement.simple('ca-certificates')] def resolve_apt_req(apt_mgr, req): return [req] APT_REQUIREMENT_RESOLVERS = [ (AptRequirement, resolve_apt_req), (BinaryRequirement, resolve_binary_req), (VagueDependencyRequirement, resolve_vague_dep_req), (PkgConfigRequirement, resolve_pkg_config_req), (PathRequirement, resolve_path_req), (CHeaderRequirement, resolve_c_header_req), (JavaScriptRuntimeRequirement, resolve_js_runtime_req), (ValaPackageRequirement, resolve_vala_package_req), (RubyGemRequirement, resolve_ruby_gem_req), (GoPackageRequirement, resolve_go_package_req), (GoRequirement, resolve_go_req), (DhAddonRequirement, resolve_dh_addon_req), (PhpClassRequirement, resolve_php_class_req), (PhpPackageRequirement, resolve_php_package_req), (RPackageRequirement, resolve_r_package_req), (NodeModuleRequirement, resolve_node_module_req), (NodePackageRequirement, resolve_node_package_req), (LibraryRequirement, resolve_library_req), (RubyFileRequirement, resolve_ruby_file_req), (XmlEntityRequirement, resolve_xml_entity_req), (SprocketsFileRequirement, resolve_sprockets_file_req), (JavaClassRequirement, resolve_java_class_req), (HaskellPackageRequirement, resolve_haskell_package_req), (MavenArtifactRequirement, resolve_maven_artifact_req), (GnomeCommonRequirement, resolve_gnome_common_req), (JDKFileRequirement, resolve_jdk_file_req), (JDKRequirement, resolve_jdk_req), (JRERequirement, resolve_jre_req), (QTRequirement, resolve_qt_req), (X11Requirement, resolve_x11_req), (LibtoolRequirement, resolve_libtool_req), (PerlModuleRequirement, resolve_perl_module_req), (PerlFileRequirement, resolve_perl_file_req), (AutoconfMacroRequirement, resolve_autoconf_macro_req), (PythonModuleRequirement, resolve_python_module_req), (PythonPackageRequirement, resolve_python_package_req), (CertificateAuthorityRequirement, resolve_ca_req), (CargoCrateRequirement, resolve_cargo_crate_req), ] def resolve_requirement_apt(apt_mgr, req: Requirement) -> List[AptRequirement]: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: if isinstance(req, rr_class): ret = rr_fn(apt_mgr, req) if not ret: return [] if not isinstance(ret, list): raise TypeError(ret) return ret raise NotImplementedError(type(req)) class AptResolver(Resolver): def __init__(self, apt, tie_breakers=None): self.apt = apt if tie_breakers is None: tie_breakers = [] self.tie_breakers = tie_breakers def __str__(self): return "apt" def __repr__(self): return "%s(%r, %r)" % (type(self).__name__, self.apt, self.tie_breakers) @classmethod def from_session(cls, session, tie_breakers=None): return cls(AptManager.from_session(session), tie_breakers=tie_breakers) def install(self, requirements): missing = [] for req in requirements: try: if not req.met(self.apt.session): missing.append(req) except NotImplementedError: missing.append(req) if not missing: return still_missing = [] apt_requirements = [] for m in missing: apt_req = self.resolve(m) if apt_req is None: still_missing.append(m) else: apt_requirements.append(apt_req) if apt_requirements: self.apt.satisfy( [PkgRelation.str(chain(*[r.relations for r in apt_requirements]))] ) if still_missing: raise UnsatisfiedRequirements(still_missing) def explain(self, requirements): apt_requirements = [] for r in requirements: apt_req = self.resolve(r) if apt_req is not None: apt_requirements.append((r, apt_req)) if apt_requirements: yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements]) def resolve(self, req: Requirement): ret = resolve_requirement_apt(self.apt, req) if not ret: return None if len(ret) == 1: return ret[0] logging.info('Need to break tie between %r with %r', ret, self.tie_breakers) for tie_breaker in self.tie_breakers: winner = tie_breaker(ret) if winner is not None: if not isinstance(winner, AptRequirement): raise TypeError(winner) return winner logging.info( 'Unable to break tie over %r, picking first: %r', ret, ret[0]) return ret[0]