Use resolver in more cases.

This commit is contained in:
Jelmer Vernooij 2021-02-24 01:28:49 +00:00
parent 9064024b83
commit 5beef23fc8
4 changed files with 627 additions and 376 deletions

View file

@ -29,8 +29,6 @@ from debian.deb822 import Release
from .. import DetailedFailure
from ..session import Session, run_with_tee
from .build import get_build_architecture
class UnidentifiedError(Exception):
@ -64,6 +62,12 @@ class AptManager(object):
def __init__(self, session):
self.session = session
def package_exists(self, package: str) -> bool:
raise NotImplementedError(self.package_exists)
def get_package_for_paths(self, paths, regex=False):
raise NotImplementedError(self.get_package_for_paths)
def missing(self, packages):
root = getattr(self.session, "location", "/")
status_path = os.path.join(root, "var/lib/dpkg/status")
@ -80,6 +84,7 @@ class AptManager(object):
return list(missing)
def install(self, packages: List[str]) -> None:
logging.info('Installing using apt: %r', packages)
packages = self.missing(packages)
if packages:
run_apt(self.session, ["install"] + packages)
@ -88,8 +93,30 @@ class AptManager(object):
run_apt(self.session, ["satisfy"] + deps)
class LocalAptManager(AptManager):
def __init__(self):
from ..session.plain import PlainSession
self.session = PlainSession()
self._apt_cache = None
def package_exists(self, package):
if self._apt_cache is None:
import apt_pkg
self._apt_cache = apt_pkg.Cache()
for p in self._apt_cache.packages:
if p.name == package:
return True
return False
def get_package_for_paths(self, paths, regex=False):
# TODO(jelmer): Make sure we use whatever is configured in self.session
return get_package_for_paths(paths, regex=regex)
class FileSearcher(object):
def search_files(self, path, regex=False):
def search_files(self, path: str, regex: bool = False) -> Iterator[str]:
raise NotImplementedError(self.search_files)
@ -98,9 +125,6 @@ class ContentsFileNotFound(Exception):
class AptContentsFileSearcher(FileSearcher):
_user_agent = 'ognibuild/0.1'
def __init__(self):
self._db = {}
@ -137,6 +161,7 @@ class AptContentsFileSearcher(FileSearcher):
@classmethod
def from_repositories(cls, sources):
from .debian.build import get_build_architecture
# TODO(jelmer): Verify signatures, etc.
urls = []
arches = [get_build_architecture(), "all"]
@ -159,11 +184,11 @@ class AptContentsFileSearcher(FileSearcher):
urls.append("%s/%s/%s" % (base_url, name, entry["name"]))
return cls.from_urls(urls)
@classmethod
def _get(cls, url):
@staticmethod
def _get(url):
from urllib.request import urlopen, Request
request = Request(url, headers={"User-Agent": cls._user_agent})
request = Request(url, headers={"User-Agent": "Debian Janitor"})
return urlopen(request)
def load_url(self, url):
@ -192,7 +217,7 @@ class GeneratedFileSearcher(FileSearcher):
def __init__(self, db):
self._db = db
def search_files(self, path, regex=False):
def search_files(self, path: str, regex: bool = False) -> Iterator[str]:
for p, pkg in sorted(self._db.items()):
if regex:
if re.match(path, p):
@ -215,7 +240,7 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher(
_apt_file_searcher = None
def search_apt_file(path: str, regex: bool = False) -> Iterator[FileSearcher]:
def search_apt_file(path: str, regex: bool = False) -> Iterator[str]:
global _apt_file_searcher
if _apt_file_searcher is None:
# TODO(jelmer): cache file
@ -223,3 +248,22 @@ def search_apt_file(path: str, regex: bool = False) -> Iterator[FileSearcher]:
if _apt_file_searcher:
yield from _apt_file_searcher.search_files(path, regex=regex)
yield from GENERATED_FILE_SEARCHER.search_files(path, regex=regex)
def get_package_for_paths(paths: List[str], regex: bool = False) -> Optional[str]:
candidates: Set[str] = set()
for path in paths:
candidates.update(search_apt_file(path, regex=regex))
if candidates:
break
if len(candidates) == 0:
logging.warning("No packages found that contain %r", paths)
return None
if len(candidates) > 1:
logging.warning(
"More than 1 packages found that contain %r: %r", path, candidates
)
# Euhr. Pick the one with the shortest name?
return sorted(candidates, key=len)[0]
else:
return candidates.pop()

View file

@ -21,7 +21,6 @@ __all__ = [
import logging
import os
import subprocess
import sys
from typing import List, Callable, Type, Tuple, Set, Optional
@ -106,9 +105,35 @@ from buildlog_consultant.sbuild import (
SbuildFailure,
)
from ..apt import AptManager, LocalAptManager
from ..resolver.apt import AptResolver
from ..requirements import BinaryRequirement
from .apt import AptManager, LocalAptManager
from ..resolver.apt import (
AptResolver,
NoAptPackage,
get_package_for_python_module,
)
from ..requirements import (
BinaryRequirement,
PathRequirement,
PkgConfigRequirement,
CHeaderRequirement,
JavaScriptRuntimeRequirement,
ValaPackageRequirement,
RubyGemRequirement,
GoPackageRequirement,
DhAddonRequirement,
PhpClassRequirement,
RPackageRequirement,
NodePackageRequirement,
LibraryRequirement,
RubyFileRequirement,
XmlEntityRequirement,
SprocketsFileRequirement,
JavaClassRequirement,
HaskellPackageRequirement,
MavenArtifactRequirement,
GnomeCommonRequirement,
JDKFileRequirement,
)
from .build import attempt_build, DEFAULT_BUILDER
@ -148,7 +173,7 @@ class DependencyContext(object):
class BuildDependencyContext(DependencyContext):
def add_dependency(self, package, minimum_version=None):
def add_dependency(self, package: str, minimum_version: Optional[Version] = None):
return add_build_dependency(
self.tree,
package,
@ -181,12 +206,12 @@ class AutopkgtestDependencyContext(DependencyContext):
def add_build_dependency(
tree,
package,
minimum_version=None,
committer=None,
subpath="",
update_changelog=True,
tree: Tree,
package: str,
minimum_version: Optional[Version] = None,
committer: Optional[str] = None,
subpath: str = "",
update_changelog: bool = True,
):
if not isinstance(package, str):
raise TypeError(package)
@ -305,62 +330,6 @@ def commit_debian_changes(
return True
def get_package_for_python_module(apt, module, python_version):
if python_version == "python3":
paths = [
os.path.join(
"/usr/lib/python3/dist-packages",
module.replace(".", "/"),
"__init__.py",
),
os.path.join(
"/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py"
),
os.path.join(
"/usr/lib/python3\\.[0-9]+/lib-dynload",
module.replace(".", "/") + "\\.cpython-.*\\.so",
),
os.path.join(
"/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py"
),
os.path.join(
"/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py"
),
]
elif python_version == "python2":
paths = [
os.path.join(
"/usr/lib/python2\\.[0-9]/dist-packages",
module.replace(".", "/"),
"__init__.py",
),
os.path.join(
"/usr/lib/python2\\.[0-9]/dist-packages",
module.replace(".", "/") + ".py",
),
os.path.join(
"/usr/lib/python2.\\.[0-9]/lib-dynload",
module.replace(".", "/") + ".so",
),
]
elif python_version == "pypy":
paths = [
os.path.join(
"/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py"
),
os.path.join(
"/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py"
),
os.path.join(
"/usr/lib/pypy/dist-packages",
module.replace(".", "/") + "\\.pypy-.*\\.so",
),
]
else:
raise AssertionError("unknown python version %r" % python_version)
return apt.get_package_for_paths(paths, regex=True)
def targeted_python_versions(tree: Tree) -> Set[str]:
with tree.get_file("debian/control") as f:
control = Deb822(f)
@ -378,13 +347,6 @@ def targeted_python_versions(tree: Tree) -> Set[str]:
return targeted
def fix_missing_javascript_runtime(error, context):
package = context.apt.get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False)
if package is None:
return False
return context.add_dependency(package)
def fix_missing_python_distribution(error, context): # noqa: C901
targeted = targeted_python_versions(context.tree)
default = not targeted
@ -496,62 +458,60 @@ def fix_missing_python_module(error, context):
return True
def fix_missing_go_package(error, context):
package = context.apt.get_package_for_paths(
[os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True
)
if package is None:
return False
return context.add_dependency(package)
def fix_missing_c_header(error, context):
package = context.apt.get_package_for_paths(
[os.path.join("/usr/include", error.header)], regex=False
)
if package is None:
package = context.apt.get_package_for_paths(
[os.path.join("/usr/include", ".*", error.header)], regex=True
)
if package is None:
return False
return context.add_dependency(package)
def fix_missing_pkg_config(error, context):
package = context.apt.get_package_for_paths(
[os.path.join("/usr/lib/pkgconfig", error.module + ".pc")]
)
if package is None:
package = context.apt.get_package_for_paths(
[os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")],
regex=True,
)
if package is None:
return False
return context.add_dependency(package, minimum_version=error.minimum_version)
def fix_missing_command(error, context):
package = context.resolve_apt(BinaryRequirement(error.command))
return context.add_dependency(package)
def fix_missing_file(error, context):
package = context.apt.get_package_for_paths([error.path])
if package is None:
return False
return context.add_dependency(package)
def fix_missing_sprockets_file(error, context):
if error.content_type == "application/javascript":
path = "/usr/share/.*/app/assets/javascripts/%s.js$" % error.name
def fix_missing_requirement(error, context):
if isinstance(error, MissingFile):
req = PathRequirement(error.path)
elif isinstance(error, MissingCommand):
req = BinaryRequirement(error.command)
elif isinstance(error, MissingPkgConfig):
req = PkgConfigRequirement(
error.module, error.minimum_version)
elif isinstance(error, MissingCHeader):
req = CHeaderRequirement(error.header)
elif isinstance(error, MissingJavaScriptRuntime):
req = JavaScriptRuntimeRequirement()
elif isinstance(error, MissingRubyGem):
req = RubyGemRequirement(error.gem, error.version)
elif isinstance(error, MissingValaPackage):
req = ValaPackageRequirement(error.package)
elif isinstance(error, MissingGoPackage):
req = GoPackageRequirement(error.package)
elif isinstance(error, DhAddonLoadFailure):
req = DhAddonRequirement(error.path)
elif isinstance(error, MissingPhpClass):
req = PhpClassRequirement(error.php_class)
elif isinstance(error, MissingRPackage):
req = RPackageRequirement(error.package, error.minimum_version)
elif isinstance(error, MissingNodeModule):
req = NodePackageRequirement(error.module)
elif isinstance(error, MissingLibrary):
req = LibraryRequirement(error.library)
elif isinstance(error, MissingRubyFile):
req = RubyFileRequirement(error.filename)
elif isinstance(error, MissingXmlEntity):
req = XmlEntityRequirement(error.url)
elif isinstance(error, MissingSprocketsFile):
req = SprocketsFileRequirement(error.content_type, error.name)
elif isinstance(error, MissingJavaClass):
req = JavaClassRequirement(error.classname)
elif isinstance(error, MissingHaskellDependencies):
# TODO(jelmer): Create multiple HaskellPackageRequirement objects?
req = HaskellPackageRequirement(error.package)
elif isinstance(error, MissingMavenArtifacts):
# TODO(jelmer): Create multiple MavenArtifactRequirement objects?
req = MavenArtifactRequirement(error.artifacts)
elif isinstance(error, MissingCSharpCompiler):
req = BinaryRequirement('msc')
elif isinstance(error, GnomeCommonMissing):
req = GnomeCommonRequirement()
elif isinstance(error, MissingJDKFile):
req = JDKFileRequirement(error.jdk_path, error.filename)
else:
logging.warning("unable to handle content type %s", error.content_type)
return False
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
return None
try:
package = context.resolve_apt(req)
except NoAptPackage:
return False
return context.add_dependency(package)
@ -595,159 +555,10 @@ def fix_missing_perl_file(error, context):
return context.add_dependency(package)
def get_package_for_node_package(apt, node_package):
paths = [
"/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package,
"/usr/lib/nodejs/%s/package.json" % node_package,
"/usr/share/nodejs/%s/package.json" % node_package,
]
return apt.get_package_for_paths(paths, regex=True)
def fix_missing_node_module(error, context):
package = get_package_for_node_package(context.apt, error.module)
if package is None:
logging.warning("no node package found for %s.", error.module)
return False
return context.add_dependency(package)
def fix_missing_dh_addon(error, context):
paths = [os.path.join("/usr/share/perl5", error.path)]
package = context.apt.get_package_for_paths(paths)
if package is None:
logging.warning("no package for debhelper addon %s", error.name)
return False
return context.add_dependency(package)
def retry_apt_failure(error, context):
return True
def fix_missing_php_class(error, context):
path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/")
package = context.apt.get_package_for_paths([path])
if package is None:
logging.warning("no package for PHP class %s", error.php_class)
return False
return context.add_dependency(package)
def fix_missing_jdk_file(error, context):
path = error.jdk_path + ".*/" + error.filename
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
logging.warning(
"no package found for %s (JDK: %s) - regex %s",
error.filename,
error.jdk_path,
path,
)
return False
return context.add_dependency(package)
def fix_missing_vala_package(error, context):
path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
logging.warning("no file found for package %s - regex %s", error.package, path)
return False
return context.add_dependency(package)
def fix_missing_xml_entity(error, context):
# Ideally we should be using the XML catalog for this, but hardcoding
# a few URLs will do for now..
URL_MAP = {
"http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/"
}
for url, path in URL_MAP.items():
if error.url.startswith(url):
search_path = os.path.join(path, error.url[len(url) :])
break
else:
return False
package = context.apt.get_package_for_paths([search_path], regex=False)
if package is None:
return False
return context.add_dependency(package)
def fix_missing_library(error, context):
paths = [
os.path.join("/usr/lib/lib%s.so$" % error.library),
os.path.join("/usr/lib/.*/lib%s.so$" % error.library),
os.path.join("/usr/lib/lib%s.a$" % error.library),
os.path.join("/usr/lib/.*/lib%s.a$" % error.library),
]
package = context.apt.get_package_for_paths(paths, regex=True)
if package is None:
logging.warning("no package for library %s", error.library)
return False
return context.add_dependency(package)
def fix_missing_ruby_gem(error, context):
paths = [
os.path.join(
"/usr/share/rubygems-integration/all/"
"specifications/%s-.*\\.gemspec" % error.gem
)
]
package = context.apt.get_package_for_paths(paths, regex=True)
if package is None:
logging.warning("no package for gem %s", error.gem)
return False
return context.add_dependency(package, minimum_version=error.version)
def fix_missing_ruby_file(error, context):
paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)]
package = context.apt.get_package_for_paths(paths)
if package is not None:
return context.add_dependency(package)
paths = [
os.path.join(
r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
"lib/%s.rb" % error.filename
)
]
package = context.apt.get_package_for_paths(paths, regex=True)
if package is not None:
return context.add_dependency(package)
logging.warning("no package for ruby file %s", error.filename)
return False
def fix_missing_r_package(error, context):
paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)]
package = context.apt.get_package_for_paths(paths, regex=True)
if package is None:
logging.warning("no package for R package %s", error.package)
return False
return context.add_dependency(package, minimum_version=error.minimum_version)
def fix_missing_java_class(error, context):
# Unfortunately this only finds classes in jars installed on the host
# system :(
output = subprocess.check_output(["java-propose-classpath", "-c" + error.classname])
classpath = [p for p in output.decode().strip(":").strip().split(":") if p]
if not classpath:
logging.warning("unable to find classpath for %s", error.classname)
return False
logging.info("Classpath for %s: %r", error.classname, classpath)
package = context.apt.get_package_for_paths(classpath)
if package is None:
logging.warning("no package for files in %r", classpath)
return False
return context.add_dependency(package)
def enable_dh_autoreconf(context):
# Debhelper >= 10 depends on dh-autoreconf and enables autoreconf by
# default.
@ -768,9 +579,8 @@ def enable_dh_autoreconf(context):
def fix_missing_configure(error, context):
if not context.tree.has_filename("configure.ac") and not context.tree.has_filename(
"configure.in"
):
if (not context.tree.has_filename("configure.ac") and
not context.tree.has_filename("configure.in")):
return False
return enable_dh_autoreconf(context)
@ -783,43 +593,6 @@ def fix_missing_automake_input(error, context):
return enable_dh_autoreconf(context)
def fix_missing_maven_artifacts(error, context):
artifact = error.artifacts[0]
parts = artifact.split(":")
if len(parts) == 4:
(group_id, artifact_id, kind, version) = parts
regex = False
elif len(parts) == 3:
(group_id, artifact_id, version) = parts
kind = "jar"
regex = False
elif len(parts) == 2:
version = ".*"
(group_id, artifact_id) = parts
kind = "jar"
regex = True
else:
raise AssertionError("invalid number of parts to artifact %s" % artifact)
paths = [
os.path.join(
"/usr/share/maven-repo",
group_id.replace(".", "/"),
artifact_id,
version,
"%s-%s.%s" % (artifact_id, version, kind),
)
]
package = context.apt.get_package_for_paths(paths, regex=regex)
if package is None:
logging.warning("no package for artifact %s", artifact)
return False
return context.add_dependency(package)
def install_gnome_common(error, context):
return context.add_dependency("gnome-common")
def install_gnome_common_dep(error, context):
if error.package == "glib-gettext":
package = context.apt.get_package_for_paths(["/usr/bin/glib-gettextize"])
@ -914,19 +687,6 @@ def fix_missing_autoconf_macro(error, context):
return context.add_dependency(package)
def fix_missing_c_sharp_compiler(error, context):
return context.add_dependency("mono-mcs")
def fix_missing_haskell_dependencies(error, context):
path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0]
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
logging.warning("no package for macro file %s", path)
return False
return context.add_dependency(package)
VERSIONED_PACKAGE_FIXERS: List[
Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]
] = [
@ -939,35 +699,14 @@ VERSIONED_PACKAGE_FIXERS: List[
APT_FIXERS: List[Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]] = [
(MissingPythonModule, fix_missing_python_module),
(MissingPythonDistribution, fix_missing_python_distribution),
(MissingCHeader, fix_missing_c_header),
(MissingPkgConfig, fix_missing_pkg_config),
(MissingCommand, fix_missing_command),
(MissingFile, fix_missing_file),
(MissingSprocketsFile, fix_missing_sprockets_file),
(MissingGoPackage, fix_missing_go_package),
(MissingPerlFile, fix_missing_perl_file),
(MissingPerlModule, fix_missing_perl_file),
(MissingXmlEntity, fix_missing_xml_entity),
(MissingNodeModule, fix_missing_node_module),
(MissingRubyGem, fix_missing_ruby_gem),
(MissingRPackage, fix_missing_r_package),
(MissingLibrary, fix_missing_library),
(MissingJavaClass, fix_missing_java_class),
(DhAddonLoadFailure, fix_missing_dh_addon),
(MissingPhpClass, fix_missing_php_class),
(AptFetchFailure, retry_apt_failure),
(MissingMavenArtifacts, fix_missing_maven_artifacts),
(GnomeCommonMissing, install_gnome_common),
(MissingGnomeCommonDependency, install_gnome_common_dep),
(MissingXfceDependency, install_xfce_dep),
(MissingConfigStatusInput, fix_missing_config_status_input),
(MissingJDKFile, fix_missing_jdk_file),
(MissingRubyFile, fix_missing_ruby_file),
(MissingJavaScriptRuntime, fix_missing_javascript_runtime),
(MissingAutoconfMacro, fix_missing_autoconf_macro),
(MissingValaPackage, fix_missing_vala_package),
(MissingCSharpCompiler, fix_missing_c_sharp_compiler),
(MissingHaskellDependencies, fix_missing_haskell_dependencies),
(Problem, fix_missing_requirement),
]

View file

@ -16,6 +16,9 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
import posixpath
from typing import Optional, List, Tuple
from . import UpstreamRequirement
@ -62,3 +65,182 @@ class CargoCrateRequirement(UpstreamRequirement):
def __init__(self, crate):
super(CargoCrateRequirement, self).__init__('cargo-crate')
self.crate = crate
class PkgConfigRequirement(UpstreamRequirement):
module: str
def __init__(self, module, minimum_version=None):
super(PkgConfigRequirement, self).__init__('pkg-config')
self.module = module
self.minimum_version = minimum_version
class PathRequirement(UpstreamRequirement):
path: str
def __init__(self, path):
super(PathRequirement, self).__init__('path')
self.path = path
class CHeaderRequirement(UpstreamRequirement):
header: str
def __init__(self, header):
super(CHeaderRequirement, self).__init__('c-header')
self.header = header
class JavaScriptRuntimeRequirement(UpstreamRequirement):
def __init__(self):
super(JavaScriptRuntimeRequirement, self).__init__(
'javascript-runtime')
class ValaPackageRequirement(UpstreamRequirement):
package: str
def __init__(self, package: str):
super(ValaPackageRequirement, self).__init__('vala')
self.package = package
class RubyGemRequirement(UpstreamRequirement):
gem: str
minimum_version: Optional[str]
def __init__(self, gem: str, minimum_version: Optional[str]):
super(RubyGemRequirement, self).__init__('gem')
self.gem = gem
self.minimum_version = minimum_version
class GoPackageRequirement(UpstreamRequirement):
package: str
def __init__(self, package: str):
super(GoPackageRequirement, self).__init__('go')
self.package = package
class DhAddonRequirement(UpstreamRequirement):
path: str
def __init__(self, path: str):
super(DhAddonRequirement, self).__init__('dh-addon')
self.path = path
class PhpClassRequirement(UpstreamRequirement):
php_class: str
def __init__(self, php_class: str):
super(PhpClassRequirement, self).__init__('php-class')
self.php_class = php_class
class RPackageRequirement(UpstreamRequirement):
package: str
minimum_version: Optional[str]
def __init__(self, package: str, minimum_version: Optional[str] = None):
super(RPackageRequirement, self).__init__('r-package')
self.package = package
self.minimum_version = minimum_version
class LibraryRequirement(UpstreamRequirement):
library: str
def __init__(self, library: str):
super(LibraryRequirement, self).__init__('lib')
self.library = library
class RubyFileRequirement(UpstreamRequirement):
filename: str
def __init__(self, filename: str):
super(RubyFileRequirement, self).__init__('ruby-file')
self.filename = filename
class XmlEntityRequirement(UpstreamRequirement):
url: str
def __init__(self, url: str):
super(XmlEntityRequirement, self).__init__('xml-entity')
self.url = url
class SprocketsFileRequirement(UpstreamRequirement):
content_type: str
name: str
def __init__(self, content_type: str, name: str):
super(SprocketsFileRequirement, self).__init__('sprockets-file')
self.content_type = content_type
self.name = name
class JavaClassRequirement(UpstreamRequirement):
classname: str
def __init__(self, classname: str):
super(JavaClassRequirement, self).__init__('java-class')
self.classname = classname
class HaskellPackageRequirement(UpstreamRequirement):
package: str
def __init__(self, package: str):
super(HaskellPackageRequirement, self).__init__('haskell-package')
self.package = package
class MavenArtifactRequirement(UpstreamRequirement):
artifacts: List[Tuple[str, str, str]]
def __init__(self, artifacts):
super(MavenArtifactRequirement, self).__init__('maven-artifact')
self.artifacts = artifacts
class GnomeCommonRequirement(UpstreamRequirement):
def __init__(self):
super(GnomeCommonRequirement, self).__init__('gnome-common')
class JDKFileRequirement(UpstreamRequirement):
jdk_path: str
filename: str
def __init__(self, jdk_path: str, filename: str):
super(JDKFileRequirement, self).__init__('jdk-file')
self.jdk_path = jdk_path
self.filename = filename
@property
def path(self):
return posixpath.join(self.jdk_path, self.filename)

View file

@ -15,13 +15,35 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
import logging
import posixpath
from ..apt import AptManager
from ..debian.apt import AptManager
from . import Resolver
from ..requirements import (
BinaryRequirement,
CHeaderRequirement,
PkgConfigRequirement,
PathRequirement,
UpstreamRequirement,
JavaScriptRuntimeRequirement,
ValaPackageRequirement,
RubyGemRequirement,
GoPackageRequirement,
DhAddonRequirement,
PhpClassRequirement,
RPackageRequirement,
NodePackageRequirement,
LibraryRequirement,
RubyFileRequirement,
XmlEntityRequirement,
SprocketsFileRequirement,
JavaClassRequirement,
HaskellPackageRequirement,
MavenArtifactRequirement,
GnomeCommonRequirement,
JDKFileRequirement,
)
@ -29,6 +51,62 @@ class NoAptPackage(Exception):
"""No apt package."""
def get_package_for_python_module(apt_mgr, module, python_version):
if python_version == "python3":
paths = [
posixpath.join(
"/usr/lib/python3/dist-packages",
module.replace(".", "/"),
"__init__.py",
),
posixpath.join(
"/usr/lib/python3/dist-packages", module.replace(".", "/") + ".py"
),
posixpath.join(
"/usr/lib/python3\\.[0-9]+/lib-dynload",
module.replace(".", "/") + "\\.cpython-.*\\.so",
),
posixpath.join(
"/usr/lib/python3\\.[0-9]+/", module.replace(".", "/") + ".py"
),
posixpath.join(
"/usr/lib/python3\\.[0-9]+/", module.replace(".", "/"), "__init__.py"
),
]
elif python_version == "python2":
paths = [
posixpath.join(
"/usr/lib/python2\\.[0-9]/dist-packages",
module.replace(".", "/"),
"__init__.py",
),
posixpath.join(
"/usr/lib/python2\\.[0-9]/dist-packages",
module.replace(".", "/") + ".py",
),
posixpath.join(
"/usr/lib/python2.\\.[0-9]/lib-dynload",
module.replace(".", "/") + ".so",
),
]
elif python_version == "pypy":
paths = [
posixpath.join(
"/usr/lib/pypy/dist-packages", module.replace(".", "/"), "__init__.py"
),
posixpath.join(
"/usr/lib/pypy/dist-packages", module.replace(".", "/") + ".py"
),
posixpath.join(
"/usr/lib/pypy/dist-packages",
module.replace(".", "/") + "\\.pypy-.*\\.so",
),
]
else:
raise AssertionError("unknown python version %r" % python_version)
return apt_mgr.get_package_for_paths(paths, regex=True)
def resolve_binary_req(apt_mgr, req):
if posixpath.isabs(req.binary_name):
paths = [req.binary_name]
@ -40,8 +118,218 @@ def resolve_binary_req(apt_mgr, req):
return apt_mgr.get_package_for_paths(paths)
def resolve_pkg_config_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")],
req.minimum_version
)
if package is None:
package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")],
regex=True,
minimum_version=req.minimum_version)
return package
def resolve_path_req(apt_mgr, req):
return apt_mgr.get_package_for_paths([req.path])
def resolve_c_header_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/include", req.header)], regex=False
)
if package is None:
package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/include", ".*", req.header)], regex=True
)
return package
def resolve_js_runtime_req(apt_mgr, req):
return apt_mgr.get_package_for_paths(
["/usr/bin/node", "/usr/bin/duk"], regex=False)
def resolve_vala_package_req(apt_mgr, req):
path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % req.package
return apt_mgr.get_package_for_paths([path], regex=True)
def resolve_ruby_gem_req(apt_mgr, req):
paths = [
posixpath.join(
"/usr/share/rubygems-integration/all/"
"specifications/%s-.*\\.gemspec" % req.gem
)
]
return apt_mgr.get_package_for_paths(
paths, regex=True, minimum_version=req.minimum_version)
def resolve_go_package_req(apt_mgr, req):
return apt_mgr.get_package_for_paths(
[posixpath.join("/usr/share/gocode/src", req.package, ".*")],
regex=True
)
def resolve_dh_addon_req(apt_mgr, req):
paths = [posixpath.join("/usr/share/perl5", req.path)]
return apt_mgr.get_package_for_paths(paths)
def resolve_php_class_req(apt_mgr, req):
path = "/usr/share/php/%s.php" % req.php_class.replace("\\", "/")
return apt_mgr.get_package_for_paths([path])
def resolve_r_package_req(apt_mgr, req):
paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % req.package)]
return apt_mgr.get_package_for_paths(paths, regex=True)
def resolve_node_package_req(apt_mgr, req):
paths = [
"/usr/share/nodejs/.*/node_modules/%s/package.json" % req.package,
"/usr/lib/nodejs/%s/package.json" % req.package,
"/usr/share/nodejs/%s/package.json" % req.package,
]
return apt_mgr.get_package_for_paths(paths, regex=True)
def resolve_library_req(apt_mgr, req):
paths = [
posixpath.join("/usr/lib/lib%s.so$" % req.library),
posixpath.join("/usr/lib/.*/lib%s.so$" % req.library),
posixpath.join("/usr/lib/lib%s.a$" % req.library),
posixpath.join("/usr/lib/.*/lib%s.a$" % req.library),
]
return apt_mgr.get_package_for_paths(paths, regex=True)
def resolve_ruby_file_req(apt_mgr, req):
paths = [posixpath.join("/usr/lib/ruby/vendor_ruby/%s.rb" % req.filename)]
package = apt_mgr.get_package_for_paths(paths)
if package is not None:
return package
paths = [
posixpath.join(
r"/usr/share/rubygems-integration/all/gems/([^/]+)/"
"lib/%s.rb" % req.filename
)
]
return apt_mgr.get_package_for_paths(paths, regex=True)
def resolve_xml_entity_req(apt_mgr, req):
# Ideally we should be using the XML catalog for this, but hardcoding
# a few URLs will do for now..
URL_MAP = {
"http://www.oasis-open.org/docbook/xml/": "/usr/share/xml/docbook/schema/dtd/"
}
for url, path in URL_MAP.items():
if req.url.startswith(url):
search_path = posixpath.join(path, req.url[len(url) :])
break
else:
return None
return apt_mgr.get_package_for_paths([search_path], regex=False)
def resolve_sprockets_file_req(apt_mgr, req):
if req.content_type == "application/javascript":
path = "/usr/share/.*/app/assets/javascripts/%s.js$" % req.name
else:
logging.warning("unable to handle content type %s", req.content_type)
return None
return apt_mgr.get_package_for_paths([path], regex=True)
def resolve_java_class_req(apt_mgr, req):
# Unfortunately this only finds classes in jars installed on the host
# system :(
# TODO(jelmer): Call in session
output = apt_mgr.session.check_output(
["java-propose-classpath", "-c" + req.classname])
classpath = [p for p in output.decode().strip(":").strip().split(":") if p]
if not classpath:
logging.warning("unable to find classpath for %s", req.classname)
return False
logging.info("Classpath for %s: %r", req.classname, classpath)
package = apt_mgr.get_package_for_paths(classpath)
if package is None:
logging.warning("no package for files in %r", classpath)
return None
return package
def resolve_haskell_package_req(apt_mgr, req):
path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % req.deps[0][0]
return apt_mgr.get_package_for_paths([path], regex=True)
def resolve_maven_artifact_req(apt_mgr, req):
artifact = req.artifacts[0]
parts = artifact.split(":")
if len(parts) == 4:
(group_id, artifact_id, kind, version) = parts
regex = False
elif len(parts) == 3:
(group_id, artifact_id, version) = parts
kind = "jar"
regex = False
elif len(parts) == 2:
version = ".*"
(group_id, artifact_id) = parts
kind = "jar"
regex = True
else:
raise AssertionError("invalid number of parts to artifact %s" % artifact)
paths = [
posixpath.join(
"/usr/share/maven-repo",
group_id.replace(".", "/"),
artifact_id,
version,
"%s-%s.%s" % (artifact_id, version, kind),
)
]
return apt_mgr.get_package_for_paths(paths, regex=regex)
def resolve_gnome_common_req(apt_mgr, req):
return 'gnome-common'
def resolve_jdk_file_req(apt_mgr, req):
path = req.jdk_path + ".*/" + req.filename
return apt_mgr.get_package_for_paths([path], regex=True)
APT_REQUIREMENT_RESOLVERS = [
(BinaryRequirement, resolve_binary_req),
(PkgConfigRequirement, resolve_pkg_config_req),
(PathRequirement, resolve_path_req),
(CHeaderRequirement, resolve_c_header_req),
(JavaScriptRuntimeRequirement, resolve_js_runtime_req),
(ValaPackageRequirement, resolve_vala_package_req),
(RubyGemRequirement, resolve_ruby_gem_req),
(GoPackageRequirement, resolve_go_package_req),
(DhAddonRequirement, resolve_dh_addon_req),
(PhpClassRequirement, resolve_php_class_req),
(RPackageRequirement, resolve_r_package_req),
(NodePackageRequirement, resolve_node_package_req),
(LibraryRequirement, resolve_library_req),
(RubyFileRequirement, resolve_ruby_file_req),
(XmlEntityRequirement, resolve_xml_entity_req),
(SprocketsFileRequirement, resolve_sprockets_file_req),
(JavaClassRequirement, resolve_java_class_req),
(HaskellPackageRequirement, resolve_haskell_package_req),
(MavenArtifactRequirement, resolve_maven_artifact_req),
(GnomeCommonRequirement, resolve_gnome_common_req),
(JDKFileRequirement, resolve_jdk_file_req),
]
@ -70,14 +358,12 @@ class AptResolver(Resolver):
def explain(self, requirements):
raise NotImplementedError(self.explain)
def resolve(self, requirements):
for req in requirements:
for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
if isinstance(req, rr_class):
package_name = rr_fn(self.apt, req)
if package_name is None:
raise NoAptPackage()
yield package_name
break
else:
raise NotImplementedError
def resolve(self, req: UpstreamRequirement):
for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
if isinstance(req, rr_class):
deb_req = rr_fn(self.apt, req)
if deb_req is None:
raise NoAptPackage()
return deb_req
else:
raise NotImplementedError