More fixes.
This commit is contained in:
Jelmer Vernooij
parent
aa2a3e47fa
commit
6b30479b97
9 changed files with 499 additions and 320 deletions
|
|
@ -70,14 +70,9 @@ class AptManager(object):
|
|||
|
||||
def package_exists(self, package):
|
||||
if self._apt_cache is None:
|
||||
import apt_pkg
|
||||
|
||||
# TODO(jelmer): Load from self.session
|
||||
self._apt_cache = apt_pkg.Cache()
|
||||
for p in self._apt_cache.packages:
|
||||
if p.name == package:
|
||||
return True
|
||||
return False
|
||||
import apt
|
||||
self._apt_cache = apt.Cache(rootdir=self.session.location)
|
||||
return package in self._apt_cache
|
||||
|
||||
def get_package_for_paths(self, paths, regex=False):
|
||||
logging.debug('Searching for packages containing %r', paths)
|
||||
|
|
@ -121,10 +116,12 @@ class RemoteAptContentsFileSearcher(FileSearcher):
|
|||
def from_session(cls, session):
|
||||
logging.info('Loading apt contents information')
|
||||
# TODO(jelmer): what about sources.list.d?
|
||||
with open(os.path.join(session.location, 'etc/apt/sources.list'), 'r') as f:
|
||||
return cls.from_repositories(
|
||||
f.readlines(),
|
||||
cache_dir=os.path.join(session.location, 'var/lib/apt/lists'))
|
||||
from aptsources.sourceslist import SourcesList
|
||||
sl = SourcesList()
|
||||
sl.load(os.path.join(session.location, 'etc/apt/sources.list'))
|
||||
return cls.from_sources_list(
|
||||
sl,
|
||||
cache_dir=os.path.join(session.location, 'var/lib/apt/lists'))
|
||||
|
||||
def __setitem__(self, path, package):
|
||||
self._db[path] = package
|
||||
|
|
@ -174,32 +171,31 @@ class RemoteAptContentsFileSearcher(FileSearcher):
|
|||
self.load_url(url)
|
||||
except ContentsFileNotFound:
|
||||
if mandatory:
|
||||
raise
|
||||
logging.debug(
|
||||
'Unable to fetch optional contents file %s', url)
|
||||
logging.warning(
|
||||
'Unable to fetch contents file %s', url)
|
||||
else:
|
||||
logging.debug(
|
||||
'Unable to fetch optional contents file %s', url)
|
||||
return self
|
||||
|
||||
@classmethod
|
||||
def from_repositories(cls, sources, cache_dir=None):
|
||||
def from_sources_list(cls, sl, cache_dir=None):
|
||||
# TODO(jelmer): Use aptsources.sourceslist.SourcesList
|
||||
from .build import get_build_architecture
|
||||
# TODO(jelmer): Verify signatures, etc.
|
||||
urls = []
|
||||
arches = [(get_build_architecture(), True), ("all", False)]
|
||||
for source in sources:
|
||||
if not source.strip():
|
||||
for source in sl.list:
|
||||
if source.invalid or source.disabled:
|
||||
continue
|
||||
if source.strip().startswith('#'):
|
||||
if source.type == 'deb-src':
|
||||
continue
|
||||
parts = source.split(" ")
|
||||
if parts[0] == "deb-src":
|
||||
continue
|
||||
if parts[0] != "deb":
|
||||
if source.type != 'deb':
|
||||
logging.warning("Invalid line in sources: %r", source)
|
||||
continue
|
||||
base_url = parts[1].strip().rstrip("/")
|
||||
name = parts[2].strip()
|
||||
components = [c.strip() for c in parts[3:]]
|
||||
base_url = source.uri.rstrip('/')
|
||||
name = source.dist.rstrip('/')
|
||||
components = source.comps
|
||||
if components:
|
||||
dists_url = base_url + "/dists"
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -63,81 +63,28 @@ from debmutate._rules import (
|
|||
|
||||
from breezy.plugins.debian.changelog import debcommit
|
||||
from buildlog_consultant import Problem
|
||||
from buildlog_consultant.common import (
|
||||
MissingConfigStatusInput,
|
||||
MissingPythonModule,
|
||||
MissingPythonDistribution,
|
||||
MissingCHeader,
|
||||
MissingPkgConfig,
|
||||
MissingCommand,
|
||||
MissingFile,
|
||||
MissingJavaScriptRuntime,
|
||||
MissingSprocketsFile,
|
||||
MissingGoPackage,
|
||||
MissingPerlFile,
|
||||
MissingPerlModule,
|
||||
MissingXmlEntity,
|
||||
MissingJDKFile,
|
||||
MissingNodeModule,
|
||||
MissingPhpClass,
|
||||
MissingRubyGem,
|
||||
MissingLibrary,
|
||||
MissingJavaClass,
|
||||
MissingCSharpCompiler,
|
||||
MissingConfigure,
|
||||
MissingAutomakeInput,
|
||||
MissingRPackage,
|
||||
MissingRubyFile,
|
||||
MissingAutoconfMacro,
|
||||
MissingValaPackage,
|
||||
MissingXfceDependency,
|
||||
MissingHaskellDependencies,
|
||||
NeedPgBuildExtUpdateControl,
|
||||
DhAddonLoadFailure,
|
||||
MissingMavenArtifacts,
|
||||
GnomeCommonMissing,
|
||||
MissingGnomeCommonDependency,
|
||||
)
|
||||
from buildlog_consultant.apt import (
|
||||
AptFetchFailure,
|
||||
)
|
||||
from buildlog_consultant.common import (
|
||||
MissingConfigStatusInput,
|
||||
MissingAutomakeInput,
|
||||
MissingConfigure,
|
||||
NeedPgBuildExtUpdateControl,
|
||||
MissingPythonModule,
|
||||
MissingPythonDistribution,
|
||||
MissingPerlFile,
|
||||
)
|
||||
from buildlog_consultant.sbuild import (
|
||||
SbuildFailure,
|
||||
)
|
||||
|
||||
from ..fix_build import BuildFixer, SimpleBuildFixer, resolve_error, DependencyContext
|
||||
from ..fix_build import BuildFixer, resolve_error, DependencyContext
|
||||
from ..buildlog import UpstreamRequirementFixer
|
||||
from ..resolver.apt import (
|
||||
NoAptPackage,
|
||||
get_package_for_python_module,
|
||||
)
|
||||
from ..requirements import (
|
||||
BinaryRequirement,
|
||||
PathRequirement,
|
||||
PkgConfigRequirement,
|
||||
CHeaderRequirement,
|
||||
JavaScriptRuntimeRequirement,
|
||||
ValaPackageRequirement,
|
||||
RubyGemRequirement,
|
||||
GoPackageRequirement,
|
||||
DhAddonRequirement,
|
||||
PhpClassRequirement,
|
||||
RPackageRequirement,
|
||||
NodePackageRequirement,
|
||||
LibraryRequirement,
|
||||
RubyFileRequirement,
|
||||
XmlEntityRequirement,
|
||||
SprocketsFileRequirement,
|
||||
JavaClassRequirement,
|
||||
HaskellPackageRequirement,
|
||||
MavenArtifactRequirement,
|
||||
GnomeCommonRequirement,
|
||||
JDKFileRequirement,
|
||||
PerlModuleRequirement,
|
||||
PerlFileRequirement,
|
||||
AutoconfMacroRequirement,
|
||||
PythonModuleRequirement,
|
||||
PythonPackageRequirement,
|
||||
)
|
||||
from .build import attempt_build, DEFAULT_BUILDER
|
||||
|
||||
|
||||
|
|
@ -437,111 +384,6 @@ def fix_missing_python_module(error, context):
|
|||
return True
|
||||
|
||||
|
||||
def problem_to_upstream_requirement(problem):
|
||||
if isinstance(problem, MissingFile):
|
||||
return PathRequirement(problem.path)
|
||||
elif isinstance(problem, MissingCommand):
|
||||
return BinaryRequirement(problem.command)
|
||||
elif isinstance(problem, MissingPkgConfig):
|
||||
return PkgConfigRequirement(
|
||||
problem.module, problem.minimum_version)
|
||||
elif isinstance(problem, MissingCHeader):
|
||||
return CHeaderRequirement(problem.header)
|
||||
elif isinstance(problem, MissingJavaScriptRuntime):
|
||||
return JavaScriptRuntimeRequirement()
|
||||
elif isinstance(problem, MissingRubyGem):
|
||||
return RubyGemRequirement(problem.gem, problem.version)
|
||||
elif isinstance(problem, MissingValaPackage):
|
||||
return ValaPackageRequirement(problem.package)
|
||||
elif isinstance(problem, MissingGoPackage):
|
||||
return GoPackageRequirement(problem.package)
|
||||
elif isinstance(problem, DhAddonLoadFailure):
|
||||
return DhAddonRequirement(problem.path)
|
||||
elif isinstance(problem, MissingPhpClass):
|
||||
return PhpClassRequirement(problem.php_class)
|
||||
elif isinstance(problem, MissingRPackage):
|
||||
return RPackageRequirement(problem.package, problem.minimum_version)
|
||||
elif isinstance(problem, MissingNodeModule):
|
||||
return NodePackageRequirement(problem.module)
|
||||
elif isinstance(problem, MissingLibrary):
|
||||
return LibraryRequirement(problem.library)
|
||||
elif isinstance(problem, MissingRubyFile):
|
||||
return RubyFileRequirement(problem.filename)
|
||||
elif isinstance(problem, MissingXmlEntity):
|
||||
return XmlEntityRequirement(problem.url)
|
||||
elif isinstance(problem, MissingSprocketsFile):
|
||||
return SprocketsFileRequirement(problem.content_type, problem.name)
|
||||
elif isinstance(problem, MissingJavaClass):
|
||||
return JavaClassRequirement(problem.classname)
|
||||
elif isinstance(problem, MissingHaskellDependencies):
|
||||
# TODO(jelmer): Create multiple HaskellPackageRequirement objects?
|
||||
return HaskellPackageRequirement(problem.package)
|
||||
elif isinstance(problem, MissingMavenArtifacts):
|
||||
# TODO(jelmer): Create multiple MavenArtifactRequirement objects?
|
||||
return MavenArtifactRequirement(problem.artifacts)
|
||||
elif isinstance(problem, MissingCSharpCompiler):
|
||||
return BinaryRequirement('msc')
|
||||
elif isinstance(problem, GnomeCommonMissing):
|
||||
return GnomeCommonRequirement()
|
||||
elif isinstance(problem, MissingJDKFile):
|
||||
return JDKFileRequirement(problem.jdk_path, problem.filename)
|
||||
elif isinstance(problem, MissingGnomeCommonDependency):
|
||||
if problem.package == "glib-gettext":
|
||||
return BinaryRequirement('glib-gettextize')
|
||||
else:
|
||||
logging.warning(
|
||||
"No known command for gnome-common dependency %s",
|
||||
problem.package)
|
||||
return None
|
||||
elif isinstance(problem, MissingXfceDependency):
|
||||
if problem.package == "gtk-doc":
|
||||
return BinaryRequirement("gtkdocize")
|
||||
else:
|
||||
logging.warning(
|
||||
"No known command for xfce dependency %s",
|
||||
problem.package)
|
||||
return None
|
||||
elif isinstance(problem, MissingPerlModule):
|
||||
return PerlModuleRequirement(
|
||||
module=problem.module,
|
||||
filename=problem.filename,
|
||||
inc=problem.inc)
|
||||
elif isinstance(problem, MissingPerlFile):
|
||||
return PerlFileRequirement(filename=problem.filename)
|
||||
elif isinstance(problem, MissingAutoconfMacro):
|
||||
return AutoconfMacroRequirement(problem.macro)
|
||||
elif isinstance(problem, MissingPythonModule):
|
||||
return PythonModuleRequirement(
|
||||
problem.module,
|
||||
python_version=problem.python_version,
|
||||
minimum_version=problem.minimum_version)
|
||||
elif isinstance(problem, MissingPythonDistribution):
|
||||
return PythonPackageRequirement(
|
||||
problem.module,
|
||||
python_version=problem.python_version,
|
||||
minimum_version=problem.minimum_version)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class UpstreamRequirementFixer(BuildFixer):
|
||||
|
||||
def can_fix(self, error):
|
||||
req = problem_to_upstream_requirement(error)
|
||||
return req is not None
|
||||
|
||||
def fix(self, error, context):
|
||||
req = problem_to_upstream_requirement(error)
|
||||
if req is None:
|
||||
return False
|
||||
|
||||
try:
|
||||
package = context.resolver.resolve(req)
|
||||
except NoAptPackage:
|
||||
return False
|
||||
return context.add_dependency(package)
|
||||
|
||||
|
||||
def retry_apt_failure(error, context):
|
||||
return True
|
||||
|
||||
|
|
@ -635,26 +477,39 @@ def fix_missing_makefile_pl(error, context):
|
|||
return False
|
||||
|
||||
|
||||
VERSIONED_PACKAGE_FIXERS: List[BuildFixer] = [
|
||||
SimpleBuildFixer(
|
||||
NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol),
|
||||
SimpleBuildFixer(MissingConfigure, fix_missing_configure),
|
||||
SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input),
|
||||
SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input),
|
||||
]
|
||||
class SimpleBuildFixer(BuildFixer):
|
||||
|
||||
def __init__(self, problem_cls, fn):
|
||||
self._problem_cls = problem_cls
|
||||
self._fn = fn
|
||||
|
||||
def can_fix(self, problem):
|
||||
return isinstance(problem, self._problem_cls)
|
||||
|
||||
def _fix(self, problem, context):
|
||||
return self._fn(problem, context)
|
||||
|
||||
|
||||
APT_FIXERS: List[BuildFixer] = [
|
||||
SimpleBuildFixer(MissingPythonModule, fix_missing_python_module),
|
||||
SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution),
|
||||
SimpleBuildFixer(AptFetchFailure, retry_apt_failure),
|
||||
UpstreamRequirementFixer(),
|
||||
]
|
||||
def versioned_package_fixers():
|
||||
return [
|
||||
SimpleBuildFixer(
|
||||
NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol),
|
||||
SimpleBuildFixer(MissingConfigure, fix_missing_configure),
|
||||
SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input),
|
||||
SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input),
|
||||
SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl),
|
||||
]
|
||||
|
||||
|
||||
GENERIC_FIXERS: List[BuildFixer] = [
|
||||
SimpleBuildFixer(MissingPerlFile, fix_missing_makefile_pl),
|
||||
]
|
||||
def apt_fixers(apt) -> List[BuildFixer]:
|
||||
from ..resolver.apt import AptResolver
|
||||
resolver = AptResolver(apt)
|
||||
return [
|
||||
SimpleBuildFixer(MissingPythonModule, fix_missing_python_module),
|
||||
SimpleBuildFixer(MissingPythonDistribution, fix_missing_python_distribution),
|
||||
SimpleBuildFixer(AptFetchFailure, retry_apt_failure),
|
||||
UpstreamRequirementFixer(resolver),
|
||||
]
|
||||
|
||||
|
||||
def build_incrementally(
|
||||
|
|
@ -720,7 +575,7 @@ def build_incrementally(
|
|||
raise
|
||||
try:
|
||||
if not resolve_error(
|
||||
e.error, context, VERSIONED_PACKAGE_FIXERS + APT_FIXERS + GENERIC_FIXERS
|
||||
e.error, context, versioned_package_fixers() + apt_fixers(apt)
|
||||
):
|
||||
logging.warning("Failed to resolve error %r. Giving up.", e.error)
|
||||
raise
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue