billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More work on resolvers.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-02-13 14:50:09 +00:00
parent 7359c07b96
commit ee5a8462f3
No known key found for this signature in database
GPG key ID: 579C160D4C9E23E8
10 changed files with 273 additions and 173 deletions
Download patch file Download diff file Expand all files Collapse all files

7
ognibuild/__init__.py
View file

@ -18,7 +18,6 @@
import os
import stat
import sys
class DetailedFailure(Exception):
@ -44,9 +43,11 @@ def shebang_binary(p):
class UpstreamRequirement(object):
def __init__(self, family, name):
# Name of the family of requirements - e.g. "python-package"
family: str
def __init__(self, family):
self.family = family
self.name = name
class UpstreamOutput(object):

3
ognibuild/__main__.py
View file

@ -25,12 +25,12 @@ from .clean import run_clean
from .dist import run_dist
from .install import run_install
from .resolver import (
AptResolver,
ExplainResolver,
AutoResolver,
NativeResolver,
MissingDependencies,
)
from .resolver.apt import AptResolver
from .test import run_test
@ -84,6 +84,7 @@ def main(): # noqa: C901
help="Ignore declared dependencies, follow build errors only",
)
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
if args.schroot:
from .session.schroot import SchrootSession

78
ognibuild/buildsystem.py
View file

@ -22,7 +22,14 @@ import os
import re
import warnings
from . import shebang_binary, UpstreamRequirement, UpstreamOutput
from . import shebang_binary, UpstreamOutput
from .requirements import (
BinaryRequirement,
PythonPackageRequirement,
PerlModuleRequirement,
NodePackageRequirement,
CargoCrateRequirement,
)
from .apt import UnidentifiedError
from .fix_build import run_with_build_fixer
@ -66,7 +73,7 @@ class Pear(BuildSystem):
self.path = path
def setup(self, resolver):
resolver.install([UpstreamRequirement("binary", "pear")])
resolver.install([BinaryRequirement("pear")])
def dist(self, session, resolver):
self.setup(resolver)
@ -94,18 +101,13 @@ class SetupPy(BuildSystem):
name = "setup.py"
def __init__(self, path):
self.path = path
from distutils.core import run_setup
self.result = run_setup(os.path.abspath(path), stop_after="init")
def setup(self, resolver):
resolver.install(
[
UpstreamRequirement("python3", "pip"),
UpstreamRequirement("binary", "python3"),
]
)
with open("setup.py", "r") as f:
resolver.install([PythonPackageRequirement('pip')])
with open(self.path, "r") as f:
setup_py_contents = f.read()
try:
with open("setup.cfg", "r") as f:
@ -114,7 +116,7 @@ class SetupPy(BuildSystem):
setup_cfg_contents = ""
if "setuptools" in setup_py_contents:
logging.info("Reference to setuptools found, installing.")
resolver.install([UpstreamRequirement("python3", "setuptools")])
resolver.install([PythonPackageRequirement("setuptools")])
if (
"setuptools_scm" in setup_py_contents
or "setuptools_scm" in setup_cfg_contents
@ -122,9 +124,9 @@ class SetupPy(BuildSystem):
logging.info("Reference to setuptools-scm found, installing.")
resolver.install(
[
UpstreamRequirement("python3", "setuptools-scm"),
UpstreamRequirement("binary", "git"),
UpstreamRequirement("binary", "mercurial"),
PythonPackageRequirement("setuptools-scm"),
BinaryRequirement("git"),
BinaryRequirement("mercurial"),
]
)
@ -150,24 +152,24 @@ class SetupPy(BuildSystem):
interpreter = shebang_binary("setup.py")
if interpreter is not None:
if interpreter in ("python3", "python2", "python"):
resolver.install([UpstreamRequirement("binary", interpreter)])
resolver.install([BinaryRequirement(interpreter)])
else:
raise ValueError("Unknown interpreter %r" % interpreter)
run_with_build_fixer(session, ["./setup.py"] + args)
else:
# Just assume it's Python 3
resolver.install([UpstreamRequirement("binary", "python3")])
resolver.install([BinaryRequirement("python3")])
run_with_build_fixer(session, ["python3", "./setup.py"] + args)
def get_declared_dependencies(self):
for require in self.result.get_requires():
yield "build", UpstreamRequirement("python3", require)
yield "build", PythonPackageRequirement(require)
if self.result.install_requires:
for require in self.result.install_requires:
yield "install", UpstreamRequirement("python3", require)
yield "install", PythonPackageRequirement(require)
if self.result.tests_require:
for require in self.result.tests_require:
yield "test", UpstreamRequirement("python3", require)
yield "test", PythonPackageRequirement(require)
def get_declared_outputs(self):
for script in self.result.scripts or []:
@ -200,8 +202,8 @@ class PyProject(BuildSystem):
)
resolver.install(
[
UpstreamRequirement("python3", "venv"),
UpstreamRequirement("python3", "pip"),
PythonPackageRequirement("venv"),
PythonPackageRequirement("pip"),
]
)
session.check_call(["pip3", "install", "poetry"], user="root")
@ -220,8 +222,8 @@ class SetupCfg(BuildSystem):
def setup(self, resolver):
resolver.install(
[
UpstreamRequirement("python3", "pep517"),
UpstreamRequirement("python3", "pip"),
PythonPackageRequirement("pep517"),
PythonPackageRequirement("pip"),
]
)
@ -244,10 +246,10 @@ class Npm(BuildSystem):
if "devDependencies" in self.package:
for name, unused_version in self.package["devDependencies"].items():
# TODO(jelmer): Look at version
yield "dev", UpstreamRequirement("npm", name)
yield "dev", NodePackageRequirement(name)
def setup(self, resolver):
resolver.install([UpstreamRequirement("binary", "npm")])
resolver.install([BinaryRequirement("npm")])
def dist(self, session, resolver):
self.setup(resolver)
@ -262,7 +264,7 @@ class Waf(BuildSystem):
self.path = path
def setup(self, resolver):
resolver.install([UpstreamRequirement("binary", "python3")])
resolver.install([BinaryRequirement("python3")])
def dist(self, session, resolver):
self.setup(resolver)
@ -277,7 +279,7 @@ class Gem(BuildSystem):
self.path = path
def setup(self, resolver):
resolver.install([UpstreamRequirement("binary", "gem2deb")])
resolver.install([BinaryRequirement("gem2deb")])
def dist(self, session, resolver):
self.setup(resolver)
@ -314,18 +316,18 @@ class DistInkt(BuildSystem):
def setup(self, resolver):
resolver.install(
[
UpstreamRequirement("perl", "Dist::Inkt"),
PerlModuleRequirement("Dist::Inkt"),
]
)
def dist(self, session, resolver):
self.setup(resolver)
if self.name == "dist-inkt":
resolver.install([UpstreamRequirement("perl-module", self.dist_inkt_class)])
resolver.install([PerlModuleRequirement(self.dist_inkt_class)])
run_with_build_fixer(session, ["distinkt-dist"])
else:
# Default to invoking Dist::Zilla
resolver.install([UpstreamRequirement("perl", "Dist::Zilla")])
resolver.install([PerlModuleRequirement("Dist::Zilla")])
run_with_build_fixer(session, ["dzil", "build", "--in", ".."])
@ -335,7 +337,7 @@ class Make(BuildSystem):
def setup(self, session, resolver):
if session.exists("Makefile.PL") and not session.exists("Makefile"):
resolver.install([UpstreamRequirement("binary", "perl")])
resolver.install([BinaryRequirement("perl")])
run_with_build_fixer(session, ["perl", "Makefile.PL"])
if not session.exists("Makefile") and not session.exists("configure"):
@ -357,10 +359,10 @@ class Make(BuildSystem):
elif session.exists("configure.ac") or session.exists("configure.in"):
resolver.install(
[
UpstreamRequirement("binary", "autoconf"),
UpstreamRequirement("binary", "automake"),
UpstreamRequirement("binary", "gettextize"),
UpstreamRequirement("binary", "libtoolize"),
BinaryRequirement("autoconf"),
BinaryRequirement("automake"),
BinaryRequirement("gettextize"),
BinaryRequirement("libtoolize"),
]
)
run_with_build_fixer(session, ["autoreconf", "-i"])
@ -370,7 +372,7 @@ class Make(BuildSystem):
def dist(self, session, resolver):
self.setup(session, resolver)
resolver.install([UpstreamRequirement("binary", "make")])
resolver.install([BinaryRequirement("make")])
try:
run_with_build_fixer(session, ["make", "dist"])
except UnidentifiedError as e:
@ -437,7 +439,7 @@ class Make(BuildSystem):
warnings.warn("Unable to parse META.yml: %s" % e)
return
for require in data.get("requires", []):
yield "build", UpstreamRequirement("perl", require)
yield "build", PerlModuleRequirement(require)
class Cargo(BuildSystem):
@ -454,7 +456,7 @@ class Cargo(BuildSystem):
if "dependencies" in self.cargo:
for name, details in self.cargo["dependencies"].items():
# TODO(jelmer): Look at details['features'], details['version']
yield "build", UpstreamRequirement("cargo-crate", name)
yield "build", CargoCrateRequirement(name)
class Golang(BuildSystem):

149
ognibuild/debian/fix_build.py
View file

@ -21,15 +21,13 @@ __all__ = [
import logging
import os
import re
import subprocess
import sys
from typing import Iterator, List, Callable, Type, Tuple, Set, Optional
from typing import List, Callable, Type, Tuple, Set, Optional
from debian.deb822 import (
Deb822,
PkgRelation,
Release,
)
from debian.changelog import Version
@ -113,6 +111,11 @@ from buildlog_consultant.sbuild import (
SbuildFailure,
)
from ..apt import AptManager, LocalAptManager
from ..resolver.apt import AptResolver
from ..requirements import BinaryRequirement
from .build import attempt_build
DEFAULT_MAX_ITERATIONS = 10
@ -128,15 +131,21 @@ class DependencyContext(object):
def __init__(
self,
tree: MutableTree,
apt: AptManager,
subpath: str = "",
committer: Optional[str] = None,
update_changelog: bool = True,
):
self.tree = tree
self.apt = apt
self.resolver = AptResolver(apt)
self.subpath = subpath
self.committer = committer
self.update_changelog = update_changelog
def resolve_apt(self, req):
return self.resolver.resolve(req)
def add_dependency(
self, package: str, minimum_version: Optional[Version] = None
) -> bool:
@ -157,11 +166,11 @@ class BuildDependencyContext(DependencyContext):
class AutopkgtestDependencyContext(DependencyContext):
def __init__(
self, testname, tree, subpath="", committer=None, update_changelog=True
self, testname, tree, apt, subpath="", committer=None, update_changelog=True
):
self.testname = testname
super(AutopkgtestDependencyContext, self).__init__(
tree, subpath, committer, update_changelog
tree, apt, subpath, committer, update_changelog
)
def add_dependency(self, package, minimum_version=None):
@ -301,27 +310,7 @@ def commit_debian_changes(
return True
def get_package_for_paths(paths, regex=False):
from .apt import search_apt_file
candidates = set()
for path in paths:
candidates.update(search_apt_file(path, regex=regex))
if candidates:
break
if len(candidates) == 0:
logging.warning("No packages found that contain %r", paths)
return None
if len(candidates) > 1:
logging.warning(
"More than 1 packages found that contain %r: %r", path, candidates
)
# Euhr. Pick the one with the shortest name?
return sorted(candidates, key=len)[0]
else:
return candidates.pop()
def get_package_for_python_module(module, python_version):
def get_package_for_python_module(apt, module, python_version):
if python_version == "python3":
paths = [
os.path.join(
@ -374,7 +363,7 @@ def get_package_for_python_module(module, python_version):
]
else:
raise AssertionError("unknown python version %r" % python_version)
return get_package_for_paths(paths, regex=True)
return apt.get_package_for_paths(paths, regex=True)
def targeted_python_versions(tree: Tree) -> Set[str]:
@ -394,23 +383,8 @@ def targeted_python_versions(tree: Tree) -> Set[str]:
return targeted
apt_cache = None
def package_exists(package):
global apt_cache
if apt_cache is None:
import apt_pkg
apt_cache = apt_pkg.Cache()
for p in apt_cache.packages:
if p.name == package:
return True
return False
def fix_missing_javascript_runtime(error, context):
package = get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False)
package = context.apt.get_package_for_paths(["/usr/bin/node", "/usr/bin/duk"], regex=False)
if package is None:
return False
return context.add_dependency(package)
@ -420,30 +394,30 @@ def fix_missing_python_distribution(error, context): # noqa: C901
targeted = targeted_python_versions(context.tree)
default = not targeted
pypy_pkg = get_package_for_paths(
pypy_pkg = context.apt.get_package_for_paths(
["/usr/lib/pypy/dist-packages/%s-.*.egg-info" % error.distribution], regex=True
)
if pypy_pkg is None:
pypy_pkg = "pypy-%s" % error.distribution
if not package_exists(pypy_pkg):
if not context.apt.package_exists(pypy_pkg):
pypy_pkg = None
py2_pkg = get_package_for_paths(
py2_pkg = context.apt.get_package_for_paths(
["/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % error.distribution],
regex=True,
)
if py2_pkg is None:
py2_pkg = "python-%s" % error.distribution
if not package_exists(py2_pkg):
if not context.apt.package_exists(py2_pkg):
py2_pkg = None
py3_pkg = get_package_for_paths(
py3_pkg = context.apt.get_package_for_paths(
["/usr/lib/python3/dist-packages/%s-.*.egg-info" % error.distribution],
regex=True,
)
if py3_pkg is None:
py3_pkg = "python3-%s" % error.distribution
if not package_exists(py3_pkg):
if not context.apt.package_exists(py3_pkg):
py3_pkg = None
extra_build_deps = []
@ -488,9 +462,9 @@ def fix_missing_python_module(error, context):
targeted = set()
default = not targeted
pypy_pkg = get_package_for_python_module(error.module, "pypy")
py2_pkg = get_package_for_python_module(error.module, "python2")
py3_pkg = get_package_for_python_module(error.module, "python3")
pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy")
py2_pkg = get_package_for_python_module(context.apt, error.module, "python2")
py3_pkg = get_package_for_python_module(context.apt, error.module, "python3")
extra_build_deps = []
if error.python_version == 2:
@ -528,7 +502,7 @@ def fix_missing_python_module(error, context):
def fix_missing_go_package(error, context):
package = get_package_for_paths(
package = context.apt.get_package_for_paths(
[os.path.join("/usr/share/gocode/src", error.package, ".*")], regex=True
)
if package is None:
@ -537,11 +511,11 @@ def fix_missing_go_package(error, context):
def fix_missing_c_header(error, context):
package = get_package_for_paths(
package = context.apt.get_package_for_paths(
[os.path.join("/usr/include", error.header)], regex=False
)
if package is None:
package = get_package_for_paths(
package = context.apt.get_package_for_paths(
[os.path.join("/usr/include", ".*", error.header)], regex=True
)
if package is None:
@ -550,11 +524,11 @@ def fix_missing_c_header(error, context):
def fix_missing_pkg_config(error, context):
package = get_package_for_paths(
package = context.apt.get_package_for_paths(
[os.path.join("/usr/lib/pkgconfig", error.module + ".pc")]
)
if package is None:
package = get_package_for_paths(
package = context.apt.get_package_for_paths(
[os.path.join("/usr/lib", ".*", "pkgconfig", error.module + ".pc")],
regex=True,
)
@ -564,21 +538,12 @@ def fix_missing_pkg_config(error, context):
def fix_missing_command(error, context):
if os.path.isabs(error.command):
paths = [error.command]
else:
paths = [
os.path.join(dirname, error.command) for dirname in ["/usr/bin", "/bin"]
]
package = get_package_for_paths(paths)
if package is None:
logging.info("No packages found that contain %r", paths)
return False
package = context.resolve_apt(BinaryRequirement(error.command))
return context.add_dependency(package)
def fix_missing_file(error, context):
package = get_package_for_paths([error.path])
package = context.apt.get_package_for_paths([error.path])
if package is None:
return False
return context.add_dependency(package)
@ -590,7 +555,7 @@ def fix_missing_sprockets_file(error, context):
else:
logging.warning("unable to handle content type %s", error.content_type)
return False
package = get_package_for_paths([path], regex=True)
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
return False
return context.add_dependency(package)
@ -619,7 +584,7 @@ def fix_missing_perl_file(error, context):
paths = [error.filename]
else:
paths = [os.path.join(inc, error.filename) for inc in error.inc]
package = get_package_for_paths(paths, regex=False)
package = context.apt.get_package_for_paths(paths, regex=False)
if package is None:
if getattr(error, "module", None):
logging.warning(
@ -635,17 +600,17 @@ def fix_missing_perl_file(error, context):
return context.add_dependency(package)
def get_package_for_node_package(node_package):
def get_package_for_node_package(apt, node_package):
paths = [
"/usr/share/nodejs/.*/node_modules/%s/package.json" % node_package,
"/usr/lib/nodejs/%s/package.json" % node_package,
"/usr/share/nodejs/%s/package.json" % node_package,
]
return get_package_for_paths(paths, regex=True)
return apt.get_package_for_paths(paths, regex=True)
def fix_missing_node_module(error, context):
package = get_package_for_node_package(error.module)
package = get_package_for_node_package(context.apt, error.module)
if package is None:
logging.warning("no node package found for %s.", error.module)
return False
@ -654,7 +619,7 @@ def fix_missing_node_module(error, context):
def fix_missing_dh_addon(error, context):
paths = [os.path.join("/usr/share/perl5", error.path)]
package = get_package_for_paths(paths)
package = context.apt.get_package_for_paths(paths)
if package is None:
logging.warning("no package for debhelper addon %s", error.name)
return False
@ -667,7 +632,7 @@ def retry_apt_failure(error, context):
def fix_missing_php_class(error, context):
path = "/usr/share/php/%s.php" % error.php_class.replace("\\", "/")
package = get_package_for_paths([path])
package = context.apt.get_package_for_paths([path])
if package is None:
logging.warning("no package for PHP class %s", error.php_class)
return False
@ -676,7 +641,7 @@ def fix_missing_php_class(error, context):
def fix_missing_jdk_file(error, context):
path = error.jdk_path + ".*/" + error.filename
package = get_package_for_paths([path], regex=True)
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
logging.warning(
"no package found for %s (JDK: %s) - regex %s",
@ -690,7 +655,7 @@ def fix_missing_jdk_file(error, context):
def fix_missing_vala_package(error, context):
path = "/usr/share/vala-[0-9.]+/vapi/%s.vapi" % error.package
package = get_package_for_paths([path], regex=True)
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
logging.warning("no file found for package %s - regex %s", error.package, path)
return False
@ -710,7 +675,7 @@ def fix_missing_xml_entity(error, context):
else:
return False
package = get_package_for_paths([search_path], regex=False)
package = context.apt.get_package_for_paths([search_path], regex=False)
if package is None:
return False
return context.add_dependency(package)
@ -723,7 +688,7 @@ def fix_missing_library(error, context):
os.path.join("/usr/lib/lib%s.a$" % error.library),
os.path.join("/usr/lib/.*/lib%s.a$" % error.library),
]
package = get_package_for_paths(paths, regex=True)
package = context.apt.get_package_for_paths(paths, regex=True)
if package is None:
logging.warning("no package for library %s", error.library)
return False
@ -737,7 +702,7 @@ def fix_missing_ruby_gem(error, context):
"specifications/%s-.*\\.gemspec" % error.gem
)
]
package = get_package_for_paths(paths, regex=True)
package = context.apt.get_package_for_paths(paths, regex=True)
if package is None:
logging.warning("no package for gem %s", error.gem)
return False
@ -746,7 +711,7 @@ def fix_missing_ruby_gem(error, context):
def fix_missing_ruby_file(error, context):
paths = [os.path.join("/usr/lib/ruby/vendor_ruby/%s.rb" % error.filename)]
package = get_package_for_paths(paths)
package = context.apt.get_package_for_paths(paths)
if package is not None:
return context.add_dependency(package)
paths = [
@ -755,7 +720,7 @@ def fix_missing_ruby_file(error, context):
"lib/%s.rb" % error.filename
)
]
package = get_package_for_paths(paths, regex=True)
package = context.apt.get_package_for_paths(paths, regex=True)
if package is not None:
return context.add_dependency(package)
@ -765,7 +730,7 @@ def fix_missing_ruby_file(error, context):
def fix_missing_r_package(error, context):
paths = [os.path.join("/usr/lib/R/site-library/.*/R/%s$" % error.package)]
package = get_package_for_paths(paths, regex=True)
package = context.apt.get_package_for_paths(paths, regex=True)
if package is None:
logging.warning("no package for R package %s", error.package)
return False
@ -781,7 +746,7 @@ def fix_missing_java_class(error, context):
logging.warning("unable to find classpath for %s", error.classname)
return False
logging.info("Classpath for %s: %r", error.classname, classpath)
package = get_package_for_paths(classpath)
package = context.apt.get_package_for_paths(classpath)
if package is None:
logging.warning("no package for files in %r", classpath)
return False
@ -849,7 +814,7 @@ def fix_missing_maven_artifacts(error, context):
"%s-%s.%s" % (artifact_id, version, kind),
)
]
package = get_package_for_paths(paths, regex=regex)
package = context.apt.get_package_for_paths(paths, regex=regex)
if package is None:
logging.warning("no package for artifact %s", artifact)
return False
@ -862,7 +827,7 @@ def install_gnome_common(error, context):
def install_gnome_common_dep(error, context):
if error.package == "glib-gettext":
package = get_package_for_paths(["/usr/bin/glib-gettextize"])
package = context.apt.get_package_for_paths(["/usr/bin/glib-gettextize"])
else:
package = None
if package is None:
@ -875,7 +840,7 @@ def install_gnome_common_dep(error, context):
def install_xfce_dep(error, context):
if error.package == "gtk-doc":
package = get_package_for_paths(["/usr/bin/gtkdocize"])
package = context.apt.get_package_for_paths(["/usr/bin/gtkdocize"])
else:
package = None
if package is None:
@ -947,7 +912,7 @@ def fix_missing_autoconf_macro(error, context):
except KeyError:
logging.info("No local m4 file found defining %s", error.macro)
return False
package = get_package_for_paths([path])
package = context.apt.get_package_for_paths([path])
if package is None:
logging.warning("no package for macro file %s", path)
return False
@ -960,7 +925,7 @@ def fix_missing_c_sharp_compiler(error, context):
def fix_missing_haskell_dependencies(error, context):
path = "/var/lib/ghc/package.conf.d/%s-.*.conf" % error.deps[0][0]
package = get_package_for_paths([path], regex=True)
package = context.apt.get_package_for_paths([path], regex=True)
if package is None:
logging.warning("no package for macro file %s", path)
return False
@ -1033,6 +998,7 @@ def resolve_error(error, context, fixers):
def build_incrementally(
local_tree,
apt,
suffix,
build_suite,
output_directory,
@ -1074,6 +1040,7 @@ def build_incrementally(
if e.context[0] == "build":
context = BuildDependencyContext(
local_tree,
apt,
subpath=subpath,
committer=committer,
update_changelog=update_changelog,
@ -1082,6 +1049,7 @@ def build_incrementally(
context = AutopkgtestDependencyContext(
e.context[1],
local_tree,
apt,
subpath=subpath,
committer=committer,
update_changelog=update_changelog,
@ -1154,9 +1122,12 @@ def main(argv=None):
args = parser.parse_args()
from breezy.workingtree import WorkingTree
apt = LocalAptManager()
tree = WorkingTree.open(".")
build_incrementally(
tree,
apt,
args.suffix,
args.suite,
args.output_directory,

2
ognibuild/dist.py
View file

@ -124,7 +124,7 @@ def create_dist_schroot(
subdir: Optional[str] = None,
) -> str:
from .buildsystem import detect_buildsystems
from .resolver import AptResolver
from .resolver.apt import AptResolver
if subdir is None:
subdir = "package"

14
ognibuild/fix_build.py
View file

@ -22,6 +22,7 @@ from buildlog_consultant.common import (
find_build_failure_description,
Problem,
MissingPerlModule,
MissingPythonDistribution,
MissingCommand,
)
@ -69,10 +70,16 @@ def fix_npm_missing_command(error, context):
return True
def fix_python_package_from_pip(error, context):
context.session.check_call(["pip", "install", error.distribution])
return True
GENERIC_INSTALL_FIXERS: List[
Tuple[Type[Problem], Callable[[Problem, DependencyContext], bool]]
] = [
(MissingPerlModule, fix_perl_module_from_cpan),
(MissingPythonDistribution, fix_python_package_from_pip),
(MissingCommand, fix_npm_missing_command),
]
@ -84,11 +91,12 @@ def run_with_build_fixer(session: Session, args: List[str]):
retcode, lines = run_with_tee(session, args)
if retcode == 0:
return
offset, line, error = find_build_failure_description(lines)
match, error = find_build_failure_description(lines)
if error is None:
logging.warning("Build failed with unidentified error. Giving up.")
if line is not None:
raise UnidentifiedError(retcode, args, lines, secondary=(offset, line))
if match is not None:
raise UnidentifiedError(
retcode, args, lines, secondary=(match.lineno, match.line))
raise UnidentifiedError(retcode, args, lines)
logging.info("Identified error: %r", error)

64
ognibuild/requirements.py Normal file
View file

@ -0,0 +1,64 @@
#!/usr/bin/python
# Copyright (C) 2019-2020 Jelmer Vernooij <jelmer@jelmer.uk>
# encoding: utf-8
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
from . import UpstreamRequirement
class PythonPackageRequirement(UpstreamRequirement):
package: str
def __init__(self, package):
super(PythonPackageRequirement, self).__init__('python-package')
self.package = package
class BinaryRequirement(UpstreamRequirement):
binary_name: str
def __init__(self, binary_name):
super(BinaryRequirement, self).__init__('binary')
self.binary_name = binary_name
class PerlModuleRequirement(UpstreamRequirement):
module: str
def __init__(self, module):
super(PerlModuleRequirement, self).__init__('perl-module')
self.module = module
class NodePackageRequirement(UpstreamRequirement):
package: str
def __init__(self, package):
super(NodePackageRequirement, self).__init__('npm-package')
self.package = package
class CargoCrateRequirement(UpstreamRequirement):
crate: str
def __init__(self, crate):
super(CargoCrateRequirement, self).__init__('cargo-crate')
self.crate = crate

42
ognibuild/resolver.py → ognibuild/resolver/__init__.py
View file

@ -17,11 +17,13 @@
class MissingDependencies(Exception):
def __init__(self, reqs):
self.requirements = reqs
class Resolver(object):
def install(self, requirements):
raise NotImplementedError(self.install)
@ -29,43 +31,6 @@ class Resolver(object):
raise NotImplementedError(self.explain)
class AptResolver(Resolver):
def __init__(self, apt):
self.apt = apt
@classmethod
def from_session(cls, session):
from .apt import AptManager
return cls(AptManager(session))
def install(self, requirements):
missing = []
for req in requirements:
pps = list(self._possible_paths(req))
if not pps or not any(self.apt.session.exists(p) for p in pps):
missing.append(req)
if missing:
self.apt.install(list(self.resolve(missing)))
def explain(self, requirements):
raise NotImplementedError(self.explain)
def _possible_paths(self, req):
if req.family == "binary":
yield "/usr/bin/%s" % req.name
else:
return
def resolve(self, requirements):
for req in requirements:
if req.family == "python3":
yield "python3-%s" % req.name
else:
list(self._possible_paths(req))
raise NotImplementedError
class NativeResolver(Resolver):
def __init__(self, session):
self.session = session
@ -94,7 +59,8 @@ class ExplainResolver(Resolver):
class AutoResolver(Resolver):
"""Automatically find out the most appropriate way to instal dependencies."""
"""Automatically find out the most appropriate way to install dependencies.
"""
def __init__(self, session):
self.session = session

84
ognibuild/resolver/apt.py Normal file
View file

@ -0,0 +1,84 @@
#!/usr/bin/python3
# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
import posixpath
from ..apt import AptManager
from . import Resolver
from ..requirements import (
BinaryRequirement,
PythonPackageRequirement,
)
class NoAptPackage(Exception):
"""No apt package."""
def resolve_binary_req(apt_mgr, req):
if posixpath.isabs(req.binary_name):
paths = [req.binary_name]
else:
paths = [
posixpath.join(dirname, req.binary_name)
for dirname in ["/usr/bin", "/bin"]
]
return apt_mgr.get_package_for_paths(paths)
APT_REQUIREMENT_RESOLVERS = [
(BinaryRequirement, resolve_binary_req),
]
class AptResolver(Resolver):
def __init__(self, apt):
self.apt = apt
@classmethod
def from_session(cls, session):
return cls(AptManager(session))
def install(self, requirements):
missing = []
for req in requirements:
try:
pps = list(req.possible_paths())
except NotImplementedError:
missing.append(req)
else:
if not pps or not any(self.apt.session.exists(p) for p in pps):
missing.append(req)
if missing:
self.apt.install(list(self.resolve(missing)))
def explain(self, requirements):
raise NotImplementedError(self.explain)
def resolve(self, requirements):
for req in requirements:
for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
if isinstance(req, rr_class):
package_name = rr_fn(self.apt, req)
if package_name is None:
raise NoAptPackage()
yield package_name
break
else:
raise NotImplementedError

3
ognibuild/tests/test_debian_fix_build.py
View file

@ -31,6 +31,7 @@ from buildlog_consultant.common import (
MissingValaPackage,
)
from ..debian import apt
from ..debian.apt import LocalAptManager
from ..debian.fix_build import (
resolve_error,
VERSIONED_PACKAGE_FIXERS,
@ -88,8 +89,10 @@ blah (0.1) UNRELEASED; urgency=medium
yield pkg
def resolve(self, error, context=("build",)):
apt = LocalAptManager()
context = BuildDependencyContext(
self.tree,
apt,
subpath="",
committer="Janitor <janitor@jelmer.uk>",
update_changelog=True,