billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix all tests.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-02-27 16:05:36 +00:00
parent 6b30479b97
commit dd14deb00d
17 changed files with 280 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

3
ognibuild/__init__.py
View file

@ -67,3 +67,6 @@ class UpstreamOutput(object):
def __init__(self, family, name): def __init__(self, family, name):
self.family = family self.family = family
self.name = name self.name = name
def __repr__(self):
return "%s(%r, %r)" % (type(self).__name__, self.family, self.name)

56
ognibuild/__main__.py
View file

@ -21,8 +21,7 @@ import sys
from . import UnidentifiedError from . import UnidentifiedError
from .buildsystem import NoBuildToolsFound, detect_buildsystems from .buildsystem import NoBuildToolsFound, detect_buildsystems
from .resolver import ( from .resolver import (
ExplainResolver, auto_resolver,
AutoResolver,
native_resolvers, native_resolvers,
MissingDependencies, MissingDependencies,
) )
@ -56,6 +55,11 @@ STAGE_MAP = {
"clean": [], "clean": [],
} }
def determine_fixers(session, resolver):
from .buildlog import UpstreamRequirementFixer
from .resolver.apt import AptResolver
return [UpstreamRequirementFixer(resolver)]
def main(): # noqa: C901 def main(): # noqa: C901
import argparse import argparse
@ -67,12 +71,17 @@ def main(): # noqa: C901
parser.add_argument("--schroot", type=str, help="schroot to run in.") parser.add_argument("--schroot", type=str, help="schroot to run in.")
parser.add_argument( parser.add_argument(
"--resolve", "--resolve",
choices=["explain", "apt", "native"], choices=["apt", "native", "auto"],
default="apt", default="auto",
help="What to do about missing dependencies", help="What to do about missing dependencies",
) )
parser.add_argument(
"--explain",
action='store_true',
help="Explain what needs to be done rather than making changes")
parser.add_argument( parser.add_argument(
"--ignore-declared-dependencies", "--ignore-declared-dependencies",
"--optimistic",
action="store_true", action="store_true",
help="Ignore declared dependencies, follow build errors only", help="Ignore declared dependencies, follow build errors only",
) )
@ -109,56 +118,53 @@ def main(): # noqa: C901
with session: with session:
if args.resolve == "apt": if args.resolve == "apt":
resolver = AptResolver.from_session(session) resolver = AptResolver.from_session(session)
elif args.resolve == "explain":
resolver = ExplainResolver.from_session(session)
elif args.resolve == "native": elif args.resolve == "native":
resolver = native_resolvers(session) resolver = native_resolvers(session)
elif args.resolver == "auto": elif args.resolve == "auto":
resolver = AutoResolver.from_session(session) resolver = auto_resolver(session)
logging.info('Using requirement resolver: %s', resolver)
os.chdir(args.directory) os.chdir(args.directory)
try: try:
bss = list(detect_buildsystems(args.directory)) bss = list(detect_buildsystems(args.directory))
logging.info('Detected buildsystems: %r', bss) logging.info('Detected buildsystems: %r', bss)
if not args.ignore_declared_dependencies: if not args.ignore_declared_dependencies and not args.explain:
stages = STAGE_MAP[args.subcommand] stages = STAGE_MAP[args.subcommand]
if stages: if stages:
for bs in bss: for bs in bss:
install_necessary_declared_requirements(resolver, bs, stages) install_necessary_declared_requirements(resolver, bs, stages)
fixers = determine_fixers(session, resolver)
if args.subcommand == "dist": if args.subcommand == "dist":
from .dist import run_dist from .dist import run_dist
run_dist(session=session, buildsystems=bss, resolver=resolver) run_dist(
session=session, buildsystems=bss, resolver=resolver,
fixers=fixers)
if args.subcommand == "build": if args.subcommand == "build":
from .build import run_build from .build import run_build
run_build(session, buildsystems=bss, resolver=resolver) run_build(
session, buildsystems=bss, resolver=resolver,
fixers=fixers)
if args.subcommand == "clean": if args.subcommand == "clean":
from .clean import run_clean from .clean import run_clean
run_clean(session, buildsystems=bss, resolver=resolver) run_clean(
session, buildsystems=bss, resolver=resolver,
fixers=fixers)
if args.subcommand == "install": if args.subcommand == "install":
from .install import run_install from .install import run_install
run_install( run_install(
session, buildsystems=bss, resolver=resolver, session, buildsystems=bss, resolver=resolver,
user=args.user) fixers=fixers, user=args.user)
if args.subcommand == "test": if args.subcommand == "test":
from .test import run_test from .test import run_test
run_test(session, buildsystems=bss, resolver=resolver) run_test(session, buildsystems=bss, resolver=resolver,
fixers=fixers)
if args.subcommand == "info": if args.subcommand == "info":
from .info import run_info from .info import run_info
run_info(session, buildsystems=bss, resolver=resolver) run_info(session, buildsystems=bss)
except UnidentifiedError: except UnidentifiedError:
return 1 return 1
except NoBuildToolsFound: except NoBuildToolsFound:
logging.info("No build tools found.") logging.info("No build tools found.")
return 1 return 1
except MissingDependencies as e:
for req in e.requirements:
logging.info("Missing dependency (%s:%s)",
req.family, req.package)
for resolver in [
AptResolver.from_session(session),
native_resolvers(session),
]:
logging.info(" %s", resolver.explain([req]))
return 2
return 0 return 0

4
ognibuild/build.py
View file

@ -18,13 +18,13 @@
from .buildsystem import NoBuildToolsFound from .buildsystem import NoBuildToolsFound
def run_build(session, buildsystems, resolver): def run_build(session, buildsystems, resolver, fixers):
# Some things want to write to the user's home directory, # Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache # e.g. pip caches in ~/.cache
session.create_home() session.create_home()
for buildsystem in buildsystems: for buildsystem in buildsystems:
buildsystem.build(session, resolver) buildsystem.build(session, resolver, fixers)
return return
raise NoBuildToolsFound() raise NoBuildToolsFound()

2
ognibuild/buildlog.py
View file

@ -189,4 +189,6 @@ class UpstreamRequirementFixer(BuildFixer):
return False return False
package = self.resolver.resolve(req) package = self.resolver.resolve(req)
if package is None:
return False
return context.add_dependency(package) return context.add_dependency(package)

136
ognibuild/buildsystem.py
View file

@ -31,7 +31,7 @@ from .requirements import (
NodePackageRequirement, NodePackageRequirement,
CargoCrateRequirement, CargoCrateRequirement,
) )
from .fix_build import run_with_build_fixer from .fix_build import run_with_build_fixers
class NoBuildToolsFound(Exception): class NoBuildToolsFound(Exception):
@ -51,19 +51,19 @@ class BuildSystem(object):
name: str name: str
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
raise NotImplementedError(self.dist) raise NotImplementedError(self.dist)
def test(self, session, resolver): def test(self, session, resolver, fixers):
raise NotImplementedError(self.test) raise NotImplementedError(self.test)
def build(self, session, resolver): def build(self, session, resolver, fixers):
raise NotImplementedError(self.build) raise NotImplementedError(self.build)
def clean(self, session, resolver): def clean(self, session, resolver, fixers):
raise NotImplementedError(self.clean) raise NotImplementedError(self.clean)
def install(self, session, resolver, install_target): def install(self, session, resolver, fixers, install_target):
raise NotImplementedError(self.install) raise NotImplementedError(self.install)
def get_declared_dependencies(self): def get_declared_dependencies(self):
@ -83,25 +83,25 @@ class Pear(BuildSystem):
def setup(self, resolver): def setup(self, resolver):
resolver.install([BinaryRequirement("pear")]) resolver.install([BinaryRequirement("pear")])
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
run_with_build_fixer(session, ["pear", "package"]) run_with_build_fixers(session, ["pear", "package"], fixers)
def test(self, session, resolver): def test(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
run_with_build_fixer(session, ["pear", "run-tests"]) run_with_build_fixers(session, ["pear", "run-tests"], fixers)
def build(self, session, resolver): def build(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
run_with_build_fixer(session, ["pear", "build", self.path]) run_with_build_fixers(session, ["pear", "build", self.path], fixers)
def clean(self, session, resolver): def clean(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
# TODO # TODO
def install(self, session, resolver, install_target): def install(self, session, resolver, fixers, install_target):
self.setup(resolver) self.setup(resolver)
run_with_build_fixer(session, ["pear", "install", self.path]) run_with_build_fixers(session, ["pear", "install", self.path], fixers)
class SetupPy(BuildSystem): class SetupPy(BuildSystem):
@ -143,41 +143,40 @@ class SetupPy(BuildSystem):
# TODO(jelmer): Install setup_requires # TODO(jelmer): Install setup_requires
def test(self, session, resolver): def test(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
self._run_setup(session, resolver, ["test"]) self._run_setup(session, resolver, ["test"], fixers)
def build(self, session, resolver): def build(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
self._run_setup(session, resolver, ["build"]) self._run_setup(session, resolver, ["build"], fixers)
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
self._run_setup(session, resolver, ["sdist"]) self._run_setup(session, resolver, ["sdist"], fixers)
def clean(self, session, resolver): def clean(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
self._run_setup(session, resolver, ["clean"]) self._run_setup(session, resolver, ["clean"], fixers)
def install(self, session, resolver, install_target): def install(self, session, resolver, fixers, install_target):
self.setup(resolver) self.setup(resolver)
extra_args = [] extra_args = []
if install_target.user: if install_target.user:
extra_args.append('--user') extra_args.append('--user')
self._run_setup(session, resolver, ["install"] + extra_args) self._run_setup(session, resolver, ["install"] + extra_args, fixers)
def _run_setup(self, session, resolver, args): def _run_setup(self, session, resolver, args, fixers):
interpreter = shebang_binary("setup.py") interpreter = shebang_binary("setup.py")
if interpreter is not None: if interpreter is not None:
if interpreter in ("python3", "python2", "python"): resolver.install([BinaryRequirement(interpreter)])
resolver.install([BinaryRequirement(interpreter)]) run_with_build_fixers(session, ["./setup.py"] + args, fixers)
else:
raise ValueError("Unknown interpreter %r" % interpreter)
run_with_build_fixer(session, ["./setup.py"] + args)
else: else:
# Just assume it's Python 3 # Just assume it's Python 3
resolver.install([BinaryRequirement("python3")]) resolver.install([BinaryRequirement("python3")])
run_with_build_fixer(session, ["python3", "./setup.py"] + args) run_with_build_fixers(
session, ["python3", "./setup.py"] + args,
fixers)
def get_declared_dependencies(self): def get_declared_dependencies(self):
for require in self.result.get_requires(): for require in self.result.get_requires():
@ -215,7 +214,7 @@ class PyProject(BuildSystem):
with open(self.path, "r") as pf: with open(self.path, "r") as pf:
return toml.load(pf) return toml.load(pf)
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
if "poetry" in self.pyproject.get("tool", []): if "poetry" in self.pyproject.get("tool", []):
logging.info( logging.info(
"Found pyproject.toml with poetry section, " "assuming poetry project." "Found pyproject.toml with poetry section, " "assuming poetry project."
@ -247,7 +246,7 @@ class SetupCfg(BuildSystem):
] ]
) )
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
session.check_call(["python3", "-m", "pep517.build", "-s", "."]) session.check_call(["python3", "-m", "pep517.build", "-s", "."])
@ -271,9 +270,9 @@ class Npm(BuildSystem):
def setup(self, resolver): def setup(self, resolver):
resolver.install([BinaryRequirement("npm")]) resolver.install([BinaryRequirement("npm")])
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
run_with_build_fixer(session, ["npm", "pack"]) run_with_build_fixers(session, ["npm", "pack"], fixers)
class Waf(BuildSystem): class Waf(BuildSystem):
@ -286,9 +285,9 @@ class Waf(BuildSystem):
def setup(self, resolver): def setup(self, resolver):
resolver.install([BinaryRequirement("python3")]) resolver.install([BinaryRequirement("python3")])
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
run_with_build_fixer(session, ["./waf", "dist"]) run_with_build_fixers(session, ["./waf", "dist"], fixers)
class Gem(BuildSystem): class Gem(BuildSystem):
@ -301,14 +300,14 @@ class Gem(BuildSystem):
def setup(self, resolver): def setup(self, resolver):
resolver.install([BinaryRequirement("gem2deb")]) resolver.install([BinaryRequirement("gem2deb")])
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
gemfiles = [ gemfiles = [
entry.name for entry in session.scandir(".") if entry.name.endswith(".gem") entry.name for entry in session.scandir(".") if entry.name.endswith(".gem")
] ]
if len(gemfiles) > 1: if len(gemfiles) > 1:
logging.warning("More than one gemfile. Trying the first?") logging.warning("More than one gemfile. Trying the first?")
run_with_build_fixer(session, ["gem2tgz", gemfiles[0]]) run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers)
class DistInkt(BuildSystem): class DistInkt(BuildSystem):
@ -340,15 +339,16 @@ class DistInkt(BuildSystem):
] ]
) )
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
if self.name == "dist-inkt": if self.name == "dist-inkt":
resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) resolver.install([PerlModuleRequirement(self.dist_inkt_class)])
run_with_build_fixer(session, ["distinkt-dist"]) run_with_build_fixers(session, ["distinkt-dist"], fixers)
else: else:
# Default to invoking Dist::Zilla # Default to invoking Dist::Zilla
resolver.install([PerlModuleRequirement("Dist::Zilla")]) resolver.install([PerlModuleRequirement("Dist::Zilla")])
run_with_build_fixer(session, ["dzil", "build", "--in", ".."]) run_with_build_fixers(
session, ["dzil", "build", "--in", ".."], fixers)
class Make(BuildSystem): class Make(BuildSystem):
@ -358,26 +358,30 @@ class Make(BuildSystem):
def __repr__(self): def __repr__(self):
return "%s()" % type(self).__name__ return "%s()" % type(self).__name__
def setup(self, session, resolver): def setup(self, session, resolver, fixers):
resolver.install([BinaryRequirement("make")]) resolver.install([BinaryRequirement("make")])
if session.exists("Makefile.PL") and not session.exists("Makefile"): if session.exists("Makefile.PL") and not session.exists("Makefile"):
resolver.install([BinaryRequirement("perl")]) resolver.install([BinaryRequirement("perl")])
run_with_build_fixer(session, ["perl", "Makefile.PL"]) run_with_build_fixers(session, ["perl", "Makefile.PL"], fixers)
if not session.exists("Makefile") and not session.exists("configure"): if not session.exists("Makefile") and not session.exists("configure"):
if session.exists("autogen.sh"): if session.exists("autogen.sh"):
if shebang_binary("autogen.sh") is None: if shebang_binary("autogen.sh") is None:
run_with_build_fixer(session, ["/bin/sh", "./autogen.sh"]) run_with_build_fixers(
session, ["/bin/sh", "./autogen.sh"], fixers)
try: try:
run_with_build_fixer(session, ["./autogen.sh"]) run_with_build_fixers(
session, ["./autogen.sh"], fixers)
except UnidentifiedError as e: except UnidentifiedError as e:
if ( if (
"Gnulib not yet bootstrapped; " "Gnulib not yet bootstrapped; "
"run ./bootstrap instead.\n" in e.lines "run ./bootstrap instead.\n" in e.lines
): ):
run_with_build_fixer(session, ["./bootstrap"]) run_with_build_fixers(
run_with_build_fixer(session, ["./autogen.sh"]) session, ["./bootstrap"], fixers)
run_with_build_fixers(
session, ["./autogen.sh"], fixers)
else: else:
raise raise
@ -390,23 +394,23 @@ class Make(BuildSystem):
BinaryRequirement("libtoolize"), BinaryRequirement("libtoolize"),
] ]
) )
run_with_build_fixer(session, ["autoreconf", "-i"]) run_with_build_fixers(session, ["autoreconf", "-i"], fixers)
if not session.exists("Makefile") and session.exists("configure"): if not session.exists("Makefile") and session.exists("configure"):
session.check_call(["./configure"]) session.check_call(["./configure"])
def build(self, session, resolver): def build(self, session, resolver, fixers):
self.setup(session, resolver) self.setup(session, resolver)
run_with_build_fixer(session, ["make", "all"]) run_with_build_fixers(session, ["make", "all"], fixers)
def install(self, session, resolver, install_target): def install(self, session, resolver, fixers, install_target):
self.setup(session, resolver) self.setup(session, resolver)
run_with_build_fixer(session, ["make", "install"]) run_with_build_fixers(session, ["make", "install"], fixers)
def dist(self, session, resolver): def dist(self, session, resolver, fixers):
self.setup(session, resolver) self.setup(session, resolver)
try: try:
run_with_build_fixer(session, ["make", "dist"]) run_with_build_fixers(session, ["make", "dist"], fixers)
except UnidentifiedError as e: except UnidentifiedError as e:
if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines: if "make: *** No rule to make target 'dist'. Stop.\n" in e.lines:
pass pass
@ -416,17 +420,17 @@ class Make(BuildSystem):
"Reconfigure the source tree " "Reconfigure the source tree "
"(via './config' or 'perl Configure'), please.\n" "(via './config' or 'perl Configure'), please.\n"
) in e.lines: ) in e.lines:
run_with_build_fixer(session, ["./config"]) run_with_build_fixers(session, ["./config"], fixers)
run_with_build_fixer(session, ["make", "dist"]) run_with_build_fixers(session, ["make", "dist"], fixers)
elif ( elif (
"Please try running 'make manifest' and then run " "Please try running 'make manifest' and then run "
"'make dist' again.\n" in e.lines "'make dist' again.\n" in e.lines
): ):
run_with_build_fixer(session, ["make", "manifest"]) run_with_build_fixers(session, ["make", "manifest"], fixers)
run_with_build_fixer(session, ["make", "dist"]) run_with_build_fixers(session, ["make", "dist"], fixers)
elif "Please run ./configure first\n" in e.lines: elif "Please run ./configure first\n" in e.lines:
run_with_build_fixer(session, ["./configure"]) run_with_build_fixers(session, ["./configure"], fixers)
run_with_build_fixer(session, ["make", "dist"]) run_with_build_fixers(session, ["make", "dist"], fixers)
elif any( elif any(
[ [
re.match( re.match(
@ -437,8 +441,8 @@ class Make(BuildSystem):
for line in e.lines for line in e.lines
] ]
): ):
run_with_build_fixer(session, ["./configure"]) run_with_build_fixers(session, ["./configure"], fixers)
run_with_build_fixer(session, ["make", "dist"]) run_with_build_fixers(session, ["make", "dist"], fixers)
elif any( elif any(
[ [
re.match( re.match(
@ -449,8 +453,8 @@ class Make(BuildSystem):
for line in e.lines for line in e.lines
] ]
): ):
run_with_build_fixer(session, ["make", "manifest"]) run_with_build_fixers(session, ["make", "manifest"], fixers)
run_with_build_fixer(session, ["make", "dist"]) run_with_build_fixers(session, ["make", "dist"], fixers)
else: else:
raise raise
else: else:

4
ognibuild/clean.py
View file

@ -18,13 +18,13 @@
from .buildsystem import NoBuildToolsFound from .buildsystem import NoBuildToolsFound
def run_clean(session, buildsystems, resolver): def run_clean(session, buildsystems, resolver, fixers):
# Some things want to write to the user's home directory, # Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache # e.g. pip caches in ~/.cache
session.create_home() session.create_home()
for buildsystem in buildsystems: for buildsystem in buildsystems:
buildsystem.clean(session, resolver) buildsystem.clean(session, resolver, fixers)
return return
raise NoBuildToolsFound() raise NoBuildToolsFound()

118
ognibuild/debian/fix_build.py
View file

@ -36,7 +36,6 @@ from breezy.tree import Tree
from debmutate.control import ( from debmutate.control import (
ensure_some_version, ensure_some_version,
ensure_minimum_version, ensure_minimum_version,
pg_buildext_updatecontrol,
ControlEditor, ControlEditor,
) )
from debmutate.debhelper import ( from debmutate.debhelper import (
@ -82,7 +81,7 @@ from buildlog_consultant.sbuild import (
from ..fix_build import BuildFixer, resolve_error, DependencyContext from ..fix_build import BuildFixer, resolve_error, DependencyContext
from ..buildlog import UpstreamRequirementFixer from ..buildlog import UpstreamRequirementFixer
from ..resolver.apt import ( from ..resolver.apt import (
NoAptPackage, AptRequirement,
get_package_for_python_module, get_package_for_python_module,
) )
from .build import attempt_build, DEFAULT_BUILDER from .build import attempt_build, DEFAULT_BUILDER
@ -99,11 +98,11 @@ class CircularDependency(Exception):
class BuildDependencyContext(DependencyContext): class BuildDependencyContext(DependencyContext):
def add_dependency(self, package: str, minimum_version: Optional[Version] = None):
def add_dependency(self, requirement: AptRequirement):
return add_build_dependency( return add_build_dependency(
self.tree, self.tree,
package, requirement,
minimum_version=minimum_version,
committer=self.committer, committer=self.committer,
subpath=self.subpath, subpath=self.subpath,
update_changelog=self.update_changelog, update_changelog=self.update_changelog,
@ -119,12 +118,11 @@ class AutopkgtestDependencyContext(DependencyContext):
tree, apt, subpath, committer, update_changelog tree, apt, subpath, committer, update_changelog
) )
def add_dependency(self, package, minimum_version=None): def add_dependency(self, requirement):
return add_test_dependency( return add_test_dependency(
self.tree, self.tree,
self.testname, self.testname,
package, requirement,
minimum_version=minimum_version,
committer=self.committer, committer=self.committer,
subpath=self.subpath, subpath=self.subpath,
update_changelog=self.update_changelog, update_changelog=self.update_changelog,
@ -133,37 +131,38 @@ class AutopkgtestDependencyContext(DependencyContext):
def add_build_dependency( def add_build_dependency(
tree: Tree, tree: Tree,
package: str, requirement: AptRequirement,
minimum_version: Optional[Version] = None,
committer: Optional[str] = None, committer: Optional[str] = None,
subpath: str = "", subpath: str = "",
update_changelog: bool = True, update_changelog: bool = True,
): ):
if not isinstance(package, str): if not isinstance(requirement, AptRequirement):
raise TypeError(package) raise TypeError(requirement)
control_path = os.path.join(tree.abspath(subpath), "debian/control") control_path = os.path.join(tree.abspath(subpath), "debian/control")
try: try:
with ControlEditor(path=control_path) as updater: with ControlEditor(path=control_path) as updater:
for binary in updater.binaries: for binary in updater.binaries:
if binary["Package"] == package: if binary["Package"] == requirement.package:
raise CircularDependency(package) raise CircularDependency(requirement.package)
if minimum_version: if requirement.minimum_version:
updater.source["Build-Depends"] = ensure_minimum_version( updater.source["Build-Depends"] = ensure_minimum_version(
updater.source.get("Build-Depends", ""), package, minimum_version updater.source.get("Build-Depends", ""),
requirement.package, requirement.minimum_version
) )
else: else:
updater.source["Build-Depends"] = ensure_some_version( updater.source["Build-Depends"] = ensure_some_version(
updater.source.get("Build-Depends", ""), package updater.source.get("Build-Depends", ""),
requirement.package
) )
except FormattingUnpreservable as e: except FormattingUnpreservable as e:
logging.info("Unable to edit %s in a way that preserves formatting.", e.path) logging.info("Unable to edit %s in a way that preserves formatting.", e.path)
return False return False
if minimum_version: if requirement.minimum_version:
desc = "%s (>= %s)" % (package, minimum_version) desc = "%s (>= %s)" % (requirement.package, requirement.minimum_version)
else: else:
desc = package desc = requirement.package
if not updater.changed: if not updater.changed:
logging.info("Giving up; dependency %s was already present.", desc) logging.info("Giving up; dependency %s was already present.", desc)
@ -182,14 +181,13 @@ def add_build_dependency(
def add_test_dependency( def add_test_dependency(
tree, tree,
testname, testname,
package, requirement,
minimum_version=None,
committer=None, committer=None,
subpath="", subpath="",
update_changelog=True, update_changelog=True,
): ):
if not isinstance(package, str): if not isinstance(requirement, AptRequirement):
raise TypeError(package) raise TypeError(requirement)
tests_control_path = os.path.join(tree.abspath(subpath), "debian/tests/control") tests_control_path = os.path.join(tree.abspath(subpath), "debian/tests/control")
@ -204,13 +202,14 @@ def add_test_dependency(
command_counter += 1 command_counter += 1
if name != testname: if name != testname:
continue continue
if minimum_version: if requirement.minimum_version:
control["Depends"] = ensure_minimum_version( control["Depends"] = ensure_minimum_version(
control.get("Depends", ""), package, minimum_version control.get("Depends", ""),
requirement.package, requirement.minimum_version
) )
else: else:
control["Depends"] = ensure_some_version( control["Depends"] = ensure_some_version(
control.get("Depends", ""), package control.get("Depends", ""), requirement.package
) )
except FormattingUnpreservable as e: except FormattingUnpreservable as e:
logging.info("Unable to edit %s in a way that preserves formatting.", e.path) logging.info("Unable to edit %s in a way that preserves formatting.", e.path)
@ -218,10 +217,11 @@ def add_test_dependency(
if not updater.changed: if not updater.changed:
return False return False
if minimum_version: if requirement.minimum_version:
desc = "%s (>= %s)" % (package, minimum_version) desc = "%s (>= %s)" % (
requirement.package, requirement.minimum_version)
else: else:
desc = package desc = requirement.package
logging.info("Adding dependency to test %s: %s", testname, desc) logging.info("Adding dependency to test %s: %s", testname, desc)
return commit_debian_changes( return commit_debian_changes(
@ -333,7 +333,9 @@ def fix_missing_python_distribution(error, context): # noqa: C901
for dep_pkg in extra_build_deps: for dep_pkg in extra_build_deps:
assert dep_pkg is not None assert dep_pkg is not None
if not context.add_dependency(dep_pkg, minimum_version=error.minimum_version): if not context.add_dependency(
AptRequirement(
dep_pkg.package, minimum_version=error.minimum_version)):
return False return False
return True return True
@ -345,9 +347,9 @@ def fix_missing_python_module(error, context):
targeted = set() targeted = set()
default = not targeted default = not targeted
pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy") pypy_pkg = get_package_for_python_module(context.apt, error.module, "pypy", None)
py2_pkg = get_package_for_python_module(context.apt, error.module, "python2") py2_pkg = get_package_for_python_module(context.apt, error.module, "python2", None)
py3_pkg = get_package_for_python_module(context.apt, error.module, "python3") py3_pkg = get_package_for_python_module(context.apt, error.module, "python3", None)
extra_build_deps = [] extra_build_deps = []
if error.python_version == 2: if error.python_version == 2:
@ -379,7 +381,8 @@ def fix_missing_python_module(error, context):
for dep_pkg in extra_build_deps: for dep_pkg in extra_build_deps:
assert dep_pkg is not None assert dep_pkg is not None
if not context.add_dependency(dep_pkg, error.minimum_version): if not context.add_dependency(
AptRequirement(dep_pkg.package, error.minimum_version)):
return False return False
return True return True
@ -402,7 +405,7 @@ def enable_dh_autoreconf(context):
return dh_invoke_add_with(line, b"autoreconf") return dh_invoke_add_with(line, b"autoreconf")
if update_rules(command_line_cb=add_with_autoreconf): if update_rules(command_line_cb=add_with_autoreconf):
return context.add_dependency("dh-autoreconf") return context.add_dependency(AptRequirement("dh-autoreconf"))
return False return False
@ -453,17 +456,27 @@ def fix_missing_config_status_input(error, context):
return True return True
def run_pgbuildext_updatecontrol(error, context): class PgBuildExtOutOfDateControlFixer(BuildFixer):
logging.info("Running 'pg_buildext updatecontrol'")
# TODO(jelmer): run in the schroot def __init__(self, session):
pg_buildext_updatecontrol(context.tree.abspath(context.subpath)) self.session = session
return commit_debian_changes(
context.tree, def can_fix(self, problem):
context.subpath, return isinstance(problem, NeedPgBuildExtUpdateControl)
"Run 'pgbuildext updatecontrol'.",
committer=context.committer, def _fix(self, problem, context):
update_changelog=False, return self._fn(problem, context)
)
def _fix(self, error, context):
logging.info("Running 'pg_buildext updatecontrol'")
self.session.check_call(["pg_buildext", "updatecontrol"])
return commit_debian_changes(
context.tree,
context.subpath,
"Run 'pgbuildext updatecontrol'.",
committer=context.committer,
update_changelog=False,
)
def fix_missing_makefile_pl(error, context): def fix_missing_makefile_pl(error, context):
@ -490,10 +503,9 @@ class SimpleBuildFixer(BuildFixer):
return self._fn(problem, context) return self._fn(problem, context)
def versioned_package_fixers(): def versioned_package_fixers(session):
return [ return [
SimpleBuildFixer( PgBuildExtOutOfDateControlFixer(session),
NeedPgBuildExtUpdateControl, run_pgbuildext_updatecontrol),
SimpleBuildFixer(MissingConfigure, fix_missing_configure), SimpleBuildFixer(MissingConfigure, fix_missing_configure),
SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input), SimpleBuildFixer(MissingAutomakeInput, fix_missing_automake_input),
SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input), SimpleBuildFixer(MissingConfigStatusInput, fix_missing_config_status_input),
@ -527,6 +539,8 @@ def build_incrementally(
update_changelog=True, update_changelog=True,
): ):
fixed_errors = [] fixed_errors = []
fixers = versioned_package_fixers(apt.session) + apt_fixers(apt)
logging.info('Using fixers: %r', fixers)
while True: while True:
try: try:
return attempt_build( return attempt_build(
@ -574,9 +588,7 @@ def build_incrementally(
logging.warning("unable to install for context %r", e.phase) logging.warning("unable to install for context %r", e.phase)
raise raise
try: try:
if not resolve_error( if not resolve_error(e.error, context, fixers):
e.error, context, versioned_package_fixers() + apt_fixers(apt)
):
logging.warning("Failed to resolve error %r. Giving up.", e.error) logging.warning("Failed to resolve error %r. Giving up.", e.error)
raise raise
except GeneratedFile: except GeneratedFile:

4
ognibuild/dist.py
View file

@ -62,13 +62,13 @@ class DistNoTarball(Exception):
"""Dist operation did not create a tarball.""" """Dist operation did not create a tarball."""
def run_dist(session, buildsystems, resolver): def run_dist(session, buildsystems, resolver, fixers):
# Some things want to write to the user's home directory, # Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache # e.g. pip caches in ~/.cache
session.create_home() session.create_home()
for buildsystem in buildsystems: for buildsystem in buildsystems:
buildsystem.dist(session, resolver) buildsystem.dist(session, resolver, fixers)
return return
raise NoBuildToolsFound() raise NoBuildToolsFound()

19
ognibuild/fix_build.py
View file

@ -79,23 +79,8 @@ class SchrootDependencyContext(DependencyContext):
return True return True
def generic_install_fixers(session): def run_with_build_fixers(
from .buildlog import UpstreamRequirementFixer session: Session, args: List[str], fixers: List[BuildFixer]):
from .resolver import CPANResolver, PypiResolver, NpmResolver
return [
UpstreamRequirementFixer(CPANResolver(session)),
UpstreamRequirementFixer(PypiResolver(session)),
UpstreamRequirementFixer(NpmResolver(session)),
]
def run_with_build_fixer(
session: Session, args: List[str],
fixers: Optional[List[BuildFixer]] = None):
if fixers is None:
from .debian.fix_build import apt_fixers
from .resolver.apt import AptResolver
fixers = generic_install_fixers(session) + apt_fixers(AptResolver.from_session(session))
logging.info("Running %r", args) logging.info("Running %r", args)
fixed_errors = [] fixed_errors = []
while True: while True:

2
ognibuild/info.py
View file

@ -18,7 +18,7 @@
from .buildsystem import NoBuildToolsFound, InstallTarget from .buildsystem import NoBuildToolsFound, InstallTarget
def run_info(session, buildsystems, resolver): def run_info(session, buildsystems):
for buildsystem in buildsystems: for buildsystem in buildsystems:
print('%r:' % buildsystem) print('%r:' % buildsystem)
deps = {} deps = {}

4
ognibuild/install.py
View file

@ -18,7 +18,7 @@
from .buildsystem import NoBuildToolsFound, InstallTarget from .buildsystem import NoBuildToolsFound, InstallTarget
def run_install(session, buildsystems, resolver, user: bool = False): def run_install(session, buildsystems, resolver, fixers, user: bool = False):
# Some things want to write to the user's home directory, # Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache # e.g. pip caches in ~/.cache
session.create_home() session.create_home()
@ -27,7 +27,7 @@ def run_install(session, buildsystems, resolver, user: bool = False):
install_target.user = user install_target.user = user
for buildsystem in buildsystems: for buildsystem in buildsystems:
buildsystem.install(session, resolver, install_target) buildsystem.install(session, resolver, fixers, install_target)
return return
raise NoBuildToolsFound() raise NoBuildToolsFound()

79
ognibuild/resolver/__init__.py
View file

@ -34,11 +34,14 @@ class Resolver(object):
raise NotImplementedError(self.met) raise NotImplementedError(self.met)
class CPANResolver(object): class CPANResolver(Resolver):
def __init__(self, session): def __init__(self, session):
self.session = session self.session = session
def __str__(self):
return "cpan"
def install(self, requirements): def install(self, requirements):
from ..requirements import PerlModuleRequirement from ..requirements import PerlModuleRequirement
missing = [] missing = []
@ -61,11 +64,42 @@ class CPANResolver(object):
raise NotImplementedError(self.met) raise NotImplementedError(self.met)
class PypiResolver(object): class CargoResolver(Resolver):
def __init__(self, session): def __init__(self, session):
self.session = session self.session = session
def __str__(self):
return "cargo"
def install(self, requirements):
from ..requirements import CargoCrateRequirement
missing = []
for requirement in requirements:
if not isinstance(requirement, CargoCrateRequirement):
missing.append(requirement)
continue
self.session.check_call(
["cargo", "install", requirement.crate],
user="root")
if missing:
raise MissingDependencies(missing)
def explain(self, requirements):
raise NotImplementedError(self.explain)
def met(self, requirement):
raise NotImplementedError(self.met)
class PypiResolver(Resolver):
def __init__(self, session):
self.session = session
def __str__(self):
return "pypi"
def install(self, requirements): def install(self, requirements):
from ..requirements import PythonPackageRequirement from ..requirements import PythonPackageRequirement
missing = [] missing = []
@ -89,11 +123,14 @@ NPM_COMMAND_PACKAGES = {
} }
class NpmResolver(object): class NpmResolver(Resolver):
def __init__(self, session): def __init__(self, session):
self.session = session self.session = session
def __str__(self):
return "npm"
def install(self, requirements): def install(self, requirements):
from ..requirements import NodePackageRequirement from ..requirements import NodePackageRequirement
missing = [] missing = []
@ -121,6 +158,9 @@ class StackedResolver(Resolver):
def __init__(self, subs): def __init__(self, subs):
self.subs = subs self.subs = subs
def __str__(self):
return "[" + ", ".join(map(str, self.subs)) + "]"
def install(self, requirements): def install(self, requirements):
for sub in self.subs: for sub in self.subs:
try: try:
@ -135,7 +175,8 @@ def native_resolvers(session):
return StackedResolver([ return StackedResolver([
CPANResolver(session), CPANResolver(session),
PypiResolver(session), PypiResolver(session),
NpmResolver(session)]) NpmResolver(session),
CargoResolver(session)])
class ExplainResolver(Resolver): class ExplainResolver(Resolver):
@ -150,19 +191,17 @@ class ExplainResolver(Resolver):
raise MissingDependencies(requirements) raise MissingDependencies(requirements)
class AutoResolver(Resolver): def auto_resolver(session):
"""Automatically find out the most appropriate way to install dependencies. # TODO(jelmer): if session is SchrootSession or if we're root, use apt
""" from .apt import AptResolver
from ..session.schroot import SchrootSession
def __init__(self, session): user = session.check_output(['echo', '$USER']).decode().strip()
self.session = session resolvers = []
if isinstance(session, SchrootSession) or user == 'root':
@classmethod resolvers.append(AptResolver.from_session(session))
def from_session(cls, session): resolvers.extend([
return cls(session) CPANResolver(session),
PypiResolver(session),
def install(self, requirements): NpmResolver(session),
raise NotImplementedError(self.install) CargoResolver(session)])
return StackedResolver(resolvers)
def explain(self, requirements):
raise NotImplementedError(self.explain)

25
ognibuild/resolver/apt.py
View file

@ -53,10 +53,6 @@ from ..requirements import (
) )
class NoAptPackage(Exception):
"""No apt package."""
class AptRequirement(object): class AptRequirement(object):
def __init__(self, package, minimum_version=None): def __init__(self, package, minimum_version=None):
@ -161,15 +157,13 @@ def resolve_binary_req(apt_mgr, req):
def resolve_pkg_config_req(apt_mgr, req): def resolve_pkg_config_req(apt_mgr, req):
package = apt_mgr.get_package_for_paths( package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")], [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")],
req.minimum_version
) )
if package is None: if package is None:
package = apt_mgr.get_package_for_paths( package = apt_mgr.get_package_for_paths(
[posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")], [posixpath.join("/usr/lib", ".*", "pkgconfig", req.module + ".pc")],
regex=True, regex=True)
minimum_version=req.minimum_version)
if package is not None: if package is not None:
return AptRequirement(package) return AptRequirement(package, minimum_version=req.minimum_version)
return None return None
@ -502,10 +496,7 @@ APT_REQUIREMENT_RESOLVERS = [
def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement) -> AptRequirement: def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement) -> AptRequirement:
for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS: for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
if isinstance(req, rr_class): if isinstance(req, rr_class):
deb_req = rr_fn(apt_mgr, req) return rr_fn(apt_mgr, req)
if deb_req is None:
raise NoAptPackage(req)
return deb_req
raise NotImplementedError(type(req)) raise NotImplementedError(type(req))
@ -514,6 +505,9 @@ class AptResolver(Resolver):
def __init__(self, apt): def __init__(self, apt):
self.apt = apt self.apt = apt
def __str__(self):
return "apt"
@classmethod @classmethod
def from_session(cls, session): def from_session(cls, session):
return cls(AptManager(session)) return cls(AptManager(session))
@ -530,10 +524,11 @@ class AptResolver(Resolver):
still_missing = [] still_missing = []
apt_requirements = [] apt_requirements = []
for m in missing: for m in missing:
try: apt_req = self.resolve(m)
apt_requirements.append(self.resolve(m)) if apt_req is None:
except NoAptPackage:
still_missing.append(m) still_missing.append(m)
else:
apt_requirements.append(m)
self.apt.install( self.apt.install(
[req.package for req in apt_requirements]) [req.package for req in apt_requirements])
if still_missing: if still_missing:

3
ognibuild/session/plain.py
View file

@ -33,6 +33,9 @@ class PlainSession(Session):
def check_call(self, args): def check_call(self, args):
return subprocess.check_call(args) return subprocess.check_call(args)
def check_output(self, args):
return subprocess.check_output(args)
def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None): def Popen(self, args, stdout=None, stderr=None, user=None, cwd=None):
return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd) return subprocess.Popen(args, stdout=stdout, stderr=stderr, cwd=cwd)

4
ognibuild/test.py
View file

@ -18,13 +18,13 @@
from .buildsystem import NoBuildToolsFound from .buildsystem import NoBuildToolsFound
def run_test(session, buildsystems, resolver): def run_test(session, buildsystems, resolver, fixers):
# Some things want to write to the user's home directory, # Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache # e.g. pip caches in ~/.cache
session.create_home() session.create_home()
for buildsystem in buildsystems: for buildsystem in buildsystems:
buildsystem.test(session, resolver) buildsystem.test(session, resolver, fixers)
return return
raise NoBuildToolsFound() raise NoBuildToolsFound()

34
ognibuild/tests/test_debian_fix_build.py
View file

@ -31,7 +31,7 @@ from buildlog_consultant.common import (
MissingValaPackage, MissingValaPackage,
) )
from ..debian import apt from ..debian import apt
from ..debian.apt import AptManager from ..debian.apt import AptManager, FileSearcher
from ..debian.fix_build import ( from ..debian.fix_build import (
resolve_error, resolve_error,
versioned_package_fixers, versioned_package_fixers,
@ -41,6 +41,21 @@ from ..debian.fix_build import (
from breezy.tests import TestCaseWithTransport from breezy.tests import TestCaseWithTransport
class DummyAptSearcher(FileSearcher):
def __init__(self, files):
self._apt_files = files
def search_files(self, path, regex=False):
for p, pkg in sorted(self._apt_files.items()):
if regex:
if re.match(path, p):
yield pkg
else:
if path == p:
yield pkg
class ResolveErrorTests(TestCaseWithTransport): class ResolveErrorTests(TestCaseWithTransport):
def setUp(self): def setUp(self):
super(ResolveErrorTests, self).setUp() super(ResolveErrorTests, self).setUp()
@ -76,21 +91,13 @@ blah (0.1) UNRELEASED; urgency=medium
) )
self.tree.add(["debian", "debian/control", "debian/changelog"]) self.tree.add(["debian", "debian/control", "debian/changelog"])
self.tree.commit("Initial commit") self.tree.commit("Initial commit")
self.overrideAttr(apt, "search_apt_file", self._search_apt_file)
self._apt_files = {} self._apt_files = {}
def _search_apt_file(self, path, regex=False):
for p, pkg in sorted(self._apt_files.items()):
if regex:
if re.match(path, p):
yield pkg
else:
if path == p:
yield pkg
def resolve(self, error, context=("build",)): def resolve(self, error, context=("build",)):
from ..session.plain import PlainSession from ..session.plain import PlainSession
apt = AptManager(PlainSession()) session = PlainSession()
apt = AptManager(session)
apt._searchers = [DummyAptSearcher(self._apt_files)]
context = BuildDependencyContext( context = BuildDependencyContext(
self.tree, self.tree,
apt, apt,
@ -98,7 +105,8 @@ blah (0.1) UNRELEASED; urgency=medium
committer="ognibuild <ognibuild@jelmer.uk>", committer="ognibuild <ognibuild@jelmer.uk>",
update_changelog=True, update_changelog=True,
) )
return resolve_error(error, context, versioned_package_fixers() + apt_fixers(apt)) fixers = versioned_package_fixers(session) + apt_fixers(apt)
return resolve_error(error, context, fixers)
def get_build_deps(self): def get_build_deps(self):
with open(self.tree.abspath("debian/control"), "r") as f: with open(self.tree.abspath("debian/control"), "r") as f:

5
setup.py
View file

@ -30,9 +30,10 @@ setup(name="ognibuild",
install_requires=[ install_requires=[
'breezy', 'breezy',
'buildlog-consultant', 'buildlog-consultant',
'python_debian',
'debmutate',
], ],
extras_require={
'debian': ['debmutate', 'python_debian', 'python_apt'],
},
tests_require=['python_debian', 'buildlog-consultant', 'breezy'], tests_require=['python_debian', 'buildlog-consultant', 'breezy'],
test_suite='ognibuild.tests.test_suite', test_suite='ognibuild.tests.test_suite',
) )