billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix style.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-03-26 12:14:25 +00:00
parent 7b3b171134
commit 3d914f5b4e
21 changed files with 754 additions and 524 deletions
Download patch file Download diff file Expand all files Collapse all files

6
ognibuild/__init__.py
View file

@ -41,7 +41,11 @@ class UnidentifiedError(Exception):
def __repr__(self): def __repr__(self):
return "<%s(%r, %r, ..., secondary=%r)>" % ( return "<%s(%r, %r, ..., secondary=%r)>" % (
type(self).__name__, self.retcode, self.argv, self.secondary) type(self).__name__,
self.retcode,
self.argv,
self.secondary,
)
def shebang_binary(p): def shebang_binary(p):

27
ognibuild/__main__.py
View file

@ -20,7 +20,12 @@ import os
import shlex import shlex
import sys import sys
from . import UnidentifiedError, DetailedFailure from . import UnidentifiedError, DetailedFailure
from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall, install_missing_reqs from .buildlog import (
InstallFixer,
ExplainInstallFixer,
ExplainInstall,
install_missing_reqs,
)
from .buildsystem import NoBuildToolsFound, detect_buildsystems from .buildsystem import NoBuildToolsFound, detect_buildsystems
from .resolver import ( from .resolver import (
auto_resolver, auto_resolver,
@ -34,8 +39,7 @@ def display_explain_commands(commands):
for command, reqs in commands: for command, reqs in commands:
if isinstance(command, list): if isinstance(command, list):
command = shlex.join(command) command = shlex.join(command)
logging.info( logging.info(" %s (to install %s)", command, ", ".join(map(str, reqs)))
' %s (to install %s)', command, ', '.join(map(str, reqs)))
def get_necessary_declared_requirements(resolver, requirements, stages): def get_necessary_declared_requirements(resolver, requirements, stages):
@ -46,7 +50,9 @@ def get_necessary_declared_requirements(resolver, requirements, stages):
return missing return missing
def install_necessary_declared_requirements(session, resolver, fixers, buildsystems, stages, explain=False): def install_necessary_declared_requirements(
session, resolver, fixers, buildsystems, stages, explain=False
):
relevant = [] relevant = []
declared_reqs = [] declared_reqs = []
for buildsystem in buildsystems: for buildsystem in buildsystems:
@ -153,8 +159,7 @@ def main(): # noqa: C901
logging.info("Using requirement resolver: %s", resolver) logging.info("Using requirement resolver: %s", resolver)
try: try:
bss = list(detect_buildsystems(args.directory)) bss = list(detect_buildsystems(args.directory))
logging.info( logging.info("Detected buildsystems: %s", ", ".join(map(str, bss)))
"Detected buildsystems: %s", ', '.join(map(str, bss)))
fixers = determine_fixers(session, resolver, explain=args.explain) fixers = determine_fixers(session, resolver, explain=args.explain)
if not args.ignore_declared_dependencies: if not args.ignore_declared_dependencies:
stages = STAGE_MAP[args.subcommand] stages = STAGE_MAP[args.subcommand]
@ -162,7 +167,8 @@ def main(): # noqa: C901
logging.info("Checking that declared requirements are present") logging.info("Checking that declared requirements are present")
try: try:
install_necessary_declared_requirements( install_necessary_declared_requirements(
session, resolver, fixers, bss, stages, explain=args.explain) session, resolver, fixers, bss, stages, explain=args.explain
)
except ExplainInstall as e: except ExplainInstall as e:
display_explain_commands(e.commands) display_explain_commands(e.commands)
return 1 return 1
@ -170,8 +176,11 @@ def main(): # noqa: C901
from .dist import run_dist from .dist import run_dist
run_dist( run_dist(
session=session, buildsystems=bss, resolver=resolver, fixers=fixers, session=session,
target_directory='.' buildsystems=bss,
resolver=resolver,
fixers=fixers,
target_directory=".",
) )
if args.subcommand == "build": if args.subcommand == "build":
from .build import run_build from .build import run_build

10
ognibuild/buildlog.py
View file

@ -141,11 +141,12 @@ def problem_to_upstream_requirement(problem): # noqa: C901
elif isinstance(problem, MissingJavaClass): elif isinstance(problem, MissingJavaClass):
return JavaClassRequirement(problem.classname) return JavaClassRequirement(problem.classname)
elif isinstance(problem, MissingHaskellDependencies): elif isinstance(problem, MissingHaskellDependencies):
return [HaskellPackageRequirement.from_string(dep) return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps]
for dep in problem.deps]
elif isinstance(problem, MissingMavenArtifacts): elif isinstance(problem, MissingMavenArtifacts):
return [MavenArtifactRequirement.from_str(artifact) return [
for artifact in problem.artifacts] MavenArtifactRequirement.from_str(artifact)
for artifact in problem.artifacts
]
elif isinstance(problem, MissingCSharpCompiler): elif isinstance(problem, MissingCSharpCompiler):
return BinaryRequirement("msc") return BinaryRequirement("msc")
elif isinstance(problem, GnomeCommonMissing): elif isinstance(problem, GnomeCommonMissing):
@ -236,7 +237,6 @@ class InstallFixer(BuildFixer):
class ExplainInstall(Exception): class ExplainInstall(Exception):
def __init__(self, commands): def __init__(self, commands):
self.commands = commands self.commands = commands

401
ognibuild/buildsystem.py
View file

@ -44,7 +44,7 @@ from .requirements import (
GoRequirement, GoRequirement,
GoPackageRequirement, GoPackageRequirement,
) )
from .fix_build import run_with_build_fixers, run_detecting_problems from .fix_build import run_with_build_fixers
from .session import which from .session import which
@ -75,7 +75,9 @@ class BuildSystem(object):
def __str__(self): def __str__(self):
return self.name return self.name
def dist(self, session, resolver, fixers, target_directory: str, quiet=False) -> str: def dist(
self, session, resolver, fixers, target_directory: str, quiet=False
) -> str:
raise NotImplementedError(self.dist) raise NotImplementedError(self.dist)
def test(self, session, resolver, fixers): def test(self, session, resolver, fixers):
@ -103,11 +105,12 @@ class BuildSystem(object):
def xmlparse_simplify_namespaces(path, namespaces): def xmlparse_simplify_namespaces(path, namespaces):
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
namespaces = ['{%s}' % ns for ns in namespaces]
namespaces = ["{%s}" % ns for ns in namespaces]
tree = ET.iterparse(path) tree = ET.iterparse(path)
for _, el in tree: for _, el in tree:
for namespace in namespaces: for namespace in namespaces:
el.tag = el.tag.replace(namespace, '') el.tag = el.tag.replace(namespace, "")
return tree.root return tree.root
@ -123,7 +126,7 @@ class Pear(BuildSystem):
def dist(self, session, resolver, fixers, target_directory: str, quiet=False): def dist(self, session, resolver, fixers, target_directory: str, quiet=False):
self.setup(resolver) self.setup(resolver)
with DistCatcher([session.external_path('.')]) as dc: with DistCatcher([session.external_path(".")]) as dc:
run_with_build_fixers(session, ["pear", "package"], fixers) run_with_build_fixers(session, ["pear", "package"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@ -146,28 +149,34 @@ class Pear(BuildSystem):
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
path = os.path.join(self.path, "package.xml") path = os.path.join(self.path, "package.xml")
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
try: try:
root = xmlparse_simplify_namespaces(path, [ root = xmlparse_simplify_namespaces(
'http://pear.php.net/dtd/package-2.0', path,
'http://pear.php.net/dtd/package-2.1']) [
"http://pear.php.net/dtd/package-2.0",
"http://pear.php.net/dtd/package-2.1",
],
)
except ET.ParseError as e: except ET.ParseError as e:
logging.warning('Unable to parse package.xml: %s', e) logging.warning("Unable to parse package.xml: %s", e)
return return
assert root.tag == 'package', 'root tag is %r' % root.tag assert root.tag == "package", "root tag is %r" % root.tag
dependencies_tag = root.find('dependencies') dependencies_tag = root.find("dependencies")
if dependencies_tag is not None: if dependencies_tag is not None:
required_tag = root.find('dependencies') required_tag = root.find("dependencies")
if required_tag is not None: if required_tag is not None:
for package_tag in root.findall('package'): for package_tag in root.findall("package"):
name = package_tag.find('name').text name = package_tag.find("name").text
min_tag = package_tag.find('min') min_tag = package_tag.find("min")
max_tag = package_tag.find('max') max_tag = package_tag.find("max")
channel_tag = package_tag.find('channel') channel_tag = package_tag.find("channel")
yield "core", PhpPackageRequirement( yield "core", PhpPackageRequirement(
name, name,
channel=(channel_tag.text if channel_tag else None), channel=(channel_tag.text if channel_tag else None),
min_version=(min_tag.text if min_tag else None), min_version=(min_tag.text if min_tag else None),
max_version=(max_tag.text if max_tag else None)) max_version=(max_tag.text if max_tag else None),
)
@classmethod @classmethod
def probe(cls, path): def probe(cls, path):
@ -179,6 +188,7 @@ class Pear(BuildSystem):
# run_setup, but setting __name__ # run_setup, but setting __name__
# Imported from Python's distutils.core, Copyright (C) PSF # Imported from Python's distutils.core, Copyright (C) PSF
def run_setup(script_name, script_args=None, stop_after="run"): def run_setup(script_name, script_args=None, stop_after="run"):
from distutils import core from distutils import core
import sys import sys
@ -260,11 +270,11 @@ with open(%(output_path)s, 'w') as f:
class SetupPy(BuildSystem): class SetupPy(BuildSystem):
name = "setup.py" name = "setup.py"
DEFAULT_PYTHON = 'python3' DEFAULT_PYTHON = "python3"
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
if os.path.exists(os.path.join(self.path, 'setup.py')): if os.path.exists(os.path.join(self.path, "setup.py")):
self.has_setup_py = True self.has_setup_py = True
else: else:
self.has_setup_py = False self.has_setup_py = False
@ -280,7 +290,9 @@ class SetupPy(BuildSystem):
self.pyproject = None self.pyproject = None
self.build_backend = None self.build_backend = None
else: else:
self.build_backend = self.pyproject.get("build-system", {}).get('build-backend') self.build_backend = self.pyproject.get("build-system", {}).get(
"build-backend"
)
def load_toml(self): def load_toml(self):
import toml import toml
@ -290,7 +302,8 @@ class SetupPy(BuildSystem):
def load_setup_cfg(self): def load_setup_cfg(self):
from setuptools.config import read_configuration from setuptools.config import read_configuration
p = os.path.join(self.path, 'setup.cfg')
p = os.path.join(self.path, "setup.cfg")
if os.path.exists(p): if os.path.exists(p):
return read_configuration(p) return read_configuration(p)
raise FileNotFoundError(p) raise FileNotFoundError(p)
@ -304,7 +317,7 @@ class SetupPy(BuildSystem):
return self._extract_setup_in_session(session, fixers) return self._extract_setup_in_session(session, fixers)
def _extract_setup_direct(self): def _extract_setup_direct(self):
p = os.path.join(self.path, 'setup.py') p = os.path.join(self.path, "setup.py")
try: try:
d = run_setup(os.path.abspath(p), stop_after="init") d = run_setup(os.path.abspath(p), stop_after="init")
except RuntimeError as e: except RuntimeError as e:
@ -313,37 +326,42 @@ class SetupPy(BuildSystem):
if d is None: if d is None:
logging.warning( logging.warning(
"'distutils.core.setup()' was never called -- " "'distutils.core.setup()' was never called -- "
"perhaps '%s' is not a Distutils setup script?" % os.path.basename(p)) "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p)
)
return None return None
return { return {
'name': d.name, "name": d.name,
'setup_requires': getattr(d, "setup_requires", []), "setup_requires": getattr(d, "setup_requires", []),
'install_requires': getattr(d, "install_requires", []), "install_requires": getattr(d, "install_requires", []),
'tests_require': getattr(d, "tests_require", []) or [], "tests_require": getattr(d, "tests_require", []) or [],
'scripts': getattr(d, "scripts", []), "scripts": getattr(d, "scripts", []),
'entry_points': getattr(d, "entry_points", None) or {}, "entry_points": getattr(d, "entry_points", None) or {},
'packages': getattr(d, "packages", []), "packages": getattr(d, "packages", []),
'requires': d.get_requires() or [], "requires": d.get_requires() or [],
} }
def _extract_setup_in_session(self, session, fixers=None): def _extract_setup_in_session(self, session, fixers=None):
import tempfile import tempfile
import json import json
interpreter = shebang_binary(os.path.join(self.path, "setup.py")) interpreter = shebang_binary(os.path.join(self.path, "setup.py"))
if interpreter is None: if interpreter is None:
interpreter = self.DEFAULT_PYTHON interpreter = self.DEFAULT_PYTHON
output_f = tempfile.NamedTemporaryFile( output_f = tempfile.NamedTemporaryFile(
dir=os.path.join(session.location, 'tmp'), mode='w+t') dir=os.path.join(session.location, "tmp"), mode="w+t"
)
with output_f: with output_f:
# TODO(jelmer): Perhaps run this in session, so we can install # TODO(jelmer): Perhaps run this in session, so we can install
# missing dependencies? # missing dependencies?
argv = [interpreter, "-c", argv = [
_setup_wrapper interpreter,
.replace('%(script_name)s', '"setup.py"') "-c",
.replace('%(output_path)s', _setup_wrapper.replace("%(script_name)s", '"setup.py"').replace(
'"/' + os.path.relpath(output_f.name, session.location) + "%(output_path)s",
'"')] '"/' + os.path.relpath(output_f.name, session.location) + '"',
),
]
try: try:
if fixers is not None: if fixers is not None:
run_with_build_fixers(session, argv, fixers) run_with_build_fixers(session, argv, fixers)
@ -359,15 +377,17 @@ class SetupPy(BuildSystem):
return "%s(%r)" % (type(self).__name__, self.path) return "%s(%r)" % (type(self).__name__, self.path)
def test(self, session, resolver, fixers): def test(self, session, resolver, fixers):
if os.path.exists(os.path.join(self.path, 'tox.ini')): if os.path.exists(os.path.join(self.path, "tox.ini")):
run_with_build_fixers(session, ['tox'], fixers) run_with_build_fixers(session, ["tox"], fixers)
elif self.pyproject: elif self.pyproject:
run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers) run_with_build_fixers(
session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers
)
elif self.has_setup_py: elif self.has_setup_py:
# Pre-emptively insall setuptools, since distutils doesn't provide # Pre-emptively insall setuptools, since distutils doesn't provide
# a 'test' subcommand and some packages fall back to distutils # a 'test' subcommand and some packages fall back to distutils
# if setuptools is not available. # if setuptools is not available.
resolver.install([PythonPackageRequirement('setuptools')]) resolver.install([PythonPackageRequirement("setuptools")])
self._run_setup(session, resolver, ["test"], fixers) self._run_setup(session, resolver, ["test"], fixers)
else: else:
raise NotImplementedError raise NotImplementedError
@ -386,13 +406,17 @@ class SetupPy(BuildSystem):
preargs.append("--quiet") preargs.append("--quiet")
# Preemptively install setuptools since some packages fail in # Preemptively install setuptools since some packages fail in
# some way without it. # some way without it.
resolver.install([PythonPackageRequirement('setuptools')]) resolver.install([PythonPackageRequirement("setuptools")])
with DistCatcher([session.external_path('dist')]) as dc: with DistCatcher([session.external_path("dist")]) as dc:
self._run_setup(session, resolver, preargs + ["sdist"], fixers) self._run_setup(session, resolver, preargs + ["sdist"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
elif self.pyproject: elif self.pyproject:
with DistCatcher([session.external_path('dist')]) as dc: with DistCatcher([session.external_path("dist")]) as dc:
run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], fixers) run_with_build_fixers(
session,
[self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."],
fixers,
)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
raise AssertionError("no setup.py or pyproject.toml") raise AssertionError("no setup.py or pyproject.toml")
@ -413,79 +437,83 @@ class SetupPy(BuildSystem):
def _run_setup(self, session, resolver, args, fixers): def _run_setup(self, session, resolver, args, fixers):
from .buildlog import install_missing_reqs from .buildlog import install_missing_reqs
# Install the setup_requires beforehand, since otherwise # Install the setup_requires beforehand, since otherwise
# setuptools might fetch eggs instead of our preferred resolver. # setuptools might fetch eggs instead of our preferred resolver.
install_missing_reqs(session, resolver, list(self._setup_requires())) install_missing_reqs(session, resolver, list(self._setup_requires()))
interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) interpreter = shebang_binary(os.path.join(self.path, "setup.py"))
if interpreter is None: if interpreter is None:
interpreter = self.DEFAULT_PYTHON interpreter = self.DEFAULT_PYTHON
argv = [interpreter, "./setup.py"] + args argv = [interpreter, "./setup.py"] + args
env = {} env = {}
# Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment # Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment
if 'SETUPTOOLS_SCM_PRETEND_VERSION' in os.environ: if "SETUPTOOLS_SCM_PRETEND_VERSION" in os.environ:
env['SETUPTOOLS_SCM_PRETEND_VERSION'] = ( env["SETUPTOOLS_SCM_PRETEND_VERSION"] = os.environ[
os.environ['SETUPTOOLS_SCM_PRETEND_VERSION']) "SETUPTOOLS_SCM_PRETEND_VERSION"
]
run_with_build_fixers(session, argv, fixers, env=env) run_with_build_fixers(session, argv, fixers, env=env)
def _setup_requires(self): def _setup_requires(self):
if self.pyproject: if self.pyproject:
if "build-system" in self.pyproject: if "build-system" in self.pyproject:
for require in self.pyproject['build-system'].get("requires", []): for require in self.pyproject["build-system"].get("requires", []):
yield PythonPackageRequirement.from_requirement_str(require) yield PythonPackageRequirement.from_requirement_str(require)
if self.config: if self.config:
options = self.config.get('options', {}) options = self.config.get("options", {})
for require in options.get('setup_requires', []): for require in options.get("setup_requires", []):
yield PythonPackageRequirement.from_requirement_str(require) yield PythonPackageRequirement.from_requirement_str(require)
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
distribution = self._extract_setup(session, fixers) distribution = self._extract_setup(session, fixers)
if distribution is not None: if distribution is not None:
for require in distribution['requires']: for require in distribution["requires"]:
yield "core", PythonPackageRequirement.from_requirement_str(require) yield "core", PythonPackageRequirement.from_requirement_str(require)
# Not present for distutils-only packages # Not present for distutils-only packages
for require in distribution['setup_requires']: for require in distribution["setup_requires"]:
yield "build", PythonPackageRequirement.from_requirement_str(require) yield "build", PythonPackageRequirement.from_requirement_str(require)
# Not present for distutils-only packages # Not present for distutils-only packages
for require in distribution['install_requires']: for require in distribution["install_requires"]:
yield "core", PythonPackageRequirement.from_requirement_str(require) yield "core", PythonPackageRequirement.from_requirement_str(require)
# Not present for distutils-only packages # Not present for distutils-only packages
for require in distribution['tests_require']: for require in distribution["tests_require"]:
yield "test", PythonPackageRequirement.from_requirement_str(require) yield "test", PythonPackageRequirement.from_requirement_str(require)
if self.pyproject: if self.pyproject:
if "build-system" in self.pyproject: if "build-system" in self.pyproject:
for require in self.pyproject['build-system'].get("requires", []): for require in self.pyproject["build-system"].get("requires", []):
yield "build", PythonPackageRequirement.from_requirement_str(require) yield "build", PythonPackageRequirement.from_requirement_str(
require
)
if self.config: if self.config:
options = self.config.get('options', {}) options = self.config.get("options", {})
for require in options.get('setup_requires', []): for require in options.get("setup_requires", []):
yield "build", PythonPackageRequirement.from_requirement_str(require) yield "build", PythonPackageRequirement.from_requirement_str(require)
for require in options.get('install_requires', []): for require in options.get("install_requires", []):
yield "core", PythonPackageRequirement.from_requirement_str(require) yield "core", PythonPackageRequirement.from_requirement_str(require)
def get_declared_outputs(self, session, fixers=None): def get_declared_outputs(self, session, fixers=None):
distribution = self._extract_setup(session, fixers) distribution = self._extract_setup(session, fixers)
all_packages = set() all_packages = set()
if distribution is not None: if distribution is not None:
for script in distribution['scripts']: for script in distribution["scripts"]:
yield BinaryOutput(os.path.basename(script)) yield BinaryOutput(os.path.basename(script))
for script in distribution["entry_points"].get("console_scripts", []): for script in distribution["entry_points"].get("console_scripts", []):
yield BinaryOutput(script.split("=")[0]) yield BinaryOutput(script.split("=")[0])
all_packages.update(distribution['packages']) all_packages.update(distribution["packages"])
if self.config: if self.config:
options = self.config.get('options', {}) options = self.config.get("options", {})
all_packages.update(options.get('packages', [])) all_packages.update(options.get("packages", []))
for script in options.get('scripts', []): for script in options.get("scripts", []):
yield BinaryOutput(os.path.basename(script)) yield BinaryOutput(os.path.basename(script))
for script in options.get("entry_points", {}).get("console_scripts", []): for script in options.get("entry_points", {}).get("console_scripts", []):
yield BinaryOutput(script.split("=")[0]) yield BinaryOutput(script.split("=")[0])
packages = set() packages = set()
for package in sorted(all_packages): for package in sorted(all_packages):
pts = package.split('.') pts = package.split(".")
b = [] b = []
for e in pts: for e in pts:
b.append(e) b.append(e)
if '.'.join(b) in packages: if ".".join(b) in packages:
break break
else: else:
packages.add(package) packages.add(package)
@ -521,12 +549,12 @@ class Octave(BuildSystem):
return False return False
# Urgh, isn't there a better way to see if this is an octave package? # Urgh, isn't there a better way to see if this is an octave package?
for entry in os.scandir(path): for entry in os.scandir(path):
if entry.name.endswith('.m'): if entry.name.endswith(".m"):
return True return True
if not entry.is_dir(): if not entry.is_dir():
continue continue
for subentry in os.scandir(entry.path): for subentry in os.scandir(entry.path):
if subentry.name.endswith('.m'): if subentry.name.endswith(".m"):
return True return True
return False return False
@ -537,17 +565,19 @@ class Octave(BuildSystem):
return cls(path) return cls(path)
def _read_description(self): def _read_description(self):
path = os.path.join(self.path, 'DESCRIPTION') path = os.path.join(self.path, "DESCRIPTION")
from email.parser import BytesParser from email.parser import BytesParser
with open(path, 'rb') as f:
with open(path, "rb") as f:
return BytesParser().parse(f) return BytesParser().parse(f)
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
def parse_list(t): def parse_list(t):
return [s.strip() for s in t.split(',') if s.strip()] return [s.strip() for s in t.split(",") if s.strip()]
description = self._read_description() description = self._read_description()
if 'Depends' in description: if "Depends" in description:
for s in parse_list(description['Depends']): for s in parse_list(description["Depends"]):
yield "build", OctavePackageRequirement.from_str(s) yield "build", OctavePackageRequirement.from_str(s)
@ -564,14 +594,14 @@ class Gradle(BuildSystem):
@classmethod @classmethod
def exists(cls, path): def exists(cls, path):
return ( return os.path.exists(os.path.join(path, "build.gradle")) or os.path.exists(
os.path.exists(os.path.join(path, "build.gradle")) or os.path.join(path, "build.gradle.kts")
os.path.exists(os.path.join(path, "build.gradle.kts"))) )
@classmethod @classmethod
def from_path(cls, path): def from_path(cls, path):
if os.path.exists(os.path.join(path, "gradlew")): if os.path.exists(os.path.join(path, "gradlew")):
return cls(path, './gradlew') return cls(path, "./gradlew")
return cls(path) return cls(path)
@classmethod @classmethod
@ -581,44 +611,50 @@ class Gradle(BuildSystem):
return cls.from_path(path) return cls.from_path(path)
def setup(self, resolver): def setup(self, resolver):
if not self.executable.startswith('./'): if not self.executable.startswith("./"):
resolver.install([BinaryRequirement(self.executable)]) resolver.install([BinaryRequirement(self.executable)])
def _run(self, session, resolver, task, args, fixers): def _run(self, session, resolver, task, args, fixers):
self.setup(resolver) self.setup(resolver)
argv = [] argv = []
if self.executable.startswith('./') and ( if self.executable.startswith("./") and (
not os.access(os.path.join(self.path, self.executable), os.X_OK)): not os.access(os.path.join(self.path, self.executable), os.X_OK)
argv.append('sh') ):
argv.append("sh")
argv.extend([self.executable, task]) argv.extend([self.executable, task])
argv.extend(args) argv.extend(args)
try: try:
run_with_build_fixers(session, argv, fixers) run_with_build_fixers(session, argv, fixers)
except UnidentifiedError as e: except UnidentifiedError as e:
if any([re.match( if any(
r"Task '" + task + "' not found in root project '.*'\.", [
line) for line in e.lines]): re.match(
r"Task '" + task + r"' not found in root project '.*'\.", line
)
for line in e.lines
]
):
raise NotImplementedError raise NotImplementedError
raise raise
def clean(self, session, resolver, fixers): def clean(self, session, resolver, fixers):
self._run(session, resolver, 'clean', [], fixers) self._run(session, resolver, "clean", [], fixers)
def build(self, session, resolver, fixers): def build(self, session, resolver, fixers):
self._run(session, resolver, 'build', [], fixers) self._run(session, resolver, "build", [], fixers)
def test(self, session, resolver, fixers): def test(self, session, resolver, fixers):
self._run(session, resolver, 'test', [], fixers) self._run(session, resolver, "test", [], fixers)
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
with DistCatcher([session.external_path('.')]) as dc: with DistCatcher([session.external_path(".")]) as dc:
self._run(session, resolver, 'distTar', [], fixers) self._run(session, resolver, "distTar", [], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
def install(self, session, resolver, fixers, install_target): def install(self, session, resolver, fixers, install_target):
raise NotImplementedError raise NotImplementedError
# TODO(jelmer): installDist just creates files under build/install/... # TODO(jelmer): installDist just creates files under build/install/...
self._run(session, resolver, 'installDist', [], fixers) self._run(session, resolver, "installDist", [], fixers)
class R(BuildSystem): class R(BuildSystem):
@ -638,7 +674,7 @@ class R(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
r_path = guaranteed_which(session, resolver, "R") r_path = guaranteed_which(session, resolver, "R")
with DistCatcher([session.external_path('.')]) as dc: with DistCatcher([session.external_path(".")]) as dc:
run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers) run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@ -652,34 +688,37 @@ class R(BuildSystem):
@classmethod @classmethod
def probe(cls, path): def probe(cls, path):
if (os.path.exists(os.path.join(path, 'DESCRIPTION')) and if os.path.exists(os.path.join(path, "DESCRIPTION")) and os.path.exists(
os.path.exists(os.path.join(path, 'NAMESPACE'))): os.path.join(path, "NAMESPACE")
):
return cls(path) return cls(path)
def _read_description(self): def _read_description(self):
path = os.path.join(self.path, 'DESCRIPTION') path = os.path.join(self.path, "DESCRIPTION")
from email.parser import BytesParser from email.parser import BytesParser
with open(path, 'rb') as f:
with open(path, "rb") as f:
return BytesParser().parse(f) return BytesParser().parse(f)
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
def parse_list(t): def parse_list(t):
return [s.strip() for s in t.split(',') if s.strip()] return [s.strip() for s in t.split(",") if s.strip()]
description = self._read_description() description = self._read_description()
if 'Suggests' in description: if "Suggests" in description:
for s in parse_list(description['Suggests']): for s in parse_list(description["Suggests"]):
yield "build", RPackageRequirement.from_str(s) yield "build", RPackageRequirement.from_str(s)
if 'Depends' in description: if "Depends" in description:
for s in parse_list(description['Depends']): for s in parse_list(description["Depends"]):
yield "build", RPackageRequirement.from_str(s) yield "build", RPackageRequirement.from_str(s)
if 'Imports' in description: if "Imports" in description:
for s in parse_list(description['Imports']): for s in parse_list(description["Imports"]):
yield "build", RPackageRequirement.from_str(s) yield "build", RPackageRequirement.from_str(s)
def get_declared_outputs(self, session, fixers=None): def get_declared_outputs(self, session, fixers=None):
description = self._read_description() description = self._read_description()
if 'Package' in description: if "Package" in description:
yield RPackageOutput(description['Package']) yield RPackageOutput(description["Package"])
class Meson(BuildSystem): class Meson(BuildSystem):
@ -693,8 +732,8 @@ class Meson(BuildSystem):
return "%s(%r)" % (type(self).__name__, self.path) return "%s(%r)" % (type(self).__name__, self.path)
def _setup(self, session, fixers): def _setup(self, session, fixers):
if not session.exists('build'): if not session.exists("build"):
session.check_call(['mkdir', 'build']) session.check_call(["mkdir", "build"])
run_with_build_fixers(session, ["meson", "setup", "build"], fixers) run_with_build_fixers(session, ["meson", "setup", "build"], fixers)
def clean(self, session, resolver, fixers): def clean(self, session, resolver, fixers):
@ -707,7 +746,7 @@ class Meson(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
self._setup(session, fixers) self._setup(session, fixers)
with DistCatcher([session.external_path('build/meson-dist')]) as dc: with DistCatcher([session.external_path("build/meson-dist")]) as dc:
run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers) run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@ -742,7 +781,7 @@ class Npm(BuildSystem):
return "%s(%r)" % (type(self).__name__, self.path) return "%s(%r)" % (type(self).__name__, self.path)
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
if 'dependencies' in self.package: if "dependencies" in self.package:
for name, unused_version in self.package["dependencies"].items(): for name, unused_version in self.package["dependencies"].items():
# TODO(jelmer): Look at version # TODO(jelmer): Look at version
yield "core", NodePackageRequirement(name) yield "core", NodePackageRequirement(name)
@ -756,13 +795,13 @@ class Npm(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
self.setup(resolver) self.setup(resolver)
with DistCatcher([session.external_path('.')]) as dc: with DistCatcher([session.external_path(".")]) as dc:
run_with_build_fixers(session, ["npm", "pack"], fixers) run_with_build_fixers(session, ["npm", "pack"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
def test(self, session, resolver, fixers): def test(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
test_script = self.package['scripts'].get('test') test_script = self.package["scripts"].get("test")
if test_script: if test_script:
run_with_build_fixers(session, shlex.split(test_script), fixers) run_with_build_fixers(session, shlex.split(test_script), fixers)
else: else:
@ -770,7 +809,7 @@ class Npm(BuildSystem):
def build(self, session, resolver, fixers): def build(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
build_script = self.package['scripts'].get('build') build_script = self.package["scripts"].get("build")
if build_script: if build_script:
run_with_build_fixers(session, shlex.split(build_script), fixers) run_with_build_fixers(session, shlex.split(build_script), fixers)
else: else:
@ -778,7 +817,7 @@ class Npm(BuildSystem):
def clean(self, session, resolver, fixers): def clean(self, session, resolver, fixers):
self.setup(resolver) self.setup(resolver)
clean_script = self.package['scripts'].get('clean') clean_script = self.package["scripts"].get("clean")
if clean_script: if clean_script:
run_with_build_fixers(session, shlex.split(clean_script), fixers) run_with_build_fixers(session, shlex.split(clean_script), fixers)
else: else:
@ -803,7 +842,7 @@ class Waf(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
self.setup(session, resolver, fixers) self.setup(session, resolver, fixers)
with DistCatcher.default(session.external_path('.')) as dc: with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["./waf", "dist"], fixers) run_with_build_fixers(session, ["./waf", "dist"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@ -835,13 +874,15 @@ class Gem(BuildSystem):
] ]
if len(gemfiles) > 1: if len(gemfiles) > 1:
logging.warning("More than one gemfile. Trying the first?") logging.warning("More than one gemfile. Trying the first?")
with DistCatcher.default(session.external_path('.')) as dc: with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers) run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@classmethod @classmethod
def probe(cls, path): def probe(cls, path):
gemfiles = [entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")] gemfiles = [
entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")
]
if gemfiles: if gemfiles:
return cls(gemfiles[0]) return cls(gemfiles[0])
@ -881,13 +922,13 @@ class DistZilla(BuildSystem):
self.setup(resolver) self.setup(resolver)
if self.name == "dist-inkt": if self.name == "dist-inkt":
resolver.install([PerlModuleRequirement(self.dist_inkt_class)]) resolver.install([PerlModuleRequirement(self.dist_inkt_class)])
with DistCatcher.default(session.external_path('.')) as dc: with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["distinkt-dist"], fixers) run_with_build_fixers(session, ["distinkt-dist"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
else: else:
# Default to invoking Dist::Zilla # Default to invoking Dist::Zilla
resolver.install([PerlModuleRequirement("Dist::Zilla")]) resolver.install([PerlModuleRequirement("Dist::Zilla")])
with DistCatcher.default(session.external_path('.')) as dc: with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers) run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@ -939,13 +980,13 @@ class RunTests(BuildSystem):
def _read_cpanfile(session, args, kind, fixers): def _read_cpanfile(session, args, kind, fixers):
for line in run_with_build_fixers(session, ['cpanfile-dump'] + args, fixers): for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers):
yield kind, PerlModuleRequirement(line) yield kind, PerlModuleRequirement(line)
def _declared_deps_from_cpanfile(session, fixers): def _declared_deps_from_cpanfile(session, fixers):
yield from _read_cpanfile(session, ['--configure', '--build'], 'build', fixers) yield from _read_cpanfile(session, ["--configure", "--build"], "build", fixers)
yield from _read_cpanfile(session, ['--test'], 'test', fixers) yield from _read_cpanfile(session, ["--test"], "test", fixers)
class Make(BuildSystem): class Make(BuildSystem):
@ -989,7 +1030,9 @@ class Make(BuildSystem):
if not makefile_exists() and session.exists("configure"): if not makefile_exists() and session.exists("configure"):
run_with_build_fixers(session, ["./configure"], fixers) run_with_build_fixers(session, ["./configure"], fixers)
if not makefile_exists() and any([n.name.endswith('.pro') for n in session.scandir(".")]): if not makefile_exists() and any(
[n.name.endswith(".pro") for n in session.scandir(".")]
):
run_with_build_fixers(session, ["qmake"], fixers) run_with_build_fixers(session, ["qmake"], fixers)
def build(self, session, resolver, fixers): def build(self, session, resolver, fixers):
@ -1010,7 +1053,7 @@ class Make(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
self.setup(session, resolver, fixers) self.setup(session, resolver, fixers)
with DistCatcher.default(session.external_path('.')) as dc: with DistCatcher.default(session.external_path(".")) as dc:
try: try:
run_with_build_fixers(session, ["make", "dist"], fixers) run_with_build_fixers(session, ["make", "dist"], fixers)
except UnidentifiedError as e: except UnidentifiedError as e:
@ -1106,7 +1149,7 @@ class Make(BuildSystem):
return cls(path) return cls(path)
for n in os.scandir(path): for n in os.scandir(path):
# qmake # qmake
if n.name.endswith('.pro'): if n.name.endswith(".pro"):
return cls(path) return cls(path)
@ -1133,8 +1176,9 @@ class Cargo(BuildSystem):
# TODO(jelmer): Look at details['version'] # TODO(jelmer): Look at details['version']
yield "build", CargoCrateRequirement( yield "build", CargoCrateRequirement(
name, name,
features=details.get('features', []), features=details.get("features", []),
version=details.get("version")) version=details.get("version"),
)
def test(self, session, resolver, fixers): def test(self, session, resolver, fixers):
run_with_build_fixers(session, ["cargo", "test"], fixers) run_with_build_fixers(session, ["cargo", "test"], fixers)
@ -1157,19 +1201,20 @@ def _parse_go_mod(f):
line = f.readline() line = f.readline()
if not line: if not line:
return line return line
return line.split('//')[0] + '\n' return line.split("//")[0] + "\n"
line = readline() line = readline()
while line: while line:
parts = line.strip().split(' ') parts = line.strip().split(" ")
if not parts or parts == ['']: if not parts or parts == [""]:
continue continue
if len(parts) == 2 and parts[1] == '(': if len(parts) == 2 and parts[1] == "(":
line = readline() line = readline()
while line.strip() != ')': while line.strip() != ")":
yield [parts[0]] + list(line.strip().split(' ')) yield [parts[0]] + list(line.strip().split(" "))
line = readline() line = readline()
if not line: if not line:
raise AssertionError('list of %s interrupted?' % parts[0]) raise AssertionError("list of %s interrupted?" % parts[0])
else: else:
yield parts yield parts
line = readline() line = readline()
@ -1199,31 +1244,30 @@ class Golang(BuildSystem):
session.check_call(["go", "clean"]) session.check_call(["go", "clean"])
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
go_mod_path = os.path.join(self.path, 'go.mod') go_mod_path = os.path.join(self.path, "go.mod")
if os.path.exists(go_mod_path): if os.path.exists(go_mod_path):
with open(go_mod_path, 'r') as f: with open(go_mod_path, "r") as f:
for parts in _parse_go_mod(f): for parts in _parse_go_mod(f):
if parts[0] == 'go': if parts[0] == "go":
yield "build", GoRequirement(parts[1]) yield "build", GoRequirement(parts[1])
elif parts[0] == 'require': elif parts[0] == "require":
yield "build", GoPackageRequirement( yield "build", GoPackageRequirement(
parts[1], parts[2].lstrip('v') if len(parts) > 2 else None) parts[1], parts[2].lstrip("v") if len(parts) > 2 else None
elif parts[0] == 'exclude': )
elif parts[0] == "exclude":
pass # TODO(jelmer): Create conflicts? pass # TODO(jelmer): Create conflicts?
elif parts[0] == 'replace': elif parts[0] == "replace":
pass # TODO(jelmer): do.. something? pass # TODO(jelmer): do.. something?
elif parts[0] == 'module': elif parts[0] == "module":
pass pass
else: else:
logging.warning( logging.warning("Unknown directive %s in go.mod", parts[0])
'Unknown directive %s in go.mod',
parts[0])
@classmethod @classmethod
def probe(cls, path): def probe(cls, path):
if os.path.exists(os.path.join(path, 'go.mod')): if os.path.exists(os.path.join(path, "go.mod")):
return Golang(path) return Golang(path)
if os.path.exists(os.path.join(path, 'go.sum')): if os.path.exists(os.path.join(path, "go.sum")):
return Golang(path) return Golang(path)
for entry in os.scandir(path): for entry in os.scandir(path):
if entry.name.endswith(".go"): if entry.name.endswith(".go"):
@ -1266,20 +1310,23 @@ class Maven(BuildSystem):
def get_declared_dependencies(self, session, fixers=None): def get_declared_dependencies(self, session, fixers=None):
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
try: try:
root = xmlparse_simplify_namespaces(self.path, root = xmlparse_simplify_namespaces(
['http://maven.apache.org/POM/4.0.0']) self.path, ["http://maven.apache.org/POM/4.0.0"]
)
except ET.ParseError as e: except ET.ParseError as e:
logging.warning('Unable to parse package.xml: %s', e) logging.warning("Unable to parse package.xml: %s", e)
return return
assert root.tag == 'project', 'root tag is %r' % root.tag assert root.tag == "project", "root tag is %r" % root.tag
deps_tag = root.find('dependencies') deps_tag = root.find("dependencies")
if deps_tag: if deps_tag:
for dep in deps_tag.findall('dependency'): for dep in deps_tag.findall("dependency"):
yield "core", MavenArtifactRequirement( yield "core", MavenArtifactRequirement(
dep.find('groupId').text, dep.find("groupId").text,
dep.find('artifactId').text, dep.find("artifactId").text,
dep.find('version').text) dep.find("version").text,
)
class Cabal(BuildSystem): class Cabal(BuildSystem):
@ -1310,9 +1357,12 @@ class Cabal(BuildSystem):
self._run(session, ["test"], fixers) self._run(session, ["test"], fixers)
def dist(self, session, resolver, fixers, target_directory, quiet=False): def dist(self, session, resolver, fixers, target_directory, quiet=False):
with DistCatcher([ with DistCatcher(
session.external_path('dist-newstyle/sdist'), [
session.external_path('dist')]) as dc: session.external_path("dist-newstyle/sdist"),
session.external_path("dist"),
]
) as dc:
self._run(session, ["sdist"], fixers) self._run(session, ["sdist"], fixers)
return dc.copy_single(target_directory) return dc.copy_single(target_directory)
@ -1377,23 +1427,38 @@ class PerlBuildTiny(BuildSystem):
BUILDSYSTEM_CLSES = [ BUILDSYSTEM_CLSES = [
Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven, Pear,
DistZilla, Gem, PerlBuildTiny, Golang, R, Octave, SetupPy,
Npm,
Waf,
Cargo,
Meson,
Cabal,
Gradle,
Maven,
DistZilla,
Gem,
PerlBuildTiny,
Golang,
R,
Octave,
# Make is intentionally at the end of the list. # Make is intentionally at the end of the list.
Make, Composer, RunTests] Make,
Composer,
RunTests,
]
def scan_buildsystems(path): def scan_buildsystems(path):
"""Detect build systems.""" """Detect build systems."""
ret = [] ret = []
ret.extend([('.', bs) for bs in detect_buildsystems(path)]) ret.extend([(".", bs) for bs in detect_buildsystems(path)])
if not ret: if not ret:
# Nothing found. Try the next level? # Nothing found. Try the next level?
for entry in os.scandir(path): for entry in os.scandir(path):
if entry.is_dir(): if entry.is_dir():
ret.extend( ret.extend([(entry.name, bs) for bs in detect_buildsystems(entry.path)])
[(entry.name, bs) for bs in detect_buildsystems(entry.path)])
return ret return ret

15
ognibuild/debian/apt.py
View file

@ -26,10 +26,17 @@ from buildlog_consultant.apt import (
from .. import DetailedFailure, UnidentifiedError from .. import DetailedFailure, UnidentifiedError
from ..session import Session, run_with_tee, get_user from ..session import Session, run_with_tee, get_user
from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_packages_for_paths from .file_search import (
FileSearcher,
AptCachedContentsFileSearcher,
GENERATED_FILE_SEARCHER,
get_packages_for_paths,
)
def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None: def run_apt(
session: Session, args: List[str], prefix: Optional[List[str]] = None
) -> None:
"""Run apt.""" """Run apt."""
if prefix is None: if prefix is None:
prefix = [] prefix = []
@ -84,7 +91,9 @@ class AptManager(object):
def get_packages_for_paths(self, paths, regex=False, case_insensitive=False): def get_packages_for_paths(self, paths, regex=False, case_insensitive=False):
logging.debug("Searching for packages containing %r", paths) logging.debug("Searching for packages containing %r", paths)
# TODO(jelmer): Make sure we use whatever is configured in self.session # TODO(jelmer): Make sure we use whatever is configured in self.session
return get_packages_for_paths(paths, self.searchers(), regex=regex, case_insensitive=case_insensitive) return get_packages_for_paths(
paths, self.searchers(), regex=regex, case_insensitive=case_insensitive
)
def missing(self, packages): def missing(self, packages):
root = getattr(self.session, "location", "/") root = getattr(self.session, "location", "/")

1
ognibuild/debian/build.py
View file

@ -33,7 +33,6 @@ import sys
from debian.changelog import Changelog from debian.changelog import Changelog
from debmutate.changelog import get_maintainer, format_datetime from debmutate.changelog import get_maintainer, format_datetime
from breezy import osutils
from breezy.mutabletree import MutableTree from breezy.mutabletree import MutableTree
from breezy.plugins.debian.builder import BuildFailedError from breezy.plugins.debian.builder import BuildFailedError
from breezy.tree import Tree from breezy.tree import Tree

17
ognibuild/debian/build_deps.py
View file

@ -22,7 +22,6 @@ import logging
class BuildDependencyTieBreaker(object): class BuildDependencyTieBreaker(object):
def __init__(self, rootdir): def __init__(self, rootdir):
self.rootdir = rootdir self.rootdir = rootdir
self._counts = None self._counts = None
@ -37,8 +36,9 @@ class BuildDependencyTieBreaker(object):
def _count(self): def _count(self):
counts = {} counts = {}
import apt_pkg import apt_pkg
apt_pkg.init() apt_pkg.init()
apt_pkg.config.set('Dir', self.rootdir) apt_pkg.config.set("Dir", self.rootdir)
apt_cache = apt_pkg.SourceRecords() apt_cache = apt_pkg.SourceRecords()
apt_cache.restart() apt_cache.restart()
while apt_cache.step(): while apt_cache.step():
@ -65,17 +65,20 @@ class BuildDependencyTieBreaker(object):
return None return None
top = max(by_count.items(), key=lambda k: k[1]) top = max(by_count.items(), key=lambda k: k[1])
logging.info( logging.info(
'Breaking tie between %r to %r based on build-depends count', "Breaking tie between %r to %r based on build-depends count",
[repr(r) for r in reqs], top[0]) [repr(r) for r in reqs],
top[0],
)
return top[0] return top[0]
if __name__ == '__main__': if __name__ == "__main__":
import argparse import argparse
from ..resolver.apt import AptRequirement from ..resolver.apt import AptRequirement
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('req', nargs='+') parser.add_argument("req", nargs="+")
args = parser.parse_args() args = parser.parse_args()
reqs = [AptRequirement.from_str(req) for req in args.req] reqs = [AptRequirement.from_str(req) for req in args.req]
tie_breaker = BuildDependencyTieBreaker('/') tie_breaker = BuildDependencyTieBreaker("/")
print(tie_breaker(reqs)) print(tie_breaker(reqs))

76
ognibuild/debian/file_search.py
View file

@ -21,7 +21,7 @@ from datetime import datetime
from debian.deb822 import Release from debian.deb822 import Release
import os import os
import re import re
from typing import Iterator, List, Optional, Set from typing import Iterator, List
import logging import logging
@ -29,7 +29,9 @@ from .. import USER_AGENT
class FileSearcher(object): class FileSearcher(object):
def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]: def search_files(
self, path: str, regex: bool = False, case_insensitive: bool = False
) -> Iterator[str]:
raise NotImplementedError(self.search_files) raise NotImplementedError(self.search_files)
@ -66,14 +68,16 @@ def contents_urls_from_sources_entry(source, arches, load_url):
try: try:
response = load_url(release_url) response = load_url(release_url)
except FileNotFoundError as e: except FileNotFoundError as e:
logging.warning('Unable to download %s or %s: %s', inrelease_url, release_url, e) logging.warning(
"Unable to download %s or %s: %s", inrelease_url, release_url, e
)
return return
existing_names = {} existing_names = {}
release = Release(response.read()) release = Release(response.read())
for hn in ['MD5Sum', 'SHA1Sum', 'SHA256Sum']: for hn in ["MD5Sum", "SHA1Sum", "SHA256Sum"]:
for entry in release.get(hn, []): for entry in release.get(hn, []):
existing_names[os.path.splitext(entry['name'])[0]] = entry['name'] existing_names[os.path.splitext(entry["name"])[0]] = entry["name"]
contents_files = set() contents_files = set()
if components: if components:
@ -117,8 +121,7 @@ def load_direct_url(url):
for ext in [".xz", ".gz", ""]: for ext in [".xz", ".gz", ""]:
try: try:
request = Request( request = Request(url + ext, headers={"User-Agent": USER_AGENT})
url + ext, headers={"User-Agent": USER_AGENT})
response = urlopen(request) response = urlopen(request)
except HTTPError as e: except HTTPError as e:
if e.status == 404: if e.status == 404:
@ -141,16 +144,17 @@ def load_url_with_cache(url, cache_dirs):
def load_apt_cache_file(url, cache_dir): def load_apt_cache_file(url, cache_dir):
fn = apt_pkg.uri_to_filename(url) fn = apt_pkg.uri_to_filename(url)
for ext in ['.xz', '.gz', '.lz4', '']: for ext in [".xz", ".gz", ".lz4", ""]:
p = os.path.join(cache_dir, fn + ext) p = os.path.join(cache_dir, fn + ext)
if not os.path.exists(p): if not os.path.exists(p):
continue continue
# return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p) # return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p)
logging.debug("Loading cached contents file %s", p) logging.debug("Loading cached contents file %s", p)
if ext == '.lz4': if ext == ".lz4":
import lz4.frame import lz4.frame
return lz4.frame.open(p, mode="rb") return lz4.frame.open(p, mode="rb")
return _unwrap(open(p, 'rb'), ext) return _unwrap(open(p, "rb"), ext)
raise FileNotFoundError(url) raise FileNotFoundError(url)
@ -174,14 +178,15 @@ class AptCachedContentsFileSearcher(FileSearcher):
sl.load("/etc/apt/sources.list") sl.load("/etc/apt/sources.list")
from .build import get_build_architecture from .build import get_build_architecture
cache_dirs = set(["/var/lib/apt/lists"]) cache_dirs = set(["/var/lib/apt/lists"])
def load_url(url): def load_url(url):
return load_url_with_cache(url, cache_dirs) return load_url_with_cache(url, cache_dirs)
urls = list( urls = list(
contents_urls_from_sourceslist( contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)
sl, get_build_architecture(), load_url)) )
self._load_urls(urls, cache_dirs, load_url) self._load_urls(urls, cache_dirs, load_url)
def load_from_session(self, session): def load_from_session(self, session):
@ -193,16 +198,19 @@ class AptCachedContentsFileSearcher(FileSearcher):
from .build import get_build_architecture from .build import get_build_architecture
cache_dirs = set([ cache_dirs = set(
os.path.join(session.location, "var/lib/apt/lists"), [
"/var/lib/apt/lists", os.path.join(session.location, "var/lib/apt/lists"),
]) "/var/lib/apt/lists",
]
)
def load_url(url): def load_url(url):
return load_url_with_cache(url, cache_dirs) return load_url_with_cache(url, cache_dirs)
urls = list( urls = list(
contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)) contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)
)
self._load_urls(urls, cache_dirs, load_url) self._load_urls(urls, cache_dirs, load_url)
def _load_urls(self, urls, cache_dirs, load_url): def _load_urls(self, urls, cache_dirs, load_url):
@ -217,7 +225,7 @@ class AptCachedContentsFileSearcher(FileSearcher):
self._db[path] = package self._db[path] = package
def search_files(self, path, regex=False, case_insensitive=False): def search_files(self, path, regex=False, case_insensitive=False):
path = path.lstrip('/').encode('utf-8', 'surrogateescape') path = path.lstrip("/").encode("utf-8", "surrogateescape")
if case_insensitive and not regex: if case_insensitive and not regex:
regex = True regex = True
path = re.escape(path) path = re.escape(path)
@ -230,12 +238,12 @@ class AptCachedContentsFileSearcher(FileSearcher):
for p, rest in self._db.items(): for p, rest in self._db.items():
if c.match(p): if c.match(p):
pkg = rest.split(b"/")[-1] pkg = rest.split(b"/")[-1]
ret.append((p, pkg.decode('utf-8'))) ret.append((p, pkg.decode("utf-8")))
for p, pkg in sorted(ret): for p, pkg in sorted(ret):
yield pkg yield pkg
else: else:
try: try:
yield self._db[path].split(b"/")[-1].decode('utf-8') yield self._db[path].split(b"/")[-1].decode("utf-8")
except KeyError: except KeyError:
pass pass
@ -243,7 +251,7 @@ class AptCachedContentsFileSearcher(FileSearcher):
start_time = datetime.now() start_time = datetime.now()
for path, rest in read_contents_file(f.readlines()): for path, rest in read_contents_file(f.readlines()):
self[path] = rest self[path] = rest
logging.debug('Read %s in %s', url, datetime.now() - start_time) logging.debug("Read %s in %s", url, datetime.now() - start_time)
class GeneratedFileSearcher(FileSearcher): class GeneratedFileSearcher(FileSearcher):
@ -257,12 +265,14 @@ class GeneratedFileSearcher(FileSearcher):
return self return self
def load_from_path(self, path): def load_from_path(self, path):
with open(path, 'r') as f: with open(path, "r") as f:
for line in f: for line in f:
(path, pkg) = line.strip().split(None, 1) (path, pkg) = line.strip().split(None, 1)
self._db[path] = pkg self._db[path] = pkg
def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]: def search_files(
self, path: str, regex: bool = False, case_insensitive: bool = False
) -> Iterator[str]:
for p, pkg in sorted(self._db.items()): for p, pkg in sorted(self._db.items()):
if regex: if regex:
flags = 0 flags = 0
@ -294,13 +304,17 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher(
def get_packages_for_paths( def get_packages_for_paths(
paths: List[str], searchers: List[FileSearcher], regex: bool = False, paths: List[str],
case_insensitive: bool = False searchers: List[FileSearcher],
regex: bool = False,
case_insensitive: bool = False,
) -> List[str]: ) -> List[str]:
candidates: List[str] = list() candidates: List[str] = list()
for path in paths: for path in paths:
for searcher in searchers: for searcher in searchers:
for pkg in searcher.search_files(path, regex=regex, case_insensitive=case_insensitive): for pkg in searcher.search_files(
path, regex=regex, case_insensitive=case_insensitive
):
if pkg not in candidates: if pkg not in candidates:
candidates.append(pkg) candidates.append(pkg)
return candidates return candidates
@ -308,10 +322,11 @@ def get_packages_for_paths(
def main(argv): def main(argv):
import argparse import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('path', help='Path to search for.', type=str, nargs='*') parser.add_argument("path", help="Path to search for.", type=str, nargs="*")
parser.add_argument('--regex', '-x', help='Search for regex.', action='store_true') parser.add_argument("--regex", "-x", help="Search for regex.", action="store_true")
parser.add_argument('--debug', action='store_true') parser.add_argument("--debug", action="store_true")
args = parser.parse_args() args = parser.parse_args()
if args.debug: if args.debug:
@ -328,6 +343,7 @@ def main(argv):
print(package) print(package)
if __name__ == '__main__': if __name__ == "__main__":
import sys import sys
sys.exit(main(sys.argv)) sys.exit(main(sys.argv))

141
ognibuild/debian/fix_build.py
View file

@ -19,13 +19,12 @@ __all__ = [
"build_incrementally", "build_incrementally",
] ]
from datetime import datetime
from functools import partial from functools import partial
import logging import logging
import os import os
import shutil import shutil
import sys import sys
from typing import List, Set, Optional, Type, Tuple from typing import List, Set, Optional, Type
from debian.deb822 import ( from debian.deb822 import (
Deb822, Deb822,
@ -33,7 +32,6 @@ from debian.deb822 import (
) )
from breezy.commit import PointlessCommit from breezy.commit import PointlessCommit
from breezy.mutabletree import MutableTree
from breezy.tree import Tree from breezy.tree import Tree
from debmutate.changelog import ChangelogEditor from debmutate.changelog import ChangelogEditor
from debmutate.control import ( from debmutate.control import (
@ -54,18 +52,20 @@ from debmutate.reformatting import (
try: try:
from breezy.workspace import reset_tree from breezy.workspace import reset_tree
except ImportError: # breezy < 3.2 except ImportError: # breezy < 3.2
def delete_items(deletables, dry_run=False): def delete_items(deletables, dry_run=False):
"""Delete files in the deletables iterable""" """Delete files in the deletables iterable"""
import errno import errno
import shutil import shutil
def onerror(function, path, excinfo): def onerror(function, path, excinfo):
"""Show warning for errors seen by rmtree. """Show warning for errors seen by rmtree."""
"""
# Handle only permission error while removing files. # Handle only permission error while removing files.
# Other errors are re-raised. # Other errors are re-raised.
if function is not os.remove or excinfo[1].errno != errno.EACCES: if function is not os.remove or excinfo[1].errno != errno.EACCES:
raise raise
logging.warning('unable to remove %s' % path) logging.warning("unable to remove %s" % path)
for path, subp in deletables: for path, subp in deletables:
if os.path.isdir(path): if os.path.isdir(path):
shutil.rmtree(path, onerror=onerror) shutil.rmtree(path, onerror=onerror)
@ -78,13 +78,18 @@ except ImportError: # breezy < 3.2
raise e raise e
logging.warning('unable to remove "%s": %s.', path, e.strerror) logging.warning('unable to remove "%s": %s.', path, e.strerror)
def reset_tree(local_tree, subpath=''): def reset_tree(local_tree, subpath=""):
from breezy.transform import revert from breezy.transform import revert
from breezy.clean_tree import iter_deletables from breezy.clean_tree import iter_deletables
revert(local_tree, local_tree.branch.basis_tree(),
[subpath] if subpath not in ('.', '') else None) revert(
deletables = list(iter_deletables( local_tree,
local_tree, unknown=True, ignored=False, detritus=False)) local_tree.branch.basis_tree(),
[subpath] if subpath not in (".", "") else None,
)
deletables = list(
iter_deletables(local_tree, unknown=True, ignored=False, detritus=False)
)
delete_items(deletables) delete_items(deletables)
@ -103,8 +108,6 @@ from buildlog_consultant.common import (
MissingAutomakeInput, MissingAutomakeInput,
MissingConfigure, MissingConfigure,
NeedPgBuildExtUpdateControl, NeedPgBuildExtUpdateControl,
MissingPythonModule,
MissingPythonDistribution,
MissingPerlFile, MissingPerlFile,
) )
from buildlog_consultant.sbuild import ( from buildlog_consultant.sbuild import (
@ -130,8 +133,9 @@ class CircularDependency(Exception):
class DebianPackagingContext(object): class DebianPackagingContext(object):
def __init__(
def __init__(self, tree, subpath, committer, update_changelog, commit_reporter=None): self, tree, subpath, committer, update_changelog, commit_reporter=None
):
self.tree = tree self.tree = tree
self.subpath = subpath self.subpath = subpath
self.committer = committer self.committer = committer
@ -144,14 +148,18 @@ class DebianPackagingContext(object):
with self.tree.lock_write(): with self.tree.lock_write():
try: try:
if update_changelog: if update_changelog:
cl_path = self.tree.abspath(os.path.join(self.subpath, "debian/changelog")) cl_path = self.tree.abspath(
os.path.join(self.subpath, "debian/changelog")
)
with ChangelogEditor(cl_path) as editor: with ChangelogEditor(cl_path) as editor:
editor.add_entry([summary]) editor.add_entry([summary])
debcommit(self.tree, committer=self.committer, subpath=self.subpath) debcommit(self.tree, committer=self.committer, subpath=self.subpath)
else: else:
self.tree.commit( self.tree.commit(
message=summary, committer=self.committer, specific_files=[self.subpath], message=summary,
reporter=self.commit_reporter committer=self.committer,
specific_files=[self.subpath],
reporter=self.commit_reporter,
) )
except PointlessCommit: except PointlessCommit:
return False return False
@ -160,7 +168,6 @@ class DebianPackagingContext(object):
class PackageDependencyFixer(BuildFixer): class PackageDependencyFixer(BuildFixer):
def __init__(self, context, apt_resolver): def __init__(self, context, apt_resolver):
self.apt_resolver = apt_resolver self.apt_resolver = apt_resolver
self.context = context self.context = context
@ -203,7 +210,7 @@ def add_dependency(context, phase, requirement: AptRequirement):
elif phase[0] == "build": elif phase[0] == "build":
return add_build_dependency(context, requirement) return add_build_dependency(context, requirement)
else: else:
logging.warning('Unknown phase %r', phase) logging.warning("Unknown phase %r", phase)
return False return False
@ -239,7 +246,9 @@ def add_test_dependency(context, testname, requirement):
if not isinstance(requirement, AptRequirement): if not isinstance(requirement, AptRequirement):
raise TypeError(requirement) raise TypeError(requirement)
tests_control_path = os.path.join(context.tree.abspath(context.subpath), "debian/tests/control") tests_control_path = os.path.join(
context.tree.abspath(context.subpath), "debian/tests/control"
)
try: try:
with Deb822Editor(path=tests_control_path) as updater: with Deb822Editor(path=tests_control_path) as updater:
@ -293,18 +302,22 @@ def python_tie_breaker(tree, subpath, reqs):
return None return None
def same(pkg, python_version): def same(pkg, python_version):
if pkg.startswith(python_version + '-'): if pkg.startswith(python_version + "-"):
return True return True
if pkg.startswith('lib%s-' % python_version): if pkg.startswith("lib%s-" % python_version):
return True return True
return False return False
for python_version in targeted: for python_version in targeted:
for req in reqs: for req in reqs:
if any(same(name, python_version) for name in req.package_names()): if any(same(name, python_version) for name in req.package_names()):
logging.info( logging.info(
'Breaking tie between %r to %r, since package already ' "Breaking tie between %r to %r, since package already "
'has %r build-dependencies', [str(req) for req in reqs], "has %r build-dependencies",
str(req), python_version) [str(req) for req in reqs],
str(req),
python_version,
)
return req return req
return None return None
@ -327,7 +340,9 @@ def enable_dh_autoreconf(context, phase):
return dh_invoke_add_with(line, b"autoreconf") return dh_invoke_add_with(line, b"autoreconf")
if update_rules(command_line_cb=add_with_autoreconf): if update_rules(command_line_cb=add_with_autoreconf):
return add_dependency(context, phase, AptRequirement.simple("dh-autoreconf")) return add_dependency(
context, phase, AptRequirement.simple("dh-autoreconf")
)
return False return False
@ -379,16 +394,18 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer):
return isinstance(problem, NeedPgBuildExtUpdateControl) return isinstance(problem, NeedPgBuildExtUpdateControl)
def __repr__(self): def __repr__(self):
return "%s()" % (type(self).__name__, ) return "%s()" % (type(self).__name__,)
def _fix(self, error, context): def _fix(self, error, context):
logging.info("Running 'pg_buildext updatecontrol'") logging.info("Running 'pg_buildext updatecontrol'")
self.session.check_call(["pg_buildext", "updatecontrol"]) self.session.check_call(["pg_buildext", "updatecontrol"])
shutil.copy( shutil.copy(
self.session.external_path('debian/control'), self.session.external_path("debian/control"),
context.tree.abspath(os.path.join(context.subpath, 'debian/control'))) context.tree.abspath(os.path.join(context.subpath, "debian/control")),
)
return self.context.commit( return self.context.commit(
"Run 'pgbuildext updatecontrol'.", update_changelog=False) "Run 'pgbuildext updatecontrol'.", update_changelog=False
)
def fix_missing_makefile_pl(error, phase, context): def fix_missing_makefile_pl(error, phase, context):
@ -410,7 +427,10 @@ class SimpleBuildFixer(BuildFixer):
def __repr__(self): def __repr__(self):
return "%s(%s, %s)" % ( return "%s(%s, %s)" % (
type(self).__name__, self._problem_cls.__name__, self._fn.__name__) type(self).__name__,
self._problem_cls.__name__,
self._fn.__name__,
)
def can_fix(self, problem: Problem): def can_fix(self, problem: Problem):
return isinstance(problem, self._problem_cls) return isinstance(problem, self._problem_cls)
@ -428,7 +448,10 @@ class DependencyBuildFixer(BuildFixer):
def __repr__(self): def __repr__(self):
return "%s(%s, %s)" % ( return "%s(%s, %s)" % (
type(self).__name__, self._problem_cls.__name__, self._fn.__name__) type(self).__name__,
self._problem_cls.__name__,
self._fn.__name__,
)
def can_fix(self, problem: Problem): def can_fix(self, problem: Problem):
return isinstance(problem, self._problem_cls) return isinstance(problem, self._problem_cls)
@ -441,8 +464,12 @@ def versioned_package_fixers(session, packaging_context):
return [ return [
PgBuildExtOutOfDateControlFixer(packaging_context, session), PgBuildExtOutOfDateControlFixer(packaging_context, session),
SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure), SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure),
SimpleBuildFixer(packaging_context, MissingAutomakeInput, fix_missing_automake_input), SimpleBuildFixer(
SimpleBuildFixer(packaging_context, MissingConfigStatusInput, fix_missing_config_status_input), packaging_context, MissingAutomakeInput, fix_missing_automake_input
),
SimpleBuildFixer(
packaging_context, MissingConfigStatusInput, fix_missing_config_status_input
),
SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl), SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl),
] ]
@ -451,14 +478,17 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]:
from ..resolver.apt import AptResolver from ..resolver.apt import AptResolver
from .udd import popcon_tie_breaker from .udd import popcon_tie_breaker
from .build_deps import BuildDependencyTieBreaker from .build_deps import BuildDependencyTieBreaker
apt_tie_breakers = [ apt_tie_breakers = [
partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath), partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath),
BuildDependencyTieBreaker.from_session(apt.session), BuildDependencyTieBreaker.from_session(apt.session),
popcon_tie_breaker, popcon_tie_breaker,
] ]
resolver = AptResolver(apt, apt_tie_breakers) resolver = AptResolver(apt, apt_tie_breakers)
return [ return [
DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure), DependencyBuildFixer(
packaging_context, apt, AptFetchFailure, retry_apt_failure
),
PackageDependencyFixer(packaging_context, resolver), PackageDependencyFixer(packaging_context, resolver),
] ]
@ -479,9 +509,11 @@ def build_incrementally(
): ):
fixed_errors = [] fixed_errors = []
packaging_context = DebianPackagingContext( packaging_context = DebianPackagingContext(
local_tree, subpath, committer, update_changelog) local_tree, subpath, committer, update_changelog
fixers = (versioned_package_fixers(apt.session, packaging_context) + )
apt_fixers(apt, packaging_context)) fixers = versioned_package_fixers(apt.session, packaging_context) + apt_fixers(
apt, packaging_context
)
logging.info("Using fixers: %r", fixers) logging.info("Using fixers: %r", fixers)
while True: while True:
try: try:
@ -534,11 +566,8 @@ def build_incrementally(
): ):
i += 1 i += 1
target_path = os.path.join(output_directory, "build.log.%d" % i) target_path = os.path.join(output_directory, "build.log.%d" % i)
os.rename( os.rename(os.path.join(output_directory, "build.log"), target_path)
os.path.join(output_directory, "build.log"), logging.debug("Storing build log at %s", target_path)
target_path
)
logging.debug('Storing build log at %s', target_path)
def main(argv=None): def main(argv=None):
@ -577,19 +606,13 @@ def main(argv=None):
help="force updating of the changelog", help="force updating of the changelog",
default=None, default=None,
) )
parser.add_argument( parser.add_argument("--schroot", type=str, help="chroot to use.")
'--schroot', parser.add_argument("--verbose", action="store_true", help="Be verbose")
type=str,
help='chroot to use.')
parser.add_argument(
'--verbose',
action='store_true',
help='Be verbose')
args = parser.parse_args() args = parser.parse_args()
from breezy.workingtree import WorkingTree from breezy.workingtree import WorkingTree
import breezy.git import breezy.git # noqa: F401
import breezy.bzr import breezy.bzr # noqa: F401
from .apt import AptManager from .apt import AptManager
from ..session.plain import PlainSession from ..session.plain import PlainSession
from ..session.schroot import SchrootSession from ..session.schroot import SchrootSession
@ -632,15 +655,15 @@ def main(argv=None):
) )
except SbuildFailure as e: except SbuildFailure as e:
if e.phase is None: if e.phase is None:
phase = 'unknown phase' phase = "unknown phase"
elif len(e.phase) == 1: elif len(e.phase) == 1:
phase = e.phase[0] phase = e.phase[0]
else: else:
phase = '%s (%s)' % (e.phase[0], e.phase[1]) phase = "%s (%s)" % (e.phase[0], e.phase[1])
if e.error: if e.error:
logging.fatal('Error during %s: %s', phase, e.error) logging.fatal("Error during %s: %s", phase, e.error)
else: else:
logging.fatal('Error during %s: %s', phase, e.description) logging.fatal("Error during %s: %s", phase, e.description)
return 1 return 1

13
ognibuild/debian/udd.py
View file

@ -21,7 +21,6 @@ import logging
class UDD(object): class UDD(object):
def connect(self): def connect(self):
import psycopg2 import psycopg2
@ -36,8 +35,9 @@ class UDD(object):
def get_most_popular(self, packages): def get_most_popular(self, packages):
cursor = self._conn.cursor() cursor = self._conn.cursor()
cursor.execute( cursor.execute(
'SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1', "SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1",
(tuple(packages), )) (tuple(packages),),
)
return cursor.fetchone()[0] return cursor.fetchone()[0]
@ -47,15 +47,14 @@ def popcon_tie_breaker(candidates):
try: try:
from .udd import UDD from .udd import UDD
except ModuleNotFoundError: except ModuleNotFoundError:
logging.warning('Unable to import UDD, not ranking by popcon') logging.warning("Unable to import UDD, not ranking by popcon")
return sorted(candidates, key=len)[0] return sorted(candidates, key=len)[0]
udd = UDD() udd = UDD()
udd.connect() udd.connect()
names = {list(c.package_names())[0]: c for c in candidates} names = {list(c.package_names())[0]: c for c in candidates}
winner = udd.get_most_popular(list(names.keys())) winner = udd.get_most_popular(list(names.keys()))
if winner is None: if winner is None:
logging.warning( logging.warning("No relevant popcon information found, not ranking by popcon")
'No relevant popcon information found, not ranking by popcon')
return None return None
logging.info('Picked winner using popcon') logging.info("Picked winner using popcon")
return names[winner] return names[winner]

53
ognibuild/dist.py
View file

@ -16,16 +16,15 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
__all__ = [ __all__ = [
'UnidentifiedError', "UnidentifiedError",
'DetailedFailure', "DetailedFailure",
'create_dist', "create_dist",
'create_dist_schroot', "create_dist_schroot",
] ]
import errno import errno
import logging import logging
import os import os
import shutil
import sys import sys
from typing import Optional, List from typing import Optional, List
@ -54,7 +53,8 @@ def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=Fa
for buildsystem in buildsystems: for buildsystem in buildsystems:
filename = buildsystem.dist( filename = buildsystem.dist(
session, resolver, fixers, target_directory, quiet=quiet) session, resolver, fixers, target_directory, quiet=quiet
)
return filename return filename
raise NoBuildToolsFound() raise NoBuildToolsFound()
@ -66,20 +66,23 @@ def create_dist(
target_dir: str, target_dir: str,
include_controldir: bool = True, include_controldir: bool = True,
subdir: Optional[str] = None, subdir: Optional[str] = None,
cleanup: bool = False cleanup: bool = False,
) -> Optional[str]: ) -> Optional[str]:
from .buildsystem import detect_buildsystems from .buildsystem import detect_buildsystems
from .buildlog import InstallFixer from .buildlog import InstallFixer
from .fix_build import BuildFixer from .fix_build import BuildFixer
from .fixers import ( from .fixers import (
GitIdentityFixer, SecretGpgKeyFixer, GitIdentityFixer,
UnexpandedAutoconfMacroFixer, ) SecretGpgKeyFixer,
UnexpandedAutoconfMacroFixer,
)
if subdir is None: if subdir is None:
subdir = "package" subdir = "package"
try: try:
export_directory, reldir = session.setup_from_vcs( export_directory, reldir = session.setup_from_vcs(
tree, include_controldir=include_controldir, subdir=subdir) tree, include_controldir=include_controldir, subdir=subdir
)
except OSError as e: except OSError as e:
if e.errno == errno.ENOSPC: if e.errno == errno.ENOSPC:
raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice()) raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice())
@ -94,8 +97,7 @@ def create_dist(
if session.is_temporary: if session.is_temporary:
# Only muck about with temporary sessions # Only muck about with temporary sessions
fixers.extend([ fixers.extend([GitIdentityFixer(session), SecretGpgKeyFixer(session)])
GitIdentityFixer(session), SecretGpgKeyFixer(session)])
session.chdir(reldir) session.chdir(reldir)
return run_dist(session, buildsystems, resolver, fixers, target_dir) return run_dist(session, buildsystems, resolver, fixers, target_dir)
@ -109,7 +111,7 @@ def create_dist_schroot(
packaging_subpath: Optional[str] = None, packaging_subpath: Optional[str] = None,
include_controldir: bool = True, include_controldir: bool = True,
subdir: Optional[str] = None, subdir: Optional[str] = None,
cleanup: bool = False cleanup: bool = False,
) -> Optional[str]: ) -> Optional[str]:
with SchrootSession(chroot) as session: with SchrootSession(chroot) as session:
if packaging_tree is not None: if packaging_tree is not None:
@ -117,10 +119,13 @@ def create_dist_schroot(
satisfy_build_deps(session, packaging_tree, packaging_subpath) satisfy_build_deps(session, packaging_tree, packaging_subpath)
return create_dist( return create_dist(
session, tree, target_dir, session,
tree,
target_dir,
include_controldir=include_controldir, include_controldir=include_controldir,
subdir=subdir, subdir=subdir,
cleanup=cleanup) cleanup=cleanup,
)
if __name__ == "__main__": if __name__ == "__main__":
@ -151,8 +156,8 @@ if __name__ == "__main__":
) )
parser.add_argument("--verbose", action="store_true", help="Be verbose") parser.add_argument("--verbose", action="store_true", help="Be verbose")
parser.add_argument( parser.add_argument(
'--include-controldir', action='store_true', "--include-controldir", action="store_true", help="Clone rather than export."
help='Clone rather than export.') )
args = parser.parse_args() args = parser.parse_args()
@ -185,15 +190,17 @@ if __name__ == "__main__":
logging.info("No build tools found, falling back to simple export.") logging.info("No build tools found, falling back to simple export.")
export(tree, "dist.tar.gz", "tgz", None) export(tree, "dist.tar.gz", "tgz", None)
except NotImplementedError: except NotImplementedError:
logging.info("Build system does not support dist tarball creation, " logging.info(
"falling back to simple export.") "Build system does not support dist tarball creation, "
"falling back to simple export."
)
export(tree, "dist.tar.gz", "tgz", None) export(tree, "dist.tar.gz", "tgz", None)
except UnidentifiedError as e: except UnidentifiedError as e:
logging.fatal('Unidentified error: %r', e.lines) logging.fatal("Unidentified error: %r", e.lines)
except DetailedFailure as e: except DetailedFailure as e:
logging.fatal('Identified error during dist creation: %s', e.error) logging.fatal("Identified error during dist creation: %s", e.error)
except DistNoTarball: except DistNoTarball:
logging.fatal('dist operation did not create a tarball') logging.fatal("dist operation did not create a tarball")
else: else:
logging.info("Created %s", ret) logging.info("Created %s", ret)
sys.exit(0) sys.exit(0)

13
ognibuild/dist_catcher.py
View file

@ -53,17 +53,17 @@ class DistCatcher(object):
@classmethod @classmethod
def default(cls, directory): def default(cls, directory):
return cls([ return cls(
os.path.join(directory, 'dist'), [os.path.join(directory, "dist"), directory, os.path.join(directory, "..")]
directory, )
os.path.join(directory, '..')])
def __enter__(self): def __enter__(self):
self.existing_files = {} self.existing_files = {}
for directory in self.directories: for directory in self.directories:
try: try:
self.existing_files[directory] = { self.existing_files[directory] = {
entry.name: entry for entry in os.scandir(directory)} entry.name: entry for entry in os.scandir(directory)
}
except FileNotFoundError: except FileNotFoundError:
self.existing_files[directory] = {} self.existing_files[directory] = {}
return self return self
@ -92,7 +92,8 @@ class DistCatcher(object):
return entry.name return entry.name
elif len(possible_new) > 1: elif len(possible_new) > 1:
logging.warning( logging.warning(
"Found multiple tarballs %r in %s.", possible_new, directory) "Found multiple tarballs %r in %s.", possible_new, directory
)
return return
if len(possible_updated) == 1: if len(possible_updated) == 1:

20
ognibuild/fix_build.py
View file

@ -17,17 +17,15 @@
from functools import partial from functools import partial
import logging import logging
from typing import List, Optional, Tuple, Callable, Any from typing import List, Tuple, Callable, Any
from buildlog_consultant import Problem from buildlog_consultant import Problem
from buildlog_consultant.common import ( from buildlog_consultant.common import (
find_build_failure_description, find_build_failure_description,
MissingCommand, MissingCommand,
) )
from breezy.mutabletree import MutableTree
from . import DetailedFailure, UnidentifiedError from . import DetailedFailure, UnidentifiedError
from .debian.apt import AptManager
from .session import Session, run_with_tee from .session import Session, run_with_tee
@ -55,7 +53,7 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs):
else: else:
if retcode == 0: if retcode == 0:
return contents return contents
lines = ''.join(contents).splitlines(False) lines = "".join(contents).splitlines(False)
match, error = find_build_failure_description(lines) match, error = find_build_failure_description(lines)
if error is None: if error is None:
if match: if match:
@ -91,22 +89,26 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]):
try: try:
resolved = resolve_error(f.error, None, fixers=fixers) resolved = resolve_error(f.error, None, fixers=fixers)
except DetailedFailure as n: except DetailedFailure as n:
logging.info('New error %r while resolving %r', n, f) logging.info("New error %r while resolving %r", n, f)
if n in to_resolve: if n in to_resolve:
raise raise
to_resolve.append(f) to_resolve.append(f)
to_resolve.append(n) to_resolve.append(n)
else: else:
if not resolved: if not resolved:
logging.warning("Failed to find resolution for error %r. Giving up.", f.error) logging.warning(
"Failed to find resolution for error %r. Giving up.", f.error
)
raise f raise f
fixed_errors.append(f.error) fixed_errors.append(f.error)
def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs): def run_with_build_fixers(
session: Session, args: List[str], fixers: List[BuildFixer], **kwargs
):
return iterate_with_build_fixers( return iterate_with_build_fixers(
fixers, fixers, partial(run_detecting_problems, session, args, **kwargs)
partial(run_detecting_problems, session, args, **kwargs)) )
def resolve_error(error, phase, fixers): def resolve_error(error, phase, fixers):

25
ognibuild/fixers.py
View file

@ -23,7 +23,7 @@ from buildlog_consultant.common import (
MissingGitIdentity, MissingGitIdentity,
MissingSecretGpgKey, MissingSecretGpgKey,
MissingAutoconfMacro, MissingAutoconfMacro,
) )
from ognibuild.requirements import AutoconfMacroRequirement from ognibuild.requirements import AutoconfMacroRequirement
from ognibuild.resolver import UnsatisfiedRequirements from ognibuild.resolver import UnsatisfiedRequirements
@ -31,7 +31,6 @@ from .fix_build import BuildFixer
class GitIdentityFixer(BuildFixer): class GitIdentityFixer(BuildFixer):
def __init__(self, session): def __init__(self, session):
self.session = session self.session = session
@ -39,16 +38,17 @@ class GitIdentityFixer(BuildFixer):
return isinstance(problem, MissingGitIdentity) return isinstance(problem, MissingGitIdentity)
def _fix(self, problem: Problem, phase: Tuple[str, ...]): def _fix(self, problem: Problem, phase: Tuple[str, ...]):
for name in ['user.email', 'user.name']: for name in ["user.email", "user.name"]:
value = subprocess.check_output( value = (
['git', 'config', '--global', name]).decode().strip() subprocess.check_output(["git", "config", "--global", name])
self.session.check_call( .decode()
['git', 'config', '--global', name, value]) .strip()
)
self.session.check_call(["git", "config", "--global", name, value])
return True return True
class SecretGpgKeyFixer(BuildFixer): class SecretGpgKeyFixer(BuildFixer):
def __init__(self, session): def __init__(self, session):
self.session = session self.session = session
@ -67,8 +67,10 @@ Expire-Date: 0
Passphrase: "" Passphrase: ""
""" """
p = self.session.Popen( p = self.session.Popen(
['gpg', '--gen-key', '--batch', '/dev/stdin'], ["gpg", "--gen-key", "--batch", "/dev/stdin"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE) stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
p.communicate(SCRIPT) p.communicate(SCRIPT)
if p.returncode == 0: if p.returncode == 0:
return True return True
@ -76,7 +78,6 @@ Passphrase: ""
class UnexpandedAutoconfMacroFixer(BuildFixer): class UnexpandedAutoconfMacroFixer(BuildFixer):
def __init__(self, session, resolver): def __init__(self, session, resolver):
self.session = session self.session = session
self.resolver = resolver self.resolver = resolver
@ -97,6 +98,6 @@ class UnexpandedAutoconfMacroFixer(BuildFixer):
return False return False
from .fix_build import run_detecting_problems from .fix_build import run_detecting_problems
run_detecting_problems(self.session, ['autoconf', '-f']) run_detecting_problems(self.session, ["autoconf", "-f"])
return True return True

75
ognibuild/requirements.py
View file

@ -19,7 +19,7 @@
import posixpath import posixpath
import re import re
import subprocess import subprocess
from typing import Optional, List, Tuple, Set from typing import Optional, List, Set
from . import Requirement from . import Requirement
@ -72,19 +72,24 @@ class PythonPackageRequirement(Requirement):
cmd = "python3" cmd = "python3"
else: else:
raise NotImplementedError raise NotImplementedError
text = self.package + ','.join([''.join(spec) for spec in self.specs]) text = self.package + ",".join(["".join(spec) for spec in self.specs])
p = session.Popen( p = session.Popen(
[cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text], [cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.communicate() p.communicate()
return p.returncode == 0 return p.returncode == 0
class PhpPackageRequirement(Requirement): class PhpPackageRequirement(Requirement):
def __init__(
def __init__(self, package: str, channel: Optional[str] = None, self,
min_version: Optional[str] = None, package: str,
max_version: Optional[str] = None): channel: Optional[str] = None,
min_version: Optional[str] = None,
max_version: Optional[str] = None,
):
self.package = package self.package = package
self.channel = channel self.channel = channel
self.min_version = min_version self.min_version = min_version
@ -92,8 +97,12 @@ class PhpPackageRequirement(Requirement):
def __repr__(self): def __repr__(self):
return "%s(%r, %r, %r, %r)" % ( return "%s(%r, %r, %r, %r)" % (
type(self).__name__, self.package, self.channel, type(self).__name__,
self.min_version, self.max_version) self.package,
self.channel,
self.min_version,
self.max_version,
)
class BinaryRequirement(Requirement): class BinaryRequirement(Requirement):
@ -109,8 +118,10 @@ class BinaryRequirement(Requirement):
def met(self, session): def met(self, session):
p = session.Popen( p = session.Popen(
["which", self.binary_name], stdout=subprocess.DEVNULL, ["which", self.binary_name],
stderr=subprocess.DEVNULL) stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.communicate() p.communicate()
return p.returncode == 0 return p.returncode == 0
@ -144,7 +155,7 @@ class VagueDependencyRequirement(Requirement):
self.name = name self.name = name
def expand(self): def expand(self):
if ' ' not in self.name: if " " not in self.name:
yield BinaryRequirement(self.name) yield BinaryRequirement(self.name)
yield LibraryRequirement(self.name) yield LibraryRequirement(self.name)
yield PkgConfigRequirement(self.name) yield PkgConfigRequirement(self.name)
@ -153,6 +164,7 @@ class VagueDependencyRequirement(Requirement):
yield LibraryRequirement(self.name.lower()) yield LibraryRequirement(self.name.lower())
yield PkgConfigRequirement(self.name.lower()) yield PkgConfigRequirement(self.name.lower())
from .resolver.apt import AptRequirement from .resolver.apt import AptRequirement
yield AptRequirement.simple(self.name.lower()) yield AptRequirement.simple(self.name.lower())
def met(self, session): def met(self, session):
@ -208,17 +220,18 @@ class CargoCrateRequirement(Requirement):
type(self).__name__, type(self).__name__,
self.crate, self.crate,
self.features, self.features,
self.version self.version,
) )
def __str__(self): def __str__(self):
if self.features: if self.features:
return "cargo crate: %s %s (%s)" % ( return "cargo crate: %s %s (%s)" % (
self.crate, self.version or '', self.crate,
', '.join(sorted(self.features))) self.version or "",
", ".join(sorted(self.features)),
)
else: else:
return "cargo crate: %s %s" % ( return "cargo crate: %s %s" % (self.crate, self.version or "")
self.crate, self.version or '')
class PkgConfigRequirement(Requirement): class PkgConfigRequirement(Requirement):
@ -346,10 +359,10 @@ class RPackageRequirement(Requirement):
@classmethod @classmethod
def from_str(cls, text): def from_str(cls, text):
# TODO(jelmer): More complex parser # TODO(jelmer): More complex parser
m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text) m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text)
if m: if m:
return cls(m.group(1), m.group(2)) return cls(m.group(1), m.group(2))
m = re.fullmatch(r'([^ ]+)', text) m = re.fullmatch(r"([^ ]+)", text)
if m: if m:
return cls(m.group(1)) return cls(m.group(1))
raise ValueError(text) raise ValueError(text)
@ -381,10 +394,10 @@ class OctavePackageRequirement(Requirement):
@classmethod @classmethod
def from_str(cls, text): def from_str(cls, text):
# TODO(jelmer): More complex parser # TODO(jelmer): More complex parser
m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text) m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text)
if m: if m:
return cls(m.group(1), m.group(2)) return cls(m.group(1), m.group(2))
m = re.fullmatch(r'([^ ]+)', text) m = re.fullmatch(r"([^ ]+)", text)
if m: if m:
return cls(m.group(1)) return cls(m.group(1))
raise ValueError(text) raise ValueError(text)
@ -468,11 +481,14 @@ class MavenArtifactRequirement(Requirement):
def __str__(self): def __str__(self):
return "maven requirement: %s:%s:%s" % ( return "maven requirement: %s:%s:%s" % (
self.group_id, self.artifact_id, self.version) self.group_id,
self.artifact_id,
self.version,
)
@classmethod @classmethod
def from_str(cls, text): def from_str(cls, text):
return cls.from_tuple(text.split(':')) return cls.from_tuple(text.split(":"))
@classmethod @classmethod
def from_tuple(cls, parts): def from_tuple(cls, parts):
@ -486,8 +502,7 @@ class MavenArtifactRequirement(Requirement):
(group_id, artifact_id) = parts (group_id, artifact_id) = parts
kind = "jar" kind = "jar"
else: else:
raise ValueError( raise ValueError("invalid number of parts to artifact %r" % parts)
"invalid number of parts to artifact %r" % parts)
return cls(group_id, artifact_id, version, kind) return cls(group_id, artifact_id, version, kind)
@ -512,31 +527,26 @@ class JDKFileRequirement(Requirement):
class JDKRequirement(Requirement): class JDKRequirement(Requirement):
def __init__(self): def __init__(self):
super(JDKRequirement, self).__init__("jdk") super(JDKRequirement, self).__init__("jdk")
class JRERequirement(Requirement): class JRERequirement(Requirement):
def __init__(self): def __init__(self):
super(JRERequirement, self).__init__("jre") super(JRERequirement, self).__init__("jre")
class QTRequirement(Requirement): class QTRequirement(Requirement):
def __init__(self): def __init__(self):
super(QTRequirement, self).__init__("qt") super(QTRequirement, self).__init__("qt")
class X11Requirement(Requirement): class X11Requirement(Requirement):
def __init__(self): def __init__(self):
super(X11Requirement, self).__init__("x11") super(X11Requirement, self).__init__("x11")
class CertificateAuthorityRequirement(Requirement): class CertificateAuthorityRequirement(Requirement):
def __init__(self, url): def __init__(self, url):
super(CertificateAuthorityRequirement, self).__init__("ca-cert") super(CertificateAuthorityRequirement, self).__init__("ca-cert")
self.url = url self.url = url
@ -561,7 +571,6 @@ class AutoconfMacroRequirement(Requirement):
class LibtoolRequirement(Requirement): class LibtoolRequirement(Requirement):
def __init__(self): def __init__(self):
super(LibtoolRequirement, self).__init__("libtool") super(LibtoolRequirement, self).__init__("libtool")
@ -593,6 +602,8 @@ class PythonModuleRequirement(Requirement):
raise NotImplementedError raise NotImplementedError
p = session.Popen( p = session.Popen(
[cmd, "-c", "import %s" % self.module], [cmd, "-c", "import %s" % self.module],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.communicate() p.communicate()
return p.returncode == 0 return p.returncode == 0

43
ognibuild/resolver/__init__.py
View file

@ -19,6 +19,7 @@
import subprocess import subprocess
from ..fix_build import run_detecting_problems from ..fix_build import run_detecting_problems
class UnsatisfiedRequirements(Exception): class UnsatisfiedRequirements(Exception):
def __init__(self, reqs): def __init__(self, reqs):
self.requirements = reqs self.requirements = reqs
@ -53,7 +54,7 @@ class CPANResolver(Resolver):
def _cmd(self, reqs): def _cmd(self, reqs):
ret = ["cpan", "-i"] ret = ["cpan", "-i"]
if self.skip_tests: if self.skip_tests:
ret.append('-T') ret.append("-T")
ret.extend([req.module for req in reqs]) ret.extend([req.module for req in reqs])
return ret return ret
@ -75,7 +76,7 @@ class CPANResolver(Resolver):
"PERL_MM_USE_DEFAULT": "1", "PERL_MM_USE_DEFAULT": "1",
"PERL_MM_OPT": "", "PERL_MM_OPT": "",
"PERL_MB_OPT": "", "PERL_MB_OPT": "",
} }
if not self.user_local: if not self.user_local:
user = "root" user = "root"
@ -111,7 +112,11 @@ class RResolver(Resolver):
def _cmd(self, req): def _cmd(self, req):
# TODO(jelmer: Handle self.user_local # TODO(jelmer: Handle self.user_local
return ["R", "-e", "install.packages('%s', repos=%r)" % (req.package, self.repos)] return [
"R",
"-e",
"install.packages('%s', repos=%r)" % (req.package, self.repos),
]
def explain(self, requirements): def explain(self, requirements):
from ..requirements import RPackageRequirement from ..requirements import RPackageRequirement
@ -187,16 +192,17 @@ class OctaveForgeResolver(Resolver):
class CRANResolver(RResolver): class CRANResolver(RResolver):
def __init__(self, session, user_local=False): def __init__(self, session, user_local=False):
super(CRANResolver, self).__init__(session, 'http://cran.r-project.org', user_local=user_local) super(CRANResolver, self).__init__(
session, "http://cran.r-project.org", user_local=user_local
)
class BioconductorResolver(RResolver): class BioconductorResolver(RResolver):
def __init__(self, session, user_local=False): def __init__(self, session, user_local=False):
super(BioconductorResolver, self).__init__( super(BioconductorResolver, self).__init__(
session, 'https://hedgehog.fhcrc.org/bioconductor', user_local=user_local) session, "https://hedgehog.fhcrc.org/bioconductor", user_local=user_local
)
class HackageResolver(Resolver): class HackageResolver(Resolver):
@ -213,7 +219,7 @@ class HackageResolver(Resolver):
def _cmd(self, reqs): def _cmd(self, reqs):
extra_args = [] extra_args = []
if self.user_local: if self.user_local:
extra_args.append('--user') extra_args.append("--user")
return ["cabal", "install"] + extra_args + [req.package for req in reqs] return ["cabal", "install"] + extra_args + [req.package for req in reqs]
def install(self, requirements): def install(self, requirements):
@ -259,7 +265,7 @@ class PypiResolver(Resolver):
def _cmd(self, reqs): def _cmd(self, reqs):
extra_args = [] extra_args = []
if self.user_local: if self.user_local:
extra_args.append('--user') extra_args.append("--user")
return ["pip", "install"] + extra_args + [req.package for req in reqs] return ["pip", "install"] + extra_args + [req.package for req in reqs]
def install(self, requirements): def install(self, requirements):
@ -276,8 +282,7 @@ class PypiResolver(Resolver):
missing.append(requirement) missing.append(requirement)
continue continue
try: try:
self.session.check_call( self.session.check_call(self._cmd([requirement]), user=user)
self._cmd([requirement]), user=user)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
missing.append(requirement) missing.append(requirement)
if missing: if missing:
@ -296,7 +301,6 @@ class PypiResolver(Resolver):
class GoResolver(Resolver): class GoResolver(Resolver):
def __init__(self, session, user_local): def __init__(self, session, user_local):
self.session = session self.session = session
self.user_local = user_local self.user_local = user_local
@ -314,7 +318,7 @@ class GoResolver(Resolver):
env = {} env = {}
else: else:
# TODO(jelmer): Isn't this Debian-specific? # TODO(jelmer): Isn't this Debian-specific?
env = {'GOPATH': '/usr/share/gocode'} env = {"GOPATH": "/usr/share/gocode"}
missing = [] missing = []
for requirement in requirements: for requirement in requirements:
@ -334,8 +338,7 @@ class GoResolver(Resolver):
continue continue
goreqs.append(requirement) goreqs.append(requirement)
if goreqs: if goreqs:
yield (["go", "get"] + [req.package for req in goreqs], yield (["go", "get"] + [req.package for req in goreqs], goreqs)
goreqs)
NPM_COMMAND_PACKAGES = { NPM_COMMAND_PACKAGES = {
@ -361,7 +364,7 @@ class NpmResolver(Resolver):
NodePackageRequirement, NodePackageRequirement,
NodeModuleRequirement, NodeModuleRequirement,
BinaryRequirement, BinaryRequirement,
) )
if self.user_local: if self.user_local:
user = None user = None
@ -379,13 +382,13 @@ class NpmResolver(Resolver):
requirement = NodePackageRequirement(package) requirement = NodePackageRequirement(package)
if isinstance(requirement, NodeModuleRequirement): if isinstance(requirement, NodeModuleRequirement):
# TODO: Is this legit? # TODO: Is this legit?
requirement = NodePackageRequirement(requirement.module.split('/')[0]) requirement = NodePackageRequirement(requirement.module.split("/")[0])
if not isinstance(requirement, NodePackageRequirement): if not isinstance(requirement, NodePackageRequirement):
missing.append(requirement) missing.append(requirement)
continue continue
self.session.check_call( self.session.check_call(
["npm", "-g", "install", requirement.package], ["npm", "-g", "install", requirement.package], user=user
user=user) )
if missing: if missing:
raise UnsatisfiedRequirements(missing) raise UnsatisfiedRequirements(missing)
@ -449,7 +452,7 @@ NATIVE_RESOLVER_CLS = [
CRANResolver, CRANResolver,
BioconductorResolver, BioconductorResolver,
OctaveForgeResolver, OctaveForgeResolver,
] ]
def native_resolvers(session, user_local): def native_resolvers(session, user_local):

264
ognibuild/resolver/apt.py
View file

@ -115,19 +115,27 @@ class AptRequirement(Requirement):
return False return False
def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False) -> List[str]: def find_package_names(
apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False
) -> List[str]:
if not isinstance(paths, list): if not isinstance(paths, list):
raise TypeError(paths) raise TypeError(paths)
return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive) return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive)
def find_reqs_simple( def find_reqs_simple(
apt_mgr: AptManager, paths: List[str], regex: bool = False, apt_mgr: AptManager,
minimum_version=None, case_insensitive=False) -> List[str]: paths: List[str],
regex: bool = False,
minimum_version=None,
case_insensitive=False,
) -> List[str]:
if not isinstance(paths, list): if not isinstance(paths, list):
raise TypeError(paths) raise TypeError(paths)
return [AptRequirement.simple(package, minimum_version=minimum_version) return [
for package in find_package_names(apt_mgr, paths, regex, case_insensitive)] AptRequirement.simple(package, minimum_version=minimum_version)
for package in find_package_names(apt_mgr, paths, regex, case_insensitive)
]
def python_spec_to_apt_rels(pkg_name, specs): def python_spec_to_apt_rels(pkg_name, specs):
@ -138,36 +146,57 @@ def python_spec_to_apt_rels(pkg_name, specs):
rels = [] rels = []
for spec in specs: for spec in specs:
deb_version = Version(spec[1]) deb_version = Version(spec[1])
if spec[0] == '~=': if spec[0] == "~=":
# PEP 440: For a given release identifier V.N , the compatible # PEP 440: For a given release identifier V.N , the compatible
# release clause is approximately equivalent to the pair of # release clause is approximately equivalent to the pair of
# comparison clauses: >= V.N, == V.* # comparison clauses: >= V.N, == V.*
parts = spec[1].split('.') parts = spec[1].split(".")
parts.pop(-1) parts.pop(-1)
parts[-1] = str(int(parts[-1])+1) parts[-1] = str(int(parts[-1]) + 1)
next_maj_deb_version = Version('.'.join(parts)) next_maj_deb_version = Version(".".join(parts))
rels.extend([{"name": pkg_name, "version": ('>=', deb_version)}, rels.extend(
{"name": pkg_name, "version": ('<<', next_maj_deb_version)}]) [
elif spec[0] == '!=': {"name": pkg_name, "version": (">=", deb_version)},
rels.extend([{"name": pkg_name, "version": ('>>', deb_version)}, {"name": pkg_name, "version": ("<<", next_maj_deb_version)},
{"name": pkg_name, "version": ('<<', deb_version)}]) ]
elif spec[1].endswith('.*') and spec[0] == '==': )
s = spec[1].split('.') elif spec[0] == "!=":
rels.extend(
[
{"name": pkg_name, "version": (">>", deb_version)},
{"name": pkg_name, "version": ("<<", deb_version)},
]
)
elif spec[1].endswith(".*") and spec[0] == "==":
s = spec[1].split(".")
s.pop(-1) s.pop(-1)
n = list(s) n = list(s)
n[-1] = str(int(n[-1])+1) n[-1] = str(int(n[-1]) + 1)
rels.extend([{"name": pkg_name, "version": ('>=', Version('.'.join(s)))}, rels.extend(
{"name": pkg_name, "version": ('<<', Version('.'.join(n)))}]) [
{"name": pkg_name, "version": (">=", Version(".".join(s)))},
{"name": pkg_name, "version": ("<<", Version(".".join(n)))},
]
)
else: else:
c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]] c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]]
rels.append([{"name": pkg_name, "version": (c, deb_version)}]) rels.append([{"name": pkg_name, "version": (c, deb_version)}])
return rels return rels
def get_package_for_python_package(apt_mgr, package, python_version: Optional[str], specs=None): def get_package_for_python_package(
pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) apt_mgr, package, python_version: Optional[str], specs=None
cpython2_regex = "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) ):
cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_")) pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(
package.replace("-", "_")
)
cpython2_regex = (
"/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info"
% re.escape(package.replace("-", "_"))
)
cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(
package.replace("-", "_")
)
if python_version == "pypy": if python_version == "pypy":
paths = [pypy_regex] paths = [pypy_regex]
elif python_version == "cpython2": elif python_version == "cpython2":
@ -177,59 +206,64 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st
elif python_version is None: elif python_version is None:
paths = [cpython3_regex, cpython2_regex, pypy_regex] paths = [cpython3_regex, cpython2_regex, pypy_regex]
else: else:
raise NotImplementedError('unsupported python version %s' % python_version) raise NotImplementedError("unsupported python version %s" % python_version)
names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True) names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True)
return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names] return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names]
def get_package_for_python_module(apt_mgr, module, python_version, specs): def get_package_for_python_module(apt_mgr, module, python_version, specs):
cpython3_regexes = [ cpython3_regexes = [
posixpath.join( posixpath.join(
"/usr/lib/python3/dist-packages", "/usr/lib/python3/dist-packages",
re.escape(module.replace(".", "/")), re.escape(module.replace(".", "/")),
"__init__.py", "__init__.py",
), ),
posixpath.join( posixpath.join(
"/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")) + ".py" "/usr/lib/python3/dist-packages",
), re.escape(module.replace(".", "/")) + ".py",
posixpath.join( ),
"/usr/lib/python3\\.[0-9]+/lib-dynload", posixpath.join(
re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so", "/usr/lib/python3\\.[0-9]+/lib-dynload",
), re.escape(module.replace(".", "/")) + "\\.cpython-.*\\.so",
posixpath.join( ),
"/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py" posixpath.join(
), "/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py"
posixpath.join( ),
"/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")), "__init__.py" posixpath.join(
), "/usr/lib/python3\\.[0-9]+/",
] re.escape(module.replace(".", "/")),
"__init__.py",
),
]
cpython2_regexes = [ cpython2_regexes = [
posixpath.join( posixpath.join(
"/usr/lib/python2\\.[0-9]/dist-packages", "/usr/lib/python2\\.[0-9]/dist-packages",
re.escape(module.replace(".", "/")), re.escape(module.replace(".", "/")),
"__init__.py", "__init__.py",
), ),
posixpath.join( posixpath.join(
"/usr/lib/python2\\.[0-9]/dist-packages", "/usr/lib/python2\\.[0-9]/dist-packages",
re.escape(module.replace(".", "/")) + ".py", re.escape(module.replace(".", "/")) + ".py",
), ),
posixpath.join( posixpath.join(
"/usr/lib/python2.\\.[0-9]/lib-dynload", "/usr/lib/python2.\\.[0-9]/lib-dynload",
re.escape(module.replace(".", "/")) + ".so", re.escape(module.replace(".", "/")) + ".so",
), ),
] ]
pypy_regexes = [ pypy_regexes = [
posixpath.join( posixpath.join(
"/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")), "__init__.py" "/usr/lib/pypy/dist-packages",
), re.escape(module.replace(".", "/")),
posixpath.join( "__init__.py",
"/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py" ),
), posixpath.join(
posixpath.join( "/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py"
"/usr/lib/pypy/dist-packages", ),
re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so", posixpath.join(
), "/usr/lib/pypy/dist-packages",
] re.escape(module.replace(".", "/")) + "\\.pypy-.*\\.so",
),
]
if python_version == "cpython3": if python_version == "cpython3":
paths = cpython3_regexes paths = cpython3_regexes
elif python_version == "cpython2": elif python_version == "cpython2":
@ -245,8 +279,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
vague_map = { vague_map = {
'the Gnu Scientific Library': 'libgsl-dev', "the Gnu Scientific Library": "libgsl-dev",
'the required FreeType library': 'libfreetype-dev', "the required FreeType library": "libfreetype-dev",
} }
@ -255,7 +289,7 @@ def resolve_vague_dep_req(apt_mgr, req):
options = [] options = []
if name in vague_map: if name in vague_map:
options.append(AptRequirement.simple(vague_map[name])) options.append(AptRequirement.simple(vague_map[name]))
if name.startswith('gnu '): if name.startswith("gnu "):
name = name[4:] name = name[4:]
for x in req.expand(): for x in req.expand():
options.extend(resolve_requirement_apt(apt_mgr, x)) options.extend(resolve_requirement_apt(apt_mgr, x))
@ -273,13 +307,23 @@ def resolve_binary_req(apt_mgr, req):
def resolve_pkg_config_req(apt_mgr, req): def resolve_pkg_config_req(apt_mgr, req):
names = find_package_names(apt_mgr, names = find_package_names(
[posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")], apt_mgr,
regex=True) [
posixpath.join(
"/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc"
)
],
regex=True,
)
if not names: if not names:
names = find_package_names( names = find_package_names(
apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")]) apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")]
return [AptRequirement.simple(name, minimum_version=req.minimum_version) for name in names] )
return [
AptRequirement.simple(name, minimum_version=req.minimum_version)
for name in names
]
def resolve_path_req(apt_mgr, req): def resolve_path_req(apt_mgr, req):
@ -288,12 +332,13 @@ def resolve_path_req(apt_mgr, req):
def resolve_c_header_req(apt_mgr, req): def resolve_c_header_req(apt_mgr, req):
reqs = find_reqs_simple( reqs = find_reqs_simple(
apt_mgr, apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False
[posixpath.join("/usr/include", req.header)], regex=False) )
if not reqs: if not reqs:
reqs = find_reqs_simple( reqs = find_reqs_simple(
apt_mgr, apt_mgr,
[posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True [posixpath.join("/usr/include", ".*", re.escape(req.header))],
regex=True,
) )
return reqs return reqs
@ -314,18 +359,21 @@ def resolve_ruby_gem_req(apt_mgr, req):
"specifications/%s-.*\\.gemspec" % re.escape(req.gem) "specifications/%s-.*\\.gemspec" % re.escape(req.gem)
) )
] ]
return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version) return find_reqs_simple(
apt_mgr, paths, regex=True, minimum_version=req.minimum_version
)
def resolve_go_package_req(apt_mgr, req): def resolve_go_package_req(apt_mgr, req):
return find_reqs_simple( return find_reqs_simple(
apt_mgr, apt_mgr,
[posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True [posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")],
regex=True,
) )
def resolve_go_req(apt_mgr, req): def resolve_go_req(apt_mgr, req):
return [AptRequirement.simple('golang-go', minimum_version='2:%s' % req.version)] return [AptRequirement.simple("golang-go", minimum_version="2:%s" % req.version)]
def resolve_dh_addon_req(apt_mgr, req): def resolve_dh_addon_req(apt_mgr, req):
@ -339,11 +387,15 @@ def resolve_php_class_req(apt_mgr, req):
def resolve_php_package_req(apt_mgr, req): def resolve_php_package_req(apt_mgr, req):
return [AptRequirement.simple('php-%s' % req.package, minimum_version=req.min_version)] return [
AptRequirement.simple("php-%s" % req.package, minimum_version=req.min_version)
]
def resolve_r_package_req(apt_mgr, req): def resolve_r_package_req(apt_mgr, req):
paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))] paths = [
posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))
]
return find_reqs_simple(apt_mgr, paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True)
@ -441,16 +493,18 @@ def resolve_maven_artifact_req(apt_mgr, req):
else: else:
version = req.version version = req.version
regex = False regex = False
def escape(x): def escape(x):
return x return x
kind = req.kind or 'jar'
kind = req.kind or "jar"
path = posixpath.join( path = posixpath.join(
escape("/usr/share/maven-repo"), escape("/usr/share/maven-repo"),
escape(req.group_id.replace(".", "/")), escape(req.group_id.replace(".", "/")),
escape(req.artifact_id), escape(req.artifact_id),
version, version,
escape("%s-") + version + escape("." + kind) escape("%s-") + version + escape("." + kind),
) )
return find_reqs_simple(apt_mgr, [path], regex=regex) return find_reqs_simple(apt_mgr, [path], regex=regex)
@ -465,15 +519,15 @@ def resolve_jdk_file_req(apt_mgr, req):
def resolve_jdk_req(apt_mgr, req): def resolve_jdk_req(apt_mgr, req):
return [AptRequirement.simple('default-jdk')] return [AptRequirement.simple("default-jdk")]
def resolve_jre_req(apt_mgr, req): def resolve_jre_req(apt_mgr, req):
return [AptRequirement.simple('default-jre')] return [AptRequirement.simple("default-jre")]
def resolve_x11_req(apt_mgr, req): def resolve_x11_req(apt_mgr, req):
return [AptRequirement.simple('libx11-dev')] return [AptRequirement.simple("libx11-dev")]
def resolve_qt_req(apt_mgr, req): def resolve_qt_req(apt_mgr, req):
@ -554,13 +608,12 @@ def resolve_python_package_req(apt_mgr, req):
def resolve_cargo_crate_req(apt_mgr, req): def resolve_cargo_crate_req(apt_mgr, req):
paths = [ paths = ["/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml" % re.escape(req.crate)]
'/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)]
return find_reqs_simple(apt_mgr, paths, regex=True) return find_reqs_simple(apt_mgr, paths, regex=True)
def resolve_ca_req(apt_mgr, req): def resolve_ca_req(apt_mgr, req):
return [AptRequirement.simple('ca-certificates')] return [AptRequirement.simple("ca-certificates")]
def resolve_apt_req(apt_mgr, req): def resolve_apt_req(apt_mgr, req):
@ -670,7 +723,16 @@ class AptResolver(Resolver):
if apt_req is not None: if apt_req is not None:
apt_requirements.append((r, apt_req)) apt_requirements.append((r, apt_req))
if apt_requirements: if apt_requirements:
yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements]) yield (
self.apt.satisfy_command(
[
PkgRelation.str(
chain(*[r.relations for o, r in apt_requirements])
)
]
),
[o for o, r in apt_requirements],
)
def resolve(self, req: Requirement): def resolve(self, req: Requirement):
ret = resolve_requirement_apt(self.apt, req) ret = resolve_requirement_apt(self.apt, req)
@ -678,14 +740,12 @@ class AptResolver(Resolver):
return None return None
if len(ret) == 1: if len(ret) == 1:
return ret[0] return ret[0]
logging.info('Need to break tie between %r with %r', ret, self.tie_breakers) logging.info("Need to break tie between %r with %r", ret, self.tie_breakers)
for tie_breaker in self.tie_breakers: for tie_breaker in self.tie_breakers:
winner = tie_breaker(ret) winner = tie_breaker(ret)
if winner is not None: if winner is not None:
if not isinstance(winner, AptRequirement): if not isinstance(winner, AptRequirement):
raise TypeError(winner) raise TypeError(winner)
return winner return winner
logging.info( logging.info("Unable to break tie over %r, picking first: %r", ret, ret[0])
'Unable to break tie over %r, picking first: %r',
ret, ret[0])
return ret[0] return ret[0]

10
ognibuild/session/__init__.py
View file

@ -55,7 +55,7 @@ class Session(object):
cwd: Optional[str] = None, cwd: Optional[str] = None,
user: Optional[str] = None, user: Optional[str] = None,
env: Optional[Dict[str, str]] = None, env: Optional[Dict[str, str]] = None,
close_fds: bool = True close_fds: bool = True,
): ):
raise NotImplementedError(self.check_call) raise NotImplementedError(self.check_call)
@ -90,8 +90,8 @@ class Session(object):
raise NotImplementedError(self.scandir) raise NotImplementedError(self.scandir)
def setup_from_vcs( def setup_from_vcs(
self, tree, include_controldir: Optional[bool] = None, self, tree, include_controldir: Optional[bool] = None, subdir="package"
subdir="package") -> Tuple[str, str]: ) -> Tuple[str, str]:
raise NotImplementedError(self.setup_from_vcs) raise NotImplementedError(self.setup_from_vcs)
def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]: def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]:
@ -108,8 +108,8 @@ class SessionSetupFailure(Exception):
def run_with_tee(session: Session, args: List[str], **kwargs): def run_with_tee(session: Session, args: List[str], **kwargs):
if 'stdin' not in kwargs: if "stdin" not in kwargs:
kwargs['stdin'] = subprocess.DEVNULL kwargs["stdin"] = subprocess.DEVNULL
p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs) p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
contents = [] contents = []
while p.poll() is None: while p.poll() is None:

34
ognibuild/session/plain.py
View file

@ -38,12 +38,13 @@ class PlainSession(Session):
raise NoSessionOpen(self) raise NoSessionOpen(self)
if user is not None: if user is not None:
import getpass import getpass
if user != getpass.getuser(): if user != getpass.getuser():
args = ["sudo", "-u", user] + args args = ["sudo", "-u", user] + args
return args return args
def __repr__(self): def __repr__(self):
return "%s()" % (type(self).__name__, ) return "%s()" % (type(self).__name__,)
def __enter__(self) -> "Session": def __enter__(self) -> "Session":
if self.es is not None: if self.es is not None:
@ -63,19 +64,23 @@ class PlainSession(Session):
pass pass
def check_call( def check_call(
self, argv: List[str], self,
cwd: Optional[str] = None, argv: List[str],
user: Optional[str] = None, cwd: Optional[str] = None,
env: Optional[Dict[str, str]] = None, user: Optional[str] = None,
close_fds: bool = True): env: Optional[Dict[str, str]] = None,
close_fds: bool = True,
):
argv = self._prepend_user(user, argv) argv = self._prepend_user(user, argv)
return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds) return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds)
def check_output( def check_output(
self, argv: List[str], self,
cwd: Optional[str] = None, argv: List[str],
user: Optional[str] = None, cwd: Optional[str] = None,
env: Optional[Dict[str, str]] = None) -> bytes: user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
) -> bytes:
argv = self._prepend_user(user, argv) argv = self._prepend_user(user, argv)
return subprocess.check_output(argv, cwd=cwd, env=env) return subprocess.check_output(argv, cwd=cwd, env=env)
@ -95,15 +100,16 @@ class PlainSession(Session):
def external_path(self, path): def external_path(self, path):
return os.path.abspath(path) return os.path.abspath(path)
def setup_from_vcs( def setup_from_vcs(self, tree, include_controldir=None, subdir="package"):
self, tree, include_controldir=None, subdir="package"):
from ..vcs import dupe_vcs_tree, export_vcs_tree from ..vcs import dupe_vcs_tree, export_vcs_tree
if include_controldir is False or ( if include_controldir is False or (
not hasattr(tree, 'base') and include_controldir is None): not hasattr(tree, "base") and include_controldir is None
):
td = self.es.enter_context(tempfile.TemporaryDirectory()) td = self.es.enter_context(tempfile.TemporaryDirectory())
export_vcs_tree(tree, td) export_vcs_tree(tree, td)
return td, td return td, td
elif not hasattr(tree, 'base'): elif not hasattr(tree, "base"):
td = self.es.enter_context(tempfile.TemporaryDirectory()) td = self.es.enter_context(tempfile.TemporaryDirectory())
dupe_vcs_tree(tree, td) dupe_vcs_tree(tree, td)
return td, td return td, td

25
ognibuild/session/schroot.py
View file

@ -57,12 +57,17 @@ class SchrootSession(Session):
if self.session_id is None: if self.session_id is None:
raise NoSessionOpen(self) raise NoSessionOpen(self)
try: try:
subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"], stderr=subprocess.PIPE) subprocess.check_output(
["schroot", "-c", "session:" + self.session_id, "-e"],
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
for line in e.stderr.splitlines(False): for line in e.stderr.splitlines(False):
if line.startswith(b'E: '): if line.startswith(b"E: "):
logging.error('%s', line[3:].decode(errors='replace')) logging.error("%s", line[3:].decode(errors="replace"))
logging.warning('Failed to close schroot session %s, leaving stray.', self.session_id) logging.warning(
"Failed to close schroot session %s, leaving stray.", self.session_id
)
self.session_id = None self.session_id = None
return False return False
self.session_id = None self.session_id = None
@ -134,10 +139,12 @@ class SchrootSession(Session):
cwd: Optional[str] = None, cwd: Optional[str] = None,
user: Optional[str] = None, user: Optional[str] = None,
env: Optional[Dict[str, str]] = None, env: Optional[Dict[str, str]] = None,
close_fds: bool = True close_fds: bool = True,
): ):
try: try:
subprocess.check_call(self._run_argv(argv, cwd, user, env=env), close_fds=close_fds) subprocess.check_call(
self._run_argv(argv, cwd, user, env=env), close_fds=close_fds
)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
raise subprocess.CalledProcessError(e.returncode, argv) raise subprocess.CalledProcessError(e.returncode, argv)
@ -191,9 +198,10 @@ class SchrootSession(Session):
return os.scandir(fullpath) return os.scandir(fullpath)
def setup_from_vcs( def setup_from_vcs(
self, tree, include_controldir: Optional[bool] = None, self, tree, include_controldir: Optional[bool] = None, subdir="package"
subdir="package"): ):
from ..vcs import dupe_vcs_tree, export_vcs_tree from ..vcs import dupe_vcs_tree, export_vcs_tree
build_dir = os.path.join(self.location, "build") build_dir = os.path.join(self.location, "build")
directory = tempfile.mkdtemp(dir=build_dir) directory = tempfile.mkdtemp(dir=build_dir)
@ -209,6 +217,7 @@ class SchrootSession(Session):
def setup_from_directory(self, path, subdir="package"): def setup_from_directory(self, path, subdir="package"):
import shutil import shutil
build_dir = os.path.join(self.location, "build") build_dir = os.path.join(self.location, "build")
directory = tempfile.mkdtemp(dir=build_dir) directory = tempfile.mkdtemp(dir=build_dir)
reldir = "/" + os.path.relpath(directory, self.location) reldir = "/" + os.path.relpath(directory, self.location)

9
ognibuild/tests/test_debian_fix_build.py
View file

@ -99,11 +99,14 @@ blah (0.1) UNRELEASED; urgency=medium
apt = AptManager(session) apt = AptManager(session)
apt._searchers = [DummyAptSearcher(self._apt_files)] apt._searchers = [DummyAptSearcher(self._apt_files)]
context = DebianPackagingContext( context = DebianPackagingContext(
self.tree, subpath="", committer="ognibuild <ognibuild@jelmer.uk>", self.tree,
subpath="",
committer="ognibuild <ognibuild@jelmer.uk>",
update_changelog=True, update_changelog=True,
commit_reporter=NullCommitReporter()) commit_reporter=NullCommitReporter(),
)
fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context) fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context)
return resolve_error(error, ("build", ), fixers) return resolve_error(error, ("build",), fixers)
def get_build_deps(self): def get_build_deps(self):
with open(self.tree.abspath("debian/control"), "r") as f: with open(self.tree.abspath("debian/control"), "r") as f: