billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix style.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-03-26 12:14:25 +00:00
parent 7b3b171134
commit 3d914f5b4e
21 changed files with 754 additions and 524 deletions
Download patch file Download diff file Expand all files Collapse all files

6
ognibuild/__init__.py
View file

@ -41,7 +41,11 @@ class UnidentifiedError(Exception):
def __repr__(self):
return "<%s(%r, %r, ..., secondary=%r)>" % (
type(self).__name__, self.retcode, self.argv, self.secondary)
type(self).__name__,
self.retcode,
self.argv,
self.secondary,
)
def shebang_binary(p):

27
ognibuild/__main__.py
View file

@ -20,7 +20,12 @@ import os
import shlex
import sys
from . import UnidentifiedError, DetailedFailure
from .buildlog import InstallFixer, ExplainInstallFixer, ExplainInstall, install_missing_reqs
from .buildlog import (
InstallFixer,
ExplainInstallFixer,
ExplainInstall,
install_missing_reqs,
)
from .buildsystem import NoBuildToolsFound, detect_buildsystems
from .resolver import (
auto_resolver,
@ -34,8 +39,7 @@ def display_explain_commands(commands):
for command, reqs in commands:
if isinstance(command, list):
command = shlex.join(command)
logging.info(
' %s (to install %s)', command, ', '.join(map(str, reqs)))
logging.info(" %s (to install %s)", command, ", ".join(map(str, reqs)))
def get_necessary_declared_requirements(resolver, requirements, stages):
@ -46,7 +50,9 @@ def get_necessary_declared_requirements(resolver, requirements, stages):
return missing
def install_necessary_declared_requirements(session, resolver, fixers, buildsystems, stages, explain=False):
def install_necessary_declared_requirements(
session, resolver, fixers, buildsystems, stages, explain=False
):
relevant = []
declared_reqs = []
for buildsystem in buildsystems:
@ -153,8 +159,7 @@ def main(): # noqa: C901
logging.info("Using requirement resolver: %s", resolver)
try:
bss = list(detect_buildsystems(args.directory))
logging.info(
"Detected buildsystems: %s", ', '.join(map(str, bss)))
logging.info("Detected buildsystems: %s", ", ".join(map(str, bss)))
fixers = determine_fixers(session, resolver, explain=args.explain)
if not args.ignore_declared_dependencies:
stages = STAGE_MAP[args.subcommand]
@ -162,7 +167,8 @@ def main(): # noqa: C901
logging.info("Checking that declared requirements are present")
try:
install_necessary_declared_requirements(
session, resolver, fixers, bss, stages, explain=args.explain)
session, resolver, fixers, bss, stages, explain=args.explain
)
except ExplainInstall as e:
display_explain_commands(e.commands)
return 1
@ -170,8 +176,11 @@ def main(): # noqa: C901
from .dist import run_dist
run_dist(
session=session, buildsystems=bss, resolver=resolver, fixers=fixers,
target_directory='.'
session=session,
buildsystems=bss,
resolver=resolver,
fixers=fixers,
target_directory=".",
)
if args.subcommand == "build":
from .build import run_build

10
ognibuild/buildlog.py
View file

@ -141,11 +141,12 @@ def problem_to_upstream_requirement(problem): # noqa: C901
elif isinstance(problem, MissingJavaClass):
return JavaClassRequirement(problem.classname)
elif isinstance(problem, MissingHaskellDependencies):
return [HaskellPackageRequirement.from_string(dep)
for dep in problem.deps]
return [HaskellPackageRequirement.from_string(dep) for dep in problem.deps]
elif isinstance(problem, MissingMavenArtifacts):
return [MavenArtifactRequirement.from_str(artifact)
for artifact in problem.artifacts]
return [
MavenArtifactRequirement.from_str(artifact)
for artifact in problem.artifacts
]
elif isinstance(problem, MissingCSharpCompiler):
return BinaryRequirement("msc")
elif isinstance(problem, GnomeCommonMissing):
@ -236,7 +237,6 @@ class InstallFixer(BuildFixer):
class ExplainInstall(Exception):
def __init__(self, commands):
self.commands = commands

399
ognibuild/buildsystem.py
View file

@ -44,7 +44,7 @@ from .requirements import (
GoRequirement,
GoPackageRequirement,
)
from .fix_build import run_with_build_fixers, run_detecting_problems
from .fix_build import run_with_build_fixers
from .session import which
@ -75,7 +75,9 @@ class BuildSystem(object):
def __str__(self):
return self.name
def dist(self, session, resolver, fixers, target_directory: str, quiet=False) -> str:
def dist(
self, session, resolver, fixers, target_directory: str, quiet=False
) -> str:
raise NotImplementedError(self.dist)
def test(self, session, resolver, fixers):
@ -103,11 +105,12 @@ class BuildSystem(object):
def xmlparse_simplify_namespaces(path, namespaces):
import xml.etree.ElementTree as ET
namespaces = ['{%s}' % ns for ns in namespaces]
namespaces = ["{%s}" % ns for ns in namespaces]
tree = ET.iterparse(path)
for _, el in tree:
for namespace in namespaces:
el.tag = el.tag.replace(namespace, '')
el.tag = el.tag.replace(namespace, "")
return tree.root
@ -123,7 +126,7 @@ class Pear(BuildSystem):
def dist(self, session, resolver, fixers, target_directory: str, quiet=False):
self.setup(resolver)
with DistCatcher([session.external_path('.')]) as dc:
with DistCatcher([session.external_path(".")]) as dc:
run_with_build_fixers(session, ["pear", "package"], fixers)
return dc.copy_single(target_directory)
@ -146,28 +149,34 @@ class Pear(BuildSystem):
def get_declared_dependencies(self, session, fixers=None):
path = os.path.join(self.path, "package.xml")
import xml.etree.ElementTree as ET
try:
root = xmlparse_simplify_namespaces(path, [
'http://pear.php.net/dtd/package-2.0',
'http://pear.php.net/dtd/package-2.1'])
root = xmlparse_simplify_namespaces(
path,
[
"http://pear.php.net/dtd/package-2.0",
"http://pear.php.net/dtd/package-2.1",
],
)
except ET.ParseError as e:
logging.warning('Unable to parse package.xml: %s', e)
logging.warning("Unable to parse package.xml: %s", e)
return
assert root.tag == 'package', 'root tag is %r' % root.tag
dependencies_tag = root.find('dependencies')
assert root.tag == "package", "root tag is %r" % root.tag
dependencies_tag = root.find("dependencies")
if dependencies_tag is not None:
required_tag = root.find('dependencies')
required_tag = root.find("dependencies")
if required_tag is not None:
for package_tag in root.findall('package'):
name = package_tag.find('name').text
min_tag = package_tag.find('min')
max_tag = package_tag.find('max')
channel_tag = package_tag.find('channel')
for package_tag in root.findall("package"):
name = package_tag.find("name").text
min_tag = package_tag.find("min")
max_tag = package_tag.find("max")
channel_tag = package_tag.find("channel")
yield "core", PhpPackageRequirement(
name,
channel=(channel_tag.text if channel_tag else None),
min_version=(min_tag.text if min_tag else None),
max_version=(max_tag.text if max_tag else None))
max_version=(max_tag.text if max_tag else None),
)
@classmethod
def probe(cls, path):
@ -179,6 +188,7 @@ class Pear(BuildSystem):
# run_setup, but setting __name__
# Imported from Python's distutils.core, Copyright (C) PSF
def run_setup(script_name, script_args=None, stop_after="run"):
from distutils import core
import sys
@ -260,11 +270,11 @@ with open(%(output_path)s, 'w') as f:
class SetupPy(BuildSystem):
name = "setup.py"
DEFAULT_PYTHON = 'python3'
DEFAULT_PYTHON = "python3"
def __init__(self, path):
self.path = path
if os.path.exists(os.path.join(self.path, 'setup.py')):
if os.path.exists(os.path.join(self.path, "setup.py")):
self.has_setup_py = True
else:
self.has_setup_py = False
@ -280,7 +290,9 @@ class SetupPy(BuildSystem):
self.pyproject = None
self.build_backend = None
else:
self.build_backend = self.pyproject.get("build-system", {}).get('build-backend')
self.build_backend = self.pyproject.get("build-system", {}).get(
"build-backend"
)
def load_toml(self):
import toml
@ -290,7 +302,8 @@ class SetupPy(BuildSystem):
def load_setup_cfg(self):
from setuptools.config import read_configuration
p = os.path.join(self.path, 'setup.cfg')
p = os.path.join(self.path, "setup.cfg")
if os.path.exists(p):
return read_configuration(p)
raise FileNotFoundError(p)
@ -304,7 +317,7 @@ class SetupPy(BuildSystem):
return self._extract_setup_in_session(session, fixers)
def _extract_setup_direct(self):
p = os.path.join(self.path, 'setup.py')
p = os.path.join(self.path, "setup.py")
try:
d = run_setup(os.path.abspath(p), stop_after="init")
except RuntimeError as e:
@ -313,37 +326,42 @@ class SetupPy(BuildSystem):
if d is None:
logging.warning(
"'distutils.core.setup()' was never called -- "
"perhaps '%s' is not a Distutils setup script?" % os.path.basename(p))
"perhaps '%s' is not a Distutils setup script?" % os.path.basename(p)
)
return None
return {
'name': d.name,
'setup_requires': getattr(d, "setup_requires", []),
'install_requires': getattr(d, "install_requires", []),
'tests_require': getattr(d, "tests_require", []) or [],
'scripts': getattr(d, "scripts", []),
'entry_points': getattr(d, "entry_points", None) or {},
'packages': getattr(d, "packages", []),
'requires': d.get_requires() or [],
"name": d.name,
"setup_requires": getattr(d, "setup_requires", []),
"install_requires": getattr(d, "install_requires", []),
"tests_require": getattr(d, "tests_require", []) or [],
"scripts": getattr(d, "scripts", []),
"entry_points": getattr(d, "entry_points", None) or {},
"packages": getattr(d, "packages", []),
"requires": d.get_requires() or [],
}
def _extract_setup_in_session(self, session, fixers=None):
import tempfile
import json
interpreter = shebang_binary(os.path.join(self.path, "setup.py"))
if interpreter is None:
interpreter = self.DEFAULT_PYTHON
output_f = tempfile.NamedTemporaryFile(
dir=os.path.join(session.location, 'tmp'), mode='w+t')
dir=os.path.join(session.location, "tmp"), mode="w+t"
)
with output_f:
# TODO(jelmer): Perhaps run this in session, so we can install
# missing dependencies?
argv = [interpreter, "-c",
_setup_wrapper
.replace('%(script_name)s', '"setup.py"')
.replace('%(output_path)s',
'"/' + os.path.relpath(output_f.name, session.location) +
'"')]
argv = [
interpreter,
"-c",
_setup_wrapper.replace("%(script_name)s", '"setup.py"').replace(
"%(output_path)s",
'"/' + os.path.relpath(output_f.name, session.location) + '"',
),
]
try:
if fixers is not None:
run_with_build_fixers(session, argv, fixers)
@ -359,15 +377,17 @@ class SetupPy(BuildSystem):
return "%s(%r)" % (type(self).__name__, self.path)
def test(self, session, resolver, fixers):
if os.path.exists(os.path.join(self.path, 'tox.ini')):
run_with_build_fixers(session, ['tox'], fixers)
if os.path.exists(os.path.join(self.path, "tox.ini")):
run_with_build_fixers(session, ["tox"], fixers)
elif self.pyproject:
run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers)
run_with_build_fixers(
session, [self.DEFAULT_PYTHON, "-m", "pep517.check", "."], fixers
)
elif self.has_setup_py:
# Pre-emptively insall setuptools, since distutils doesn't provide
# a 'test' subcommand and some packages fall back to distutils
# if setuptools is not available.
resolver.install([PythonPackageRequirement('setuptools')])
resolver.install([PythonPackageRequirement("setuptools")])
self._run_setup(session, resolver, ["test"], fixers)
else:
raise NotImplementedError
@ -386,13 +406,17 @@ class SetupPy(BuildSystem):
preargs.append("--quiet")
# Preemptively install setuptools since some packages fail in
# some way without it.
resolver.install([PythonPackageRequirement('setuptools')])
with DistCatcher([session.external_path('dist')]) as dc:
resolver.install([PythonPackageRequirement("setuptools")])
with DistCatcher([session.external_path("dist")]) as dc:
self._run_setup(session, resolver, preargs + ["sdist"], fixers)
return dc.copy_single(target_directory)
elif self.pyproject:
with DistCatcher([session.external_path('dist')]) as dc:
run_with_build_fixers(session, [self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."], fixers)
with DistCatcher([session.external_path("dist")]) as dc:
run_with_build_fixers(
session,
[self.DEFAULT_PYTHON, "-m", "pep517.build", "--source", "."],
fixers,
)
return dc.copy_single(target_directory)
raise AssertionError("no setup.py or pyproject.toml")
@ -413,79 +437,83 @@ class SetupPy(BuildSystem):
def _run_setup(self, session, resolver, args, fixers):
from .buildlog import install_missing_reqs
# Install the setup_requires beforehand, since otherwise
# setuptools might fetch eggs instead of our preferred resolver.
install_missing_reqs(session, resolver, list(self._setup_requires()))
interpreter = shebang_binary(os.path.join(self.path, 'setup.py'))
interpreter = shebang_binary(os.path.join(self.path, "setup.py"))
if interpreter is None:
interpreter = self.DEFAULT_PYTHON
argv = [interpreter, "./setup.py"] + args
env = {}
# Inherit SETUPTOOLS_SCM_PRETEND_VERSION from the current environment
if 'SETUPTOOLS_SCM_PRETEND_VERSION' in os.environ:
env['SETUPTOOLS_SCM_PRETEND_VERSION'] = (
os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'])
if "SETUPTOOLS_SCM_PRETEND_VERSION" in os.environ:
env["SETUPTOOLS_SCM_PRETEND_VERSION"] = os.environ[
"SETUPTOOLS_SCM_PRETEND_VERSION"
]
run_with_build_fixers(session, argv, fixers, env=env)
def _setup_requires(self):
if self.pyproject:
if "build-system" in self.pyproject:
for require in self.pyproject['build-system'].get("requires", []):
for require in self.pyproject["build-system"].get("requires", []):
yield PythonPackageRequirement.from_requirement_str(require)
if self.config:
options = self.config.get('options', {})
for require in options.get('setup_requires', []):
options = self.config.get("options", {})
for require in options.get("setup_requires", []):
yield PythonPackageRequirement.from_requirement_str(require)
def get_declared_dependencies(self, session, fixers=None):
distribution = self._extract_setup(session, fixers)
if distribution is not None:
for require in distribution['requires']:
for require in distribution["requires"]:
yield "core", PythonPackageRequirement.from_requirement_str(require)
# Not present for distutils-only packages
for require in distribution['setup_requires']:
for require in distribution["setup_requires"]:
yield "build", PythonPackageRequirement.from_requirement_str(require)
# Not present for distutils-only packages
for require in distribution['install_requires']:
for require in distribution["install_requires"]:
yield "core", PythonPackageRequirement.from_requirement_str(require)
# Not present for distutils-only packages
for require in distribution['tests_require']:
for require in distribution["tests_require"]:
yield "test", PythonPackageRequirement.from_requirement_str(require)
if self.pyproject:
if "build-system" in self.pyproject:
for require in self.pyproject['build-system'].get("requires", []):
yield "build", PythonPackageRequirement.from_requirement_str(require)
for require in self.pyproject["build-system"].get("requires", []):
yield "build", PythonPackageRequirement.from_requirement_str(
require
)
if self.config:
options = self.config.get('options', {})
for require in options.get('setup_requires', []):
options = self.config.get("options", {})
for require in options.get("setup_requires", []):
yield "build", PythonPackageRequirement.from_requirement_str(require)
for require in options.get('install_requires', []):
for require in options.get("install_requires", []):
yield "core", PythonPackageRequirement.from_requirement_str(require)
def get_declared_outputs(self, session, fixers=None):
distribution = self._extract_setup(session, fixers)
all_packages = set()
if distribution is not None:
for script in distribution['scripts']:
for script in distribution["scripts"]:
yield BinaryOutput(os.path.basename(script))
for script in distribution["entry_points"].get("console_scripts", []):
yield BinaryOutput(script.split("=")[0])
all_packages.update(distribution['packages'])
all_packages.update(distribution["packages"])
if self.config:
options = self.config.get('options', {})
all_packages.update(options.get('packages', []))
for script in options.get('scripts', []):
options = self.config.get("options", {})
all_packages.update(options.get("packages", []))
for script in options.get("scripts", []):
yield BinaryOutput(os.path.basename(script))
for script in options.get("entry_points", {}).get("console_scripts", []):
yield BinaryOutput(script.split("=")[0])
packages = set()
for package in sorted(all_packages):
pts = package.split('.')
pts = package.split(".")
b = []
for e in pts:
b.append(e)
if '.'.join(b) in packages:
if ".".join(b) in packages:
break
else:
packages.add(package)
@ -521,12 +549,12 @@ class Octave(BuildSystem):
return False
# Urgh, isn't there a better way to see if this is an octave package?
for entry in os.scandir(path):
if entry.name.endswith('.m'):
if entry.name.endswith(".m"):
return True
if not entry.is_dir():
continue
for subentry in os.scandir(entry.path):
if subentry.name.endswith('.m'):
if subentry.name.endswith(".m"):
return True
return False
@ -537,17 +565,19 @@ class Octave(BuildSystem):
return cls(path)
def _read_description(self):
path = os.path.join(self.path, 'DESCRIPTION')
path = os.path.join(self.path, "DESCRIPTION")
from email.parser import BytesParser
with open(path, 'rb') as f:
with open(path, "rb") as f:
return BytesParser().parse(f)
def get_declared_dependencies(self, session, fixers=None):
def parse_list(t):
return [s.strip() for s in t.split(',') if s.strip()]
return [s.strip() for s in t.split(",") if s.strip()]
description = self._read_description()
if 'Depends' in description:
for s in parse_list(description['Depends']):
if "Depends" in description:
for s in parse_list(description["Depends"]):
yield "build", OctavePackageRequirement.from_str(s)
@ -564,14 +594,14 @@ class Gradle(BuildSystem):
@classmethod
def exists(cls, path):
return (
os.path.exists(os.path.join(path, "build.gradle")) or
os.path.exists(os.path.join(path, "build.gradle.kts")))
return os.path.exists(os.path.join(path, "build.gradle")) or os.path.exists(
os.path.join(path, "build.gradle.kts")
)
@classmethod
def from_path(cls, path):
if os.path.exists(os.path.join(path, "gradlew")):
return cls(path, './gradlew')
return cls(path, "./gradlew")
return cls(path)
@classmethod
@ -581,44 +611,50 @@ class Gradle(BuildSystem):
return cls.from_path(path)
def setup(self, resolver):
if not self.executable.startswith('./'):
if not self.executable.startswith("./"):
resolver.install([BinaryRequirement(self.executable)])
def _run(self, session, resolver, task, args, fixers):
self.setup(resolver)
argv = []
if self.executable.startswith('./') and (
not os.access(os.path.join(self.path, self.executable), os.X_OK)):
argv.append('sh')
if self.executable.startswith("./") and (
not os.access(os.path.join(self.path, self.executable), os.X_OK)
):
argv.append("sh")
argv.extend([self.executable, task])
argv.extend(args)
try:
run_with_build_fixers(session, argv, fixers)
except UnidentifiedError as e:
if any([re.match(
r"Task '" + task + "' not found in root project '.*'\.",
line) for line in e.lines]):
if any(
[
re.match(
r"Task '" + task + r"' not found in root project '.*'\.", line
)
for line in e.lines
]
):
raise NotImplementedError
raise
def clean(self, session, resolver, fixers):
self._run(session, resolver, 'clean', [], fixers)
self._run(session, resolver, "clean", [], fixers)
def build(self, session, resolver, fixers):
self._run(session, resolver, 'build', [], fixers)
self._run(session, resolver, "build", [], fixers)
def test(self, session, resolver, fixers):
self._run(session, resolver, 'test', [], fixers)
self._run(session, resolver, "test", [], fixers)
def dist(self, session, resolver, fixers, target_directory, quiet=False):
with DistCatcher([session.external_path('.')]) as dc:
self._run(session, resolver, 'distTar', [], fixers)
with DistCatcher([session.external_path(".")]) as dc:
self._run(session, resolver, "distTar", [], fixers)
return dc.copy_single(target_directory)
def install(self, session, resolver, fixers, install_target):
raise NotImplementedError
# TODO(jelmer): installDist just creates files under build/install/...
self._run(session, resolver, 'installDist', [], fixers)
self._run(session, resolver, "installDist", [], fixers)
class R(BuildSystem):
@ -638,7 +674,7 @@ class R(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False):
r_path = guaranteed_which(session, resolver, "R")
with DistCatcher([session.external_path('.')]) as dc:
with DistCatcher([session.external_path(".")]) as dc:
run_with_build_fixers(session, [r_path, "CMD", "build", "."], fixers)
return dc.copy_single(target_directory)
@ -652,34 +688,37 @@ class R(BuildSystem):
@classmethod
def probe(cls, path):
if (os.path.exists(os.path.join(path, 'DESCRIPTION')) and
os.path.exists(os.path.join(path, 'NAMESPACE'))):
if os.path.exists(os.path.join(path, "DESCRIPTION")) and os.path.exists(
os.path.join(path, "NAMESPACE")
):
return cls(path)
def _read_description(self):
path = os.path.join(self.path, 'DESCRIPTION')
path = os.path.join(self.path, "DESCRIPTION")
from email.parser import BytesParser
with open(path, 'rb') as f:
with open(path, "rb") as f:
return BytesParser().parse(f)
def get_declared_dependencies(self, session, fixers=None):
def parse_list(t):
return [s.strip() for s in t.split(',') if s.strip()]
return [s.strip() for s in t.split(",") if s.strip()]
description = self._read_description()
if 'Suggests' in description:
for s in parse_list(description['Suggests']):
if "Suggests" in description:
for s in parse_list(description["Suggests"]):
yield "build", RPackageRequirement.from_str(s)
if 'Depends' in description:
for s in parse_list(description['Depends']):
if "Depends" in description:
for s in parse_list(description["Depends"]):
yield "build", RPackageRequirement.from_str(s)
if 'Imports' in description:
for s in parse_list(description['Imports']):
if "Imports" in description:
for s in parse_list(description["Imports"]):
yield "build", RPackageRequirement.from_str(s)
def get_declared_outputs(self, session, fixers=None):
description = self._read_description()
if 'Package' in description:
yield RPackageOutput(description['Package'])
if "Package" in description:
yield RPackageOutput(description["Package"])
class Meson(BuildSystem):
@ -693,8 +732,8 @@ class Meson(BuildSystem):
return "%s(%r)" % (type(self).__name__, self.path)
def _setup(self, session, fixers):
if not session.exists('build'):
session.check_call(['mkdir', 'build'])
if not session.exists("build"):
session.check_call(["mkdir", "build"])
run_with_build_fixers(session, ["meson", "setup", "build"], fixers)
def clean(self, session, resolver, fixers):
@ -707,7 +746,7 @@ class Meson(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False):
self._setup(session, fixers)
with DistCatcher([session.external_path('build/meson-dist')]) as dc:
with DistCatcher([session.external_path("build/meson-dist")]) as dc:
run_with_build_fixers(session, ["ninja", "-C", "build", "dist"], fixers)
return dc.copy_single(target_directory)
@ -742,7 +781,7 @@ class Npm(BuildSystem):
return "%s(%r)" % (type(self).__name__, self.path)
def get_declared_dependencies(self, session, fixers=None):
if 'dependencies' in self.package:
if "dependencies" in self.package:
for name, unused_version in self.package["dependencies"].items():
# TODO(jelmer): Look at version
yield "core", NodePackageRequirement(name)
@ -756,13 +795,13 @@ class Npm(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False):
self.setup(resolver)
with DistCatcher([session.external_path('.')]) as dc:
with DistCatcher([session.external_path(".")]) as dc:
run_with_build_fixers(session, ["npm", "pack"], fixers)
return dc.copy_single(target_directory)
def test(self, session, resolver, fixers):
self.setup(resolver)
test_script = self.package['scripts'].get('test')
test_script = self.package["scripts"].get("test")
if test_script:
run_with_build_fixers(session, shlex.split(test_script), fixers)
else:
@ -770,7 +809,7 @@ class Npm(BuildSystem):
def build(self, session, resolver, fixers):
self.setup(resolver)
build_script = self.package['scripts'].get('build')
build_script = self.package["scripts"].get("build")
if build_script:
run_with_build_fixers(session, shlex.split(build_script), fixers)
else:
@ -778,7 +817,7 @@ class Npm(BuildSystem):
def clean(self, session, resolver, fixers):
self.setup(resolver)
clean_script = self.package['scripts'].get('clean')
clean_script = self.package["scripts"].get("clean")
if clean_script:
run_with_build_fixers(session, shlex.split(clean_script), fixers)
else:
@ -803,7 +842,7 @@ class Waf(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False):
self.setup(session, resolver, fixers)
with DistCatcher.default(session.external_path('.')) as dc:
with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["./waf", "dist"], fixers)
return dc.copy_single(target_directory)
@ -835,13 +874,15 @@ class Gem(BuildSystem):
]
if len(gemfiles) > 1:
logging.warning("More than one gemfile. Trying the first?")
with DistCatcher.default(session.external_path('.')) as dc:
with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["gem2tgz", gemfiles[0]], fixers)
return dc.copy_single(target_directory)
@classmethod
def probe(cls, path):
gemfiles = [entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")]
gemfiles = [
entry.path for entry in os.scandir(path) if entry.name.endswith(".gem")
]
if gemfiles:
return cls(gemfiles[0])
@ -881,13 +922,13 @@ class DistZilla(BuildSystem):
self.setup(resolver)
if self.name == "dist-inkt":
resolver.install([PerlModuleRequirement(self.dist_inkt_class)])
with DistCatcher.default(session.external_path('.')) as dc:
with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["distinkt-dist"], fixers)
return dc.copy_single(target_directory)
else:
# Default to invoking Dist::Zilla
resolver.install([PerlModuleRequirement("Dist::Zilla")])
with DistCatcher.default(session.external_path('.')) as dc:
with DistCatcher.default(session.external_path(".")) as dc:
run_with_build_fixers(session, ["dzil", "build", "--tgz"], fixers)
return dc.copy_single(target_directory)
@ -939,13 +980,13 @@ class RunTests(BuildSystem):
def _read_cpanfile(session, args, kind, fixers):
for line in run_with_build_fixers(session, ['cpanfile-dump'] + args, fixers):
for line in run_with_build_fixers(session, ["cpanfile-dump"] + args, fixers):
yield kind, PerlModuleRequirement(line)
def _declared_deps_from_cpanfile(session, fixers):
yield from _read_cpanfile(session, ['--configure', '--build'], 'build', fixers)
yield from _read_cpanfile(session, ['--test'], 'test', fixers)
yield from _read_cpanfile(session, ["--configure", "--build"], "build", fixers)
yield from _read_cpanfile(session, ["--test"], "test", fixers)
class Make(BuildSystem):
@ -989,7 +1030,9 @@ class Make(BuildSystem):
if not makefile_exists() and session.exists("configure"):
run_with_build_fixers(session, ["./configure"], fixers)
if not makefile_exists() and any([n.name.endswith('.pro') for n in session.scandir(".")]):
if not makefile_exists() and any(
[n.name.endswith(".pro") for n in session.scandir(".")]
):
run_with_build_fixers(session, ["qmake"], fixers)
def build(self, session, resolver, fixers):
@ -1010,7 +1053,7 @@ class Make(BuildSystem):
def dist(self, session, resolver, fixers, target_directory, quiet=False):
self.setup(session, resolver, fixers)
with DistCatcher.default(session.external_path('.')) as dc:
with DistCatcher.default(session.external_path(".")) as dc:
try:
run_with_build_fixers(session, ["make", "dist"], fixers)
except UnidentifiedError as e:
@ -1106,7 +1149,7 @@ class Make(BuildSystem):
return cls(path)
for n in os.scandir(path):
# qmake
if n.name.endswith('.pro'):
if n.name.endswith(".pro"):
return cls(path)
@ -1133,8 +1176,9 @@ class Cargo(BuildSystem):
# TODO(jelmer): Look at details['version']
yield "build", CargoCrateRequirement(
name,
features=details.get('features', []),
version=details.get("version"))
features=details.get("features", []),
version=details.get("version"),
)
def test(self, session, resolver, fixers):
run_with_build_fixers(session, ["cargo", "test"], fixers)
@ -1157,19 +1201,20 @@ def _parse_go_mod(f):
line = f.readline()
if not line:
return line
return line.split('//')[0] + '\n'
return line.split("//")[0] + "\n"
line = readline()
while line:
parts = line.strip().split(' ')
if not parts or parts == ['']:
parts = line.strip().split(" ")
if not parts or parts == [""]:
continue
if len(parts) == 2 and parts[1] == '(':
if len(parts) == 2 and parts[1] == "(":
line = readline()
while line.strip() != ')':
yield [parts[0]] + list(line.strip().split(' '))
while line.strip() != ")":
yield [parts[0]] + list(line.strip().split(" "))
line = readline()
if not line:
raise AssertionError('list of %s interrupted?' % parts[0])
raise AssertionError("list of %s interrupted?" % parts[0])
else:
yield parts
line = readline()
@ -1199,31 +1244,30 @@ class Golang(BuildSystem):
session.check_call(["go", "clean"])
def get_declared_dependencies(self, session, fixers=None):
go_mod_path = os.path.join(self.path, 'go.mod')
go_mod_path = os.path.join(self.path, "go.mod")
if os.path.exists(go_mod_path):
with open(go_mod_path, 'r') as f:
with open(go_mod_path, "r") as f:
for parts in _parse_go_mod(f):
if parts[0] == 'go':
if parts[0] == "go":
yield "build", GoRequirement(parts[1])
elif parts[0] == 'require':
elif parts[0] == "require":
yield "build", GoPackageRequirement(
parts[1], parts[2].lstrip('v') if len(parts) > 2 else None)
elif parts[0] == 'exclude':
parts[1], parts[2].lstrip("v") if len(parts) > 2 else None
)
elif parts[0] == "exclude":
pass # TODO(jelmer): Create conflicts?
elif parts[0] == 'replace':
elif parts[0] == "replace":
pass # TODO(jelmer): do.. something?
elif parts[0] == 'module':
elif parts[0] == "module":
pass
else:
logging.warning(
'Unknown directive %s in go.mod',
parts[0])
logging.warning("Unknown directive %s in go.mod", parts[0])
@classmethod
def probe(cls, path):
if os.path.exists(os.path.join(path, 'go.mod')):
if os.path.exists(os.path.join(path, "go.mod")):
return Golang(path)
if os.path.exists(os.path.join(path, 'go.sum')):
if os.path.exists(os.path.join(path, "go.sum")):
return Golang(path)
for entry in os.scandir(path):
if entry.name.endswith(".go"):
@ -1266,20 +1310,23 @@ class Maven(BuildSystem):
def get_declared_dependencies(self, session, fixers=None):
import xml.etree.ElementTree as ET
try:
root = xmlparse_simplify_namespaces(self.path,
['http://maven.apache.org/POM/4.0.0'])
root = xmlparse_simplify_namespaces(
self.path, ["http://maven.apache.org/POM/4.0.0"]
)
except ET.ParseError as e:
logging.warning('Unable to parse package.xml: %s', e)
logging.warning("Unable to parse package.xml: %s", e)
return
assert root.tag == 'project', 'root tag is %r' % root.tag
deps_tag = root.find('dependencies')
assert root.tag == "project", "root tag is %r" % root.tag
deps_tag = root.find("dependencies")
if deps_tag:
for dep in deps_tag.findall('dependency'):
for dep in deps_tag.findall("dependency"):
yield "core", MavenArtifactRequirement(
dep.find('groupId').text,
dep.find('artifactId').text,
dep.find('version').text)
dep.find("groupId").text,
dep.find("artifactId").text,
dep.find("version").text,
)
class Cabal(BuildSystem):
@ -1310,9 +1357,12 @@ class Cabal(BuildSystem):
self._run(session, ["test"], fixers)
def dist(self, session, resolver, fixers, target_directory, quiet=False):
with DistCatcher([
session.external_path('dist-newstyle/sdist'),
session.external_path('dist')]) as dc:
with DistCatcher(
[
session.external_path("dist-newstyle/sdist"),
session.external_path("dist"),
]
) as dc:
self._run(session, ["sdist"], fixers)
return dc.copy_single(target_directory)
@ -1377,23 +1427,38 @@ class PerlBuildTiny(BuildSystem):
BUILDSYSTEM_CLSES = [
Pear, SetupPy, Npm, Waf, Cargo, Meson, Cabal, Gradle, Maven,
DistZilla, Gem, PerlBuildTiny, Golang, R, Octave,
Pear,
SetupPy,
Npm,
Waf,
Cargo,
Meson,
Cabal,
Gradle,
Maven,
DistZilla,
Gem,
PerlBuildTiny,
Golang,
R,
Octave,
# Make is intentionally at the end of the list.
Make, Composer, RunTests]
Make,
Composer,
RunTests,
]
def scan_buildsystems(path):
"""Detect build systems."""
ret = []
ret.extend([('.', bs) for bs in detect_buildsystems(path)])
ret.extend([(".", bs) for bs in detect_buildsystems(path)])
if not ret:
# Nothing found. Try the next level?
for entry in os.scandir(path):
if entry.is_dir():
ret.extend(
[(entry.name, bs) for bs in detect_buildsystems(entry.path)])
ret.extend([(entry.name, bs) for bs in detect_buildsystems(entry.path)])
return ret

15
ognibuild/debian/apt.py
View file

@ -26,10 +26,17 @@ from buildlog_consultant.apt import (
from .. import DetailedFailure, UnidentifiedError
from ..session import Session, run_with_tee, get_user
from .file_search import FileSearcher, AptCachedContentsFileSearcher, GENERATED_FILE_SEARCHER, get_packages_for_paths
from .file_search import (
FileSearcher,
AptCachedContentsFileSearcher,
GENERATED_FILE_SEARCHER,
get_packages_for_paths,
)
def run_apt(session: Session, args: List[str], prefix: Optional[List[str]] = None) -> None:
def run_apt(
session: Session, args: List[str], prefix: Optional[List[str]] = None
) -> None:
"""Run apt."""
if prefix is None:
prefix = []
@ -84,7 +91,9 @@ class AptManager(object):
def get_packages_for_paths(self, paths, regex=False, case_insensitive=False):
logging.debug("Searching for packages containing %r", paths)
# TODO(jelmer): Make sure we use whatever is configured in self.session
return get_packages_for_paths(paths, self.searchers(), regex=regex, case_insensitive=case_insensitive)
return get_packages_for_paths(
paths, self.searchers(), regex=regex, case_insensitive=case_insensitive
)
def missing(self, packages):
root = getattr(self.session, "location", "/")

1
ognibuild/debian/build.py
View file

@ -33,7 +33,6 @@ import sys
from debian.changelog import Changelog
from debmutate.changelog import get_maintainer, format_datetime
from breezy import osutils
from breezy.mutabletree import MutableTree
from breezy.plugins.debian.builder import BuildFailedError
from breezy.tree import Tree

17
ognibuild/debian/build_deps.py
View file

@ -22,7 +22,6 @@ import logging
class BuildDependencyTieBreaker(object):
def __init__(self, rootdir):
self.rootdir = rootdir
self._counts = None
@ -37,8 +36,9 @@ class BuildDependencyTieBreaker(object):
def _count(self):
counts = {}
import apt_pkg
apt_pkg.init()
apt_pkg.config.set('Dir', self.rootdir)
apt_pkg.config.set("Dir", self.rootdir)
apt_cache = apt_pkg.SourceRecords()
apt_cache.restart()
while apt_cache.step():
@ -65,17 +65,20 @@ class BuildDependencyTieBreaker(object):
return None
top = max(by_count.items(), key=lambda k: k[1])
logging.info(
'Breaking tie between %r to %r based on build-depends count',
[repr(r) for r in reqs], top[0])
"Breaking tie between %r to %r based on build-depends count",
[repr(r) for r in reqs],
top[0],
)
return top[0]
if __name__ == '__main__':
if __name__ == "__main__":
import argparse
from ..resolver.apt import AptRequirement
parser = argparse.ArgumentParser()
parser.add_argument('req', nargs='+')
parser.add_argument("req", nargs="+")
args = parser.parse_args()
reqs = [AptRequirement.from_str(req) for req in args.req]
tie_breaker = BuildDependencyTieBreaker('/')
tie_breaker = BuildDependencyTieBreaker("/")
print(tie_breaker(reqs))

72
ognibuild/debian/file_search.py
View file

@ -21,7 +21,7 @@ from datetime import datetime
from debian.deb822 import Release
import os
import re
from typing import Iterator, List, Optional, Set
from typing import Iterator, List
import logging
@ -29,7 +29,9 @@ from .. import USER_AGENT
class FileSearcher(object):
def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]:
def search_files(
self, path: str, regex: bool = False, case_insensitive: bool = False
) -> Iterator[str]:
raise NotImplementedError(self.search_files)
@ -66,14 +68,16 @@ def contents_urls_from_sources_entry(source, arches, load_url):
try:
response = load_url(release_url)
except FileNotFoundError as e:
logging.warning('Unable to download %s or %s: %s', inrelease_url, release_url, e)
logging.warning(
"Unable to download %s or %s: %s", inrelease_url, release_url, e
)
return
existing_names = {}
release = Release(response.read())
for hn in ['MD5Sum', 'SHA1Sum', 'SHA256Sum']:
for hn in ["MD5Sum", "SHA1Sum", "SHA256Sum"]:
for entry in release.get(hn, []):
existing_names[os.path.splitext(entry['name'])[0]] = entry['name']
existing_names[os.path.splitext(entry["name"])[0]] = entry["name"]
contents_files = set()
if components:
@ -117,8 +121,7 @@ def load_direct_url(url):
for ext in [".xz", ".gz", ""]:
try:
request = Request(
url + ext, headers={"User-Agent": USER_AGENT})
request = Request(url + ext, headers={"User-Agent": USER_AGENT})
response = urlopen(request)
except HTTPError as e:
if e.status == 404:
@ -141,16 +144,17 @@ def load_url_with_cache(url, cache_dirs):
def load_apt_cache_file(url, cache_dir):
fn = apt_pkg.uri_to_filename(url)
for ext in ['.xz', '.gz', '.lz4', '']:
for ext in [".xz", ".gz", ".lz4", ""]:
p = os.path.join(cache_dir, fn + ext)
if not os.path.exists(p):
continue
# return os.popen('/usr/lib/apt/apt-helper cat-file %s' % p)
logging.debug("Loading cached contents file %s", p)
if ext == '.lz4':
if ext == ".lz4":
import lz4.frame
return lz4.frame.open(p, mode="rb")
return _unwrap(open(p, 'rb'), ext)
return _unwrap(open(p, "rb"), ext)
raise FileNotFoundError(url)
@ -174,14 +178,15 @@ class AptCachedContentsFileSearcher(FileSearcher):
sl.load("/etc/apt/sources.list")
from .build import get_build_architecture
cache_dirs = set(["/var/lib/apt/lists"])
def load_url(url):
return load_url_with_cache(url, cache_dirs)
urls = list(
contents_urls_from_sourceslist(
sl, get_build_architecture(), load_url))
contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)
)
self._load_urls(urls, cache_dirs, load_url)
def load_from_session(self, session):
@ -193,16 +198,19 @@ class AptCachedContentsFileSearcher(FileSearcher):
from .build import get_build_architecture
cache_dirs = set([
cache_dirs = set(
[
os.path.join(session.location, "var/lib/apt/lists"),
"/var/lib/apt/lists",
])
]
)
def load_url(url):
return load_url_with_cache(url, cache_dirs)
urls = list(
contents_urls_from_sourceslist(sl, get_build_architecture(), load_url))
contents_urls_from_sourceslist(sl, get_build_architecture(), load_url)
)
self._load_urls(urls, cache_dirs, load_url)
def _load_urls(self, urls, cache_dirs, load_url):
@ -217,7 +225,7 @@ class AptCachedContentsFileSearcher(FileSearcher):
self._db[path] = package
def search_files(self, path, regex=False, case_insensitive=False):
path = path.lstrip('/').encode('utf-8', 'surrogateescape')
path = path.lstrip("/").encode("utf-8", "surrogateescape")
if case_insensitive and not regex:
regex = True
path = re.escape(path)
@ -230,12 +238,12 @@ class AptCachedContentsFileSearcher(FileSearcher):
for p, rest in self._db.items():
if c.match(p):
pkg = rest.split(b"/")[-1]
ret.append((p, pkg.decode('utf-8')))
ret.append((p, pkg.decode("utf-8")))
for p, pkg in sorted(ret):
yield pkg
else:
try:
yield self._db[path].split(b"/")[-1].decode('utf-8')
yield self._db[path].split(b"/")[-1].decode("utf-8")
except KeyError:
pass
@ -243,7 +251,7 @@ class AptCachedContentsFileSearcher(FileSearcher):
start_time = datetime.now()
for path, rest in read_contents_file(f.readlines()):
self[path] = rest
logging.debug('Read %s in %s', url, datetime.now() - start_time)
logging.debug("Read %s in %s", url, datetime.now() - start_time)
class GeneratedFileSearcher(FileSearcher):
@ -257,12 +265,14 @@ class GeneratedFileSearcher(FileSearcher):
return self
def load_from_path(self, path):
with open(path, 'r') as f:
with open(path, "r") as f:
for line in f:
(path, pkg) = line.strip().split(None, 1)
self._db[path] = pkg
def search_files(self, path: str, regex: bool = False, case_insensitive: bool = False) -> Iterator[str]:
def search_files(
self, path: str, regex: bool = False, case_insensitive: bool = False
) -> Iterator[str]:
for p, pkg in sorted(self._db.items()):
if regex:
flags = 0
@ -294,13 +304,17 @@ GENERATED_FILE_SEARCHER = GeneratedFileSearcher(
def get_packages_for_paths(
paths: List[str], searchers: List[FileSearcher], regex: bool = False,
case_insensitive: bool = False
paths: List[str],
searchers: List[FileSearcher],
regex: bool = False,
case_insensitive: bool = False,
) -> List[str]:
candidates: List[str] = list()
for path in paths:
for searcher in searchers:
for pkg in searcher.search_files(path, regex=regex, case_insensitive=case_insensitive):
for pkg in searcher.search_files(
path, regex=regex, case_insensitive=case_insensitive
):
if pkg not in candidates:
candidates.append(pkg)
return candidates
@ -308,10 +322,11 @@ def get_packages_for_paths(
def main(argv):
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('path', help='Path to search for.', type=str, nargs='*')
parser.add_argument('--regex', '-x', help='Search for regex.', action='store_true')
parser.add_argument('--debug', action='store_true')
parser.add_argument("path", help="Path to search for.", type=str, nargs="*")
parser.add_argument("--regex", "-x", help="Search for regex.", action="store_true")
parser.add_argument("--debug", action="store_true")
args = parser.parse_args()
if args.debug:
@ -328,6 +343,7 @@ def main(argv):
print(package)
if __name__ == '__main__':
if __name__ == "__main__":
import sys
sys.exit(main(sys.argv))

137
ognibuild/debian/fix_build.py
View file

@ -19,13 +19,12 @@ __all__ = [
"build_incrementally",
]
from datetime import datetime
from functools import partial
import logging
import os
import shutil
import sys
from typing import List, Set, Optional, Type, Tuple
from typing import List, Set, Optional, Type
from debian.deb822 import (
Deb822,
@ -33,7 +32,6 @@ from debian.deb822 import (
)
from breezy.commit import PointlessCommit
from breezy.mutabletree import MutableTree
from breezy.tree import Tree
from debmutate.changelog import ChangelogEditor
from debmutate.control import (
@ -54,18 +52,20 @@ from debmutate.reformatting import (
try:
from breezy.workspace import reset_tree
except ImportError: # breezy < 3.2
def delete_items(deletables, dry_run=False):
"""Delete files in the deletables iterable"""
import errno
import shutil
def onerror(function, path, excinfo):
"""Show warning for errors seen by rmtree.
"""
"""Show warning for errors seen by rmtree."""
# Handle only permission error while removing files.
# Other errors are re-raised.
if function is not os.remove or excinfo[1].errno != errno.EACCES:
raise
logging.warning('unable to remove %s' % path)
logging.warning("unable to remove %s" % path)
for path, subp in deletables:
if os.path.isdir(path):
shutil.rmtree(path, onerror=onerror)
@ -78,13 +78,18 @@ except ImportError: # breezy < 3.2
raise e
logging.warning('unable to remove "%s": %s.', path, e.strerror)
def reset_tree(local_tree, subpath=''):
def reset_tree(local_tree, subpath=""):
from breezy.transform import revert
from breezy.clean_tree import iter_deletables
revert(local_tree, local_tree.branch.basis_tree(),
[subpath] if subpath not in ('.', '') else None)
deletables = list(iter_deletables(
local_tree, unknown=True, ignored=False, detritus=False))
revert(
local_tree,
local_tree.branch.basis_tree(),
[subpath] if subpath not in (".", "") else None,
)
deletables = list(
iter_deletables(local_tree, unknown=True, ignored=False, detritus=False)
)
delete_items(deletables)
@ -103,8 +108,6 @@ from buildlog_consultant.common import (
MissingAutomakeInput,
MissingConfigure,
NeedPgBuildExtUpdateControl,
MissingPythonModule,
MissingPythonDistribution,
MissingPerlFile,
)
from buildlog_consultant.sbuild import (
@ -130,8 +133,9 @@ class CircularDependency(Exception):
class DebianPackagingContext(object):
def __init__(self, tree, subpath, committer, update_changelog, commit_reporter=None):
def __init__(
self, tree, subpath, committer, update_changelog, commit_reporter=None
):
self.tree = tree
self.subpath = subpath
self.committer = committer
@ -144,14 +148,18 @@ class DebianPackagingContext(object):
with self.tree.lock_write():
try:
if update_changelog:
cl_path = self.tree.abspath(os.path.join(self.subpath, "debian/changelog"))
cl_path = self.tree.abspath(
os.path.join(self.subpath, "debian/changelog")
)
with ChangelogEditor(cl_path) as editor:
editor.add_entry([summary])
debcommit(self.tree, committer=self.committer, subpath=self.subpath)
else:
self.tree.commit(
message=summary, committer=self.committer, specific_files=[self.subpath],
reporter=self.commit_reporter
message=summary,
committer=self.committer,
specific_files=[self.subpath],
reporter=self.commit_reporter,
)
except PointlessCommit:
return False
@ -160,7 +168,6 @@ class DebianPackagingContext(object):
class PackageDependencyFixer(BuildFixer):
def __init__(self, context, apt_resolver):
self.apt_resolver = apt_resolver
self.context = context
@ -203,7 +210,7 @@ def add_dependency(context, phase, requirement: AptRequirement):
elif phase[0] == "build":
return add_build_dependency(context, requirement)
else:
logging.warning('Unknown phase %r', phase)
logging.warning("Unknown phase %r", phase)
return False
@ -239,7 +246,9 @@ def add_test_dependency(context, testname, requirement):
if not isinstance(requirement, AptRequirement):
raise TypeError(requirement)
tests_control_path = os.path.join(context.tree.abspath(context.subpath), "debian/tests/control")
tests_control_path = os.path.join(
context.tree.abspath(context.subpath), "debian/tests/control"
)
try:
with Deb822Editor(path=tests_control_path) as updater:
@ -293,18 +302,22 @@ def python_tie_breaker(tree, subpath, reqs):
return None
def same(pkg, python_version):
if pkg.startswith(python_version + '-'):
if pkg.startswith(python_version + "-"):
return True
if pkg.startswith('lib%s-' % python_version):
if pkg.startswith("lib%s-" % python_version):
return True
return False
for python_version in targeted:
for req in reqs:
if any(same(name, python_version) for name in req.package_names()):
logging.info(
'Breaking tie between %r to %r, since package already '
'has %r build-dependencies', [str(req) for req in reqs],
str(req), python_version)
"Breaking tie between %r to %r, since package already "
"has %r build-dependencies",
[str(req) for req in reqs],
str(req),
python_version,
)
return req
return None
@ -327,7 +340,9 @@ def enable_dh_autoreconf(context, phase):
return dh_invoke_add_with(line, b"autoreconf")
if update_rules(command_line_cb=add_with_autoreconf):
return add_dependency(context, phase, AptRequirement.simple("dh-autoreconf"))
return add_dependency(
context, phase, AptRequirement.simple("dh-autoreconf")
)
return False
@ -385,10 +400,12 @@ class PgBuildExtOutOfDateControlFixer(BuildFixer):
logging.info("Running 'pg_buildext updatecontrol'")
self.session.check_call(["pg_buildext", "updatecontrol"])
shutil.copy(
self.session.external_path('debian/control'),
context.tree.abspath(os.path.join(context.subpath, 'debian/control')))
self.session.external_path("debian/control"),
context.tree.abspath(os.path.join(context.subpath, "debian/control")),
)
return self.context.commit(
"Run 'pgbuildext updatecontrol'.", update_changelog=False)
"Run 'pgbuildext updatecontrol'.", update_changelog=False
)
def fix_missing_makefile_pl(error, phase, context):
@ -410,7 +427,10 @@ class SimpleBuildFixer(BuildFixer):
def __repr__(self):
return "%s(%s, %s)" % (
type(self).__name__, self._problem_cls.__name__, self._fn.__name__)
type(self).__name__,
self._problem_cls.__name__,
self._fn.__name__,
)
def can_fix(self, problem: Problem):
return isinstance(problem, self._problem_cls)
@ -428,7 +448,10 @@ class DependencyBuildFixer(BuildFixer):
def __repr__(self):
return "%s(%s, %s)" % (
type(self).__name__, self._problem_cls.__name__, self._fn.__name__)
type(self).__name__,
self._problem_cls.__name__,
self._fn.__name__,
)
def can_fix(self, problem: Problem):
return isinstance(problem, self._problem_cls)
@ -441,8 +464,12 @@ def versioned_package_fixers(session, packaging_context):
return [
PgBuildExtOutOfDateControlFixer(packaging_context, session),
SimpleBuildFixer(packaging_context, MissingConfigure, fix_missing_configure),
SimpleBuildFixer(packaging_context, MissingAutomakeInput, fix_missing_automake_input),
SimpleBuildFixer(packaging_context, MissingConfigStatusInput, fix_missing_config_status_input),
SimpleBuildFixer(
packaging_context, MissingAutomakeInput, fix_missing_automake_input
),
SimpleBuildFixer(
packaging_context, MissingConfigStatusInput, fix_missing_config_status_input
),
SimpleBuildFixer(packaging_context, MissingPerlFile, fix_missing_makefile_pl),
]
@ -451,6 +478,7 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]:
from ..resolver.apt import AptResolver
from .udd import popcon_tie_breaker
from .build_deps import BuildDependencyTieBreaker
apt_tie_breakers = [
partial(python_tie_breaker, packaging_context.tree, packaging_context.subpath),
BuildDependencyTieBreaker.from_session(apt.session),
@ -458,7 +486,9 @@ def apt_fixers(apt, packaging_context) -> List[BuildFixer]:
]
resolver = AptResolver(apt, apt_tie_breakers)
return [
DependencyBuildFixer(packaging_context, apt, AptFetchFailure, retry_apt_failure),
DependencyBuildFixer(
packaging_context, apt, AptFetchFailure, retry_apt_failure
),
PackageDependencyFixer(packaging_context, resolver),
]
@ -479,9 +509,11 @@ def build_incrementally(
):
fixed_errors = []
packaging_context = DebianPackagingContext(
local_tree, subpath, committer, update_changelog)
fixers = (versioned_package_fixers(apt.session, packaging_context) +
apt_fixers(apt, packaging_context))
local_tree, subpath, committer, update_changelog
)
fixers = versioned_package_fixers(apt.session, packaging_context) + apt_fixers(
apt, packaging_context
)
logging.info("Using fixers: %r", fixers)
while True:
try:
@ -534,11 +566,8 @@ def build_incrementally(
):
i += 1
target_path = os.path.join(output_directory, "build.log.%d" % i)
os.rename(
os.path.join(output_directory, "build.log"),
target_path
)
logging.debug('Storing build log at %s', target_path)
os.rename(os.path.join(output_directory, "build.log"), target_path)
logging.debug("Storing build log at %s", target_path)
def main(argv=None):
@ -577,19 +606,13 @@ def main(argv=None):
help="force updating of the changelog",
default=None,
)
parser.add_argument(
'--schroot',
type=str,
help='chroot to use.')
parser.add_argument(
'--verbose',
action='store_true',
help='Be verbose')
parser.add_argument("--schroot", type=str, help="chroot to use.")
parser.add_argument("--verbose", action="store_true", help="Be verbose")
args = parser.parse_args()
from breezy.workingtree import WorkingTree
import breezy.git
import breezy.bzr
import breezy.git # noqa: F401
import breezy.bzr # noqa: F401
from .apt import AptManager
from ..session.plain import PlainSession
from ..session.schroot import SchrootSession
@ -632,15 +655,15 @@ def main(argv=None):
)
except SbuildFailure as e:
if e.phase is None:
phase = 'unknown phase'
phase = "unknown phase"
elif len(e.phase) == 1:
phase = e.phase[0]
else:
phase = '%s (%s)' % (e.phase[0], e.phase[1])
phase = "%s (%s)" % (e.phase[0], e.phase[1])
if e.error:
logging.fatal('Error during %s: %s', phase, e.error)
logging.fatal("Error during %s: %s", phase, e.error)
else:
logging.fatal('Error during %s: %s', phase, e.description)
logging.fatal("Error during %s: %s", phase, e.description)
return 1

13
ognibuild/debian/udd.py
View file

@ -21,7 +21,6 @@ import logging
class UDD(object):
def connect(self):
import psycopg2
@ -36,8 +35,9 @@ class UDD(object):
def get_most_popular(self, packages):
cursor = self._conn.cursor()
cursor.execute(
'SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1',
(tuple(packages), ))
"SELECT package FROM popcon WHERE package IN %s ORDER BY insts DESC LIMIT 1",
(tuple(packages),),
)
return cursor.fetchone()[0]
@ -47,15 +47,14 @@ def popcon_tie_breaker(candidates):
try:
from .udd import UDD
except ModuleNotFoundError:
logging.warning('Unable to import UDD, not ranking by popcon')
logging.warning("Unable to import UDD, not ranking by popcon")
return sorted(candidates, key=len)[0]
udd = UDD()
udd.connect()
names = {list(c.package_names())[0]: c for c in candidates}
winner = udd.get_most_popular(list(names.keys()))
if winner is None:
logging.warning(
'No relevant popcon information found, not ranking by popcon')
logging.warning("No relevant popcon information found, not ranking by popcon")
return None
logging.info('Picked winner using popcon')
logging.info("Picked winner using popcon")
return names[winner]

51
ognibuild/dist.py
View file

@ -16,16 +16,15 @@
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
__all__ = [
'UnidentifiedError',
'DetailedFailure',
'create_dist',
'create_dist_schroot',
"UnidentifiedError",
"DetailedFailure",
"create_dist",
"create_dist_schroot",
]
import errno
import logging
import os
import shutil
import sys
from typing import Optional, List
@ -54,7 +53,8 @@ def run_dist(session, buildsystems, resolver, fixers, target_directory, quiet=Fa
for buildsystem in buildsystems:
filename = buildsystem.dist(
session, resolver, fixers, target_directory, quiet=quiet)
session, resolver, fixers, target_directory, quiet=quiet
)
return filename
raise NoBuildToolsFound()
@ -66,20 +66,23 @@ def create_dist(
target_dir: str,
include_controldir: bool = True,
subdir: Optional[str] = None,
cleanup: bool = False
cleanup: bool = False,
) -> Optional[str]:
from .buildsystem import detect_buildsystems
from .buildlog import InstallFixer
from .fix_build import BuildFixer
from .fixers import (
GitIdentityFixer, SecretGpgKeyFixer,
UnexpandedAutoconfMacroFixer, )
GitIdentityFixer,
SecretGpgKeyFixer,
UnexpandedAutoconfMacroFixer,
)
if subdir is None:
subdir = "package"
try:
export_directory, reldir = session.setup_from_vcs(
tree, include_controldir=include_controldir, subdir=subdir)
tree, include_controldir=include_controldir, subdir=subdir
)
except OSError as e:
if e.errno == errno.ENOSPC:
raise DetailedFailure(1, ["mkdtemp"], NoSpaceOnDevice())
@ -94,8 +97,7 @@ def create_dist(
if session.is_temporary:
# Only muck about with temporary sessions
fixers.extend([
GitIdentityFixer(session), SecretGpgKeyFixer(session)])
fixers.extend([GitIdentityFixer(session), SecretGpgKeyFixer(session)])
session.chdir(reldir)
return run_dist(session, buildsystems, resolver, fixers, target_dir)
@ -109,7 +111,7 @@ def create_dist_schroot(
packaging_subpath: Optional[str] = None,
include_controldir: bool = True,
subdir: Optional[str] = None,
cleanup: bool = False
cleanup: bool = False,
) -> Optional[str]:
with SchrootSession(chroot) as session:
if packaging_tree is not None:
@ -117,10 +119,13 @@ def create_dist_schroot(
satisfy_build_deps(session, packaging_tree, packaging_subpath)
return create_dist(
session, tree, target_dir,
session,
tree,
target_dir,
include_controldir=include_controldir,
subdir=subdir,
cleanup=cleanup)
cleanup=cleanup,
)
if __name__ == "__main__":
@ -151,8 +156,8 @@ if __name__ == "__main__":
)
parser.add_argument("--verbose", action="store_true", help="Be verbose")
parser.add_argument(
'--include-controldir', action='store_true',
help='Clone rather than export.')
"--include-controldir", action="store_true", help="Clone rather than export."
)
args = parser.parse_args()
@ -185,15 +190,17 @@ if __name__ == "__main__":
logging.info("No build tools found, falling back to simple export.")
export(tree, "dist.tar.gz", "tgz", None)
except NotImplementedError:
logging.info("Build system does not support dist tarball creation, "
"falling back to simple export.")
logging.info(
"Build system does not support dist tarball creation, "
"falling back to simple export."
)
export(tree, "dist.tar.gz", "tgz", None)
except UnidentifiedError as e:
logging.fatal('Unidentified error: %r', e.lines)
logging.fatal("Unidentified error: %r", e.lines)
except DetailedFailure as e:
logging.fatal('Identified error during dist creation: %s', e.error)
logging.fatal("Identified error during dist creation: %s", e.error)
except DistNoTarball:
logging.fatal('dist operation did not create a tarball')
logging.fatal("dist operation did not create a tarball")
else:
logging.info("Created %s", ret)
sys.exit(0)

13
ognibuild/dist_catcher.py
View file

@ -53,17 +53,17 @@ class DistCatcher(object):
@classmethod
def default(cls, directory):
return cls([
os.path.join(directory, 'dist'),
directory,
os.path.join(directory, '..')])
return cls(
[os.path.join(directory, "dist"), directory, os.path.join(directory, "..")]
)
def __enter__(self):
self.existing_files = {}
for directory in self.directories:
try:
self.existing_files[directory] = {
entry.name: entry for entry in os.scandir(directory)}
entry.name: entry for entry in os.scandir(directory)
}
except FileNotFoundError:
self.existing_files[directory] = {}
return self
@ -92,7 +92,8 @@ class DistCatcher(object):
return entry.name
elif len(possible_new) > 1:
logging.warning(
"Found multiple tarballs %r in %s.", possible_new, directory)
"Found multiple tarballs %r in %s.", possible_new, directory
)
return
if len(possible_updated) == 1:

20
ognibuild/fix_build.py
View file

@ -17,17 +17,15 @@
from functools import partial
import logging
from typing import List, Optional, Tuple, Callable, Any
from typing import List, Tuple, Callable, Any
from buildlog_consultant import Problem
from buildlog_consultant.common import (
find_build_failure_description,
MissingCommand,
)
from breezy.mutabletree import MutableTree
from . import DetailedFailure, UnidentifiedError
from .debian.apt import AptManager
from .session import Session, run_with_tee
@ -55,7 +53,7 @@ def run_detecting_problems(session: Session, args: List[str], **kwargs):
else:
if retcode == 0:
return contents
lines = ''.join(contents).splitlines(False)
lines = "".join(contents).splitlines(False)
match, error = find_build_failure_description(lines)
if error is None:
if match:
@ -91,22 +89,26 @@ def iterate_with_build_fixers(fixers: List[BuildFixer], cb: Callable[[], Any]):
try:
resolved = resolve_error(f.error, None, fixers=fixers)
except DetailedFailure as n:
logging.info('New error %r while resolving %r', n, f)
logging.info("New error %r while resolving %r", n, f)
if n in to_resolve:
raise
to_resolve.append(f)
to_resolve.append(n)
else:
if not resolved:
logging.warning("Failed to find resolution for error %r. Giving up.", f.error)
logging.warning(
"Failed to find resolution for error %r. Giving up.", f.error
)
raise f
fixed_errors.append(f.error)
def run_with_build_fixers(session: Session, args: List[str], fixers: List[BuildFixer], **kwargs):
def run_with_build_fixers(
session: Session, args: List[str], fixers: List[BuildFixer], **kwargs
):
return iterate_with_build_fixers(
fixers,
partial(run_detecting_problems, session, args, **kwargs))
fixers, partial(run_detecting_problems, session, args, **kwargs)
)
def resolve_error(error, phase, fixers):

23
ognibuild/fixers.py
View file

@ -31,7 +31,6 @@ from .fix_build import BuildFixer
class GitIdentityFixer(BuildFixer):
def __init__(self, session):
self.session = session
@ -39,16 +38,17 @@ class GitIdentityFixer(BuildFixer):
return isinstance(problem, MissingGitIdentity)
def _fix(self, problem: Problem, phase: Tuple[str, ...]):
for name in ['user.email', 'user.name']:
value = subprocess.check_output(
['git', 'config', '--global', name]).decode().strip()
self.session.check_call(
['git', 'config', '--global', name, value])
for name in ["user.email", "user.name"]:
value = (
subprocess.check_output(["git", "config", "--global", name])
.decode()
.strip()
)
self.session.check_call(["git", "config", "--global", name, value])
return True
class SecretGpgKeyFixer(BuildFixer):
def __init__(self, session):
self.session = session
@ -67,8 +67,10 @@ Expire-Date: 0
Passphrase: ""
"""
p = self.session.Popen(
['gpg', '--gen-key', '--batch', '/dev/stdin'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
["gpg", "--gen-key", "--batch", "/dev/stdin"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
p.communicate(SCRIPT)
if p.returncode == 0:
return True
@ -76,7 +78,6 @@ Passphrase: ""
class UnexpandedAutoconfMacroFixer(BuildFixer):
def __init__(self, session, resolver):
self.session = session
self.resolver = resolver
@ -97,6 +98,6 @@ class UnexpandedAutoconfMacroFixer(BuildFixer):
return False
from .fix_build import run_detecting_problems
run_detecting_problems(self.session, ['autoconf', '-f'])
run_detecting_problems(self.session, ["autoconf", "-f"])
return True

73
ognibuild/requirements.py
View file

@ -19,7 +19,7 @@
import posixpath
import re
import subprocess
from typing import Optional, List, Tuple, Set
from typing import Optional, List, Set
from . import Requirement
@ -72,19 +72,24 @@ class PythonPackageRequirement(Requirement):
cmd = "python3"
else:
raise NotImplementedError
text = self.package + ','.join([''.join(spec) for spec in self.specs])
text = self.package + ",".join(["".join(spec) for spec in self.specs])
p = session.Popen(
[cmd, "-c", "import pkg_resources; pkg_resources.require(%r)" % text],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.communicate()
return p.returncode == 0
class PhpPackageRequirement(Requirement):
def __init__(self, package: str, channel: Optional[str] = None,
def __init__(
self,
package: str,
channel: Optional[str] = None,
min_version: Optional[str] = None,
max_version: Optional[str] = None):
max_version: Optional[str] = None,
):
self.package = package
self.channel = channel
self.min_version = min_version
@ -92,8 +97,12 @@ class PhpPackageRequirement(Requirement):
def __repr__(self):
return "%s(%r, %r, %r, %r)" % (
type(self).__name__, self.package, self.channel,
self.min_version, self.max_version)
type(self).__name__,
self.package,
self.channel,
self.min_version,
self.max_version,
)
class BinaryRequirement(Requirement):
@ -109,8 +118,10 @@ class BinaryRequirement(Requirement):
def met(self, session):
p = session.Popen(
["which", self.binary_name], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
["which", self.binary_name],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.communicate()
return p.returncode == 0
@ -144,7 +155,7 @@ class VagueDependencyRequirement(Requirement):
self.name = name
def expand(self):
if ' ' not in self.name:
if " " not in self.name:
yield BinaryRequirement(self.name)
yield LibraryRequirement(self.name)
yield PkgConfigRequirement(self.name)
@ -153,6 +164,7 @@ class VagueDependencyRequirement(Requirement):
yield LibraryRequirement(self.name.lower())
yield PkgConfigRequirement(self.name.lower())
from .resolver.apt import AptRequirement
yield AptRequirement.simple(self.name.lower())
def met(self, session):
@ -208,17 +220,18 @@ class CargoCrateRequirement(Requirement):
type(self).__name__,
self.crate,
self.features,
self.version
self.version,
)
def __str__(self):
if self.features:
return "cargo crate: %s %s (%s)" % (
self.crate, self.version or '',
', '.join(sorted(self.features)))
self.crate,
self.version or "",
", ".join(sorted(self.features)),
)
else:
return "cargo crate: %s %s" % (
self.crate, self.version or '')
return "cargo crate: %s %s" % (self.crate, self.version or "")
class PkgConfigRequirement(Requirement):
@ -346,10 +359,10 @@ class RPackageRequirement(Requirement):
@classmethod
def from_str(cls, text):
# TODO(jelmer): More complex parser
m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text)
m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text)
if m:
return cls(m.group(1), m.group(2))
m = re.fullmatch(r'([^ ]+)', text)
m = re.fullmatch(r"([^ ]+)", text)
if m:
return cls(m.group(1))
raise ValueError(text)
@ -381,10 +394,10 @@ class OctavePackageRequirement(Requirement):
@classmethod
def from_str(cls, text):
# TODO(jelmer): More complex parser
m = re.fullmatch(r'(.*)\s+\(>=\s+(.*)\)', text)
m = re.fullmatch(r"(.*)\s+\(>=\s+(.*)\)", text)
if m:
return cls(m.group(1), m.group(2))
m = re.fullmatch(r'([^ ]+)', text)
m = re.fullmatch(r"([^ ]+)", text)
if m:
return cls(m.group(1))
raise ValueError(text)
@ -468,11 +481,14 @@ class MavenArtifactRequirement(Requirement):
def __str__(self):
return "maven requirement: %s:%s:%s" % (
self.group_id, self.artifact_id, self.version)
self.group_id,
self.artifact_id,
self.version,
)
@classmethod
def from_str(cls, text):
return cls.from_tuple(text.split(':'))
return cls.from_tuple(text.split(":"))
@classmethod
def from_tuple(cls, parts):
@ -486,8 +502,7 @@ class MavenArtifactRequirement(Requirement):
(group_id, artifact_id) = parts
kind = "jar"
else:
raise ValueError(
"invalid number of parts to artifact %r" % parts)
raise ValueError("invalid number of parts to artifact %r" % parts)
return cls(group_id, artifact_id, version, kind)
@ -512,31 +527,26 @@ class JDKFileRequirement(Requirement):
class JDKRequirement(Requirement):
def __init__(self):
super(JDKRequirement, self).__init__("jdk")
class JRERequirement(Requirement):
def __init__(self):
super(JRERequirement, self).__init__("jre")
class QTRequirement(Requirement):
def __init__(self):
super(QTRequirement, self).__init__("qt")
class X11Requirement(Requirement):
def __init__(self):
super(X11Requirement, self).__init__("x11")
class CertificateAuthorityRequirement(Requirement):
def __init__(self, url):
super(CertificateAuthorityRequirement, self).__init__("ca-cert")
self.url = url
@ -561,7 +571,6 @@ class AutoconfMacroRequirement(Requirement):
class LibtoolRequirement(Requirement):
def __init__(self):
super(LibtoolRequirement, self).__init__("libtool")
@ -593,6 +602,8 @@ class PythonModuleRequirement(Requirement):
raise NotImplementedError
p = session.Popen(
[cmd, "-c", "import %s" % self.module],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
p.communicate()
return p.returncode == 0

37
ognibuild/resolver/__init__.py
View file

@ -19,6 +19,7 @@
import subprocess
from ..fix_build import run_detecting_problems
class UnsatisfiedRequirements(Exception):
def __init__(self, reqs):
self.requirements = reqs
@ -53,7 +54,7 @@ class CPANResolver(Resolver):
def _cmd(self, reqs):
ret = ["cpan", "-i"]
if self.skip_tests:
ret.append('-T')
ret.append("-T")
ret.extend([req.module for req in reqs])
return ret
@ -111,7 +112,11 @@ class RResolver(Resolver):
def _cmd(self, req):
# TODO(jelmer: Handle self.user_local
return ["R", "-e", "install.packages('%s', repos=%r)" % (req.package, self.repos)]
return [
"R",
"-e",
"install.packages('%s', repos=%r)" % (req.package, self.repos),
]
def explain(self, requirements):
from ..requirements import RPackageRequirement
@ -187,16 +192,17 @@ class OctaveForgeResolver(Resolver):
class CRANResolver(RResolver):
def __init__(self, session, user_local=False):
super(CRANResolver, self).__init__(session, 'http://cran.r-project.org', user_local=user_local)
super(CRANResolver, self).__init__(
session, "http://cran.r-project.org", user_local=user_local
)
class BioconductorResolver(RResolver):
def __init__(self, session, user_local=False):
super(BioconductorResolver, self).__init__(
session, 'https://hedgehog.fhcrc.org/bioconductor', user_local=user_local)
session, "https://hedgehog.fhcrc.org/bioconductor", user_local=user_local
)
class HackageResolver(Resolver):
@ -213,7 +219,7 @@ class HackageResolver(Resolver):
def _cmd(self, reqs):
extra_args = []
if self.user_local:
extra_args.append('--user')
extra_args.append("--user")
return ["cabal", "install"] + extra_args + [req.package for req in reqs]
def install(self, requirements):
@ -259,7 +265,7 @@ class PypiResolver(Resolver):
def _cmd(self, reqs):
extra_args = []
if self.user_local:
extra_args.append('--user')
extra_args.append("--user")
return ["pip", "install"] + extra_args + [req.package for req in reqs]
def install(self, requirements):
@ -276,8 +282,7 @@ class PypiResolver(Resolver):
missing.append(requirement)
continue
try:
self.session.check_call(
self._cmd([requirement]), user=user)
self.session.check_call(self._cmd([requirement]), user=user)
except subprocess.CalledProcessError:
missing.append(requirement)
if missing:
@ -296,7 +301,6 @@ class PypiResolver(Resolver):
class GoResolver(Resolver):
def __init__(self, session, user_local):
self.session = session
self.user_local = user_local
@ -314,7 +318,7 @@ class GoResolver(Resolver):
env = {}
else:
# TODO(jelmer): Isn't this Debian-specific?
env = {'GOPATH': '/usr/share/gocode'}
env = {"GOPATH": "/usr/share/gocode"}
missing = []
for requirement in requirements:
@ -334,8 +338,7 @@ class GoResolver(Resolver):
continue
goreqs.append(requirement)
if goreqs:
yield (["go", "get"] + [req.package for req in goreqs],
goreqs)
yield (["go", "get"] + [req.package for req in goreqs], goreqs)
NPM_COMMAND_PACKAGES = {
@ -379,13 +382,13 @@ class NpmResolver(Resolver):
requirement = NodePackageRequirement(package)
if isinstance(requirement, NodeModuleRequirement):
# TODO: Is this legit?
requirement = NodePackageRequirement(requirement.module.split('/')[0])
requirement = NodePackageRequirement(requirement.module.split("/")[0])
if not isinstance(requirement, NodePackageRequirement):
missing.append(requirement)
continue
self.session.check_call(
["npm", "-g", "install", requirement.package],
user=user)
["npm", "-g", "install", requirement.package], user=user
)
if missing:
raise UnsatisfiedRequirements(missing)

168
ognibuild/resolver/apt.py
View file

@ -115,19 +115,27 @@ class AptRequirement(Requirement):
return False
def find_package_names(apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False) -> List[str]:
def find_package_names(
apt_mgr: AptManager, paths: List[str], regex: bool = False, case_insensitive=False
) -> List[str]:
if not isinstance(paths, list):
raise TypeError(paths)
return apt_mgr.get_packages_for_paths(paths, regex, case_insensitive)
def find_reqs_simple(
apt_mgr: AptManager, paths: List[str], regex: bool = False,
minimum_version=None, case_insensitive=False) -> List[str]:
apt_mgr: AptManager,
paths: List[str],
regex: bool = False,
minimum_version=None,
case_insensitive=False,
) -> List[str]:
if not isinstance(paths, list):
raise TypeError(paths)
return [AptRequirement.simple(package, minimum_version=minimum_version)
for package in find_package_names(apt_mgr, paths, regex, case_insensitive)]
return [
AptRequirement.simple(package, minimum_version=minimum_version)
for package in find_package_names(apt_mgr, paths, regex, case_insensitive)
]
def python_spec_to_apt_rels(pkg_name, specs):
@ -138,36 +146,57 @@ def python_spec_to_apt_rels(pkg_name, specs):
rels = []
for spec in specs:
deb_version = Version(spec[1])
if spec[0] == '~=':
if spec[0] == "~=":
# PEP 440: For a given release identifier V.N , the compatible
# release clause is approximately equivalent to the pair of
# comparison clauses: >= V.N, == V.*
parts = spec[1].split('.')
parts = spec[1].split(".")
parts.pop(-1)
parts[-1] = str(int(parts[-1]) + 1)
next_maj_deb_version = Version('.'.join(parts))
rels.extend([{"name": pkg_name, "version": ('>=', deb_version)},
{"name": pkg_name, "version": ('<<', next_maj_deb_version)}])
elif spec[0] == '!=':
rels.extend([{"name": pkg_name, "version": ('>>', deb_version)},
{"name": pkg_name, "version": ('<<', deb_version)}])
elif spec[1].endswith('.*') and spec[0] == '==':
s = spec[1].split('.')
next_maj_deb_version = Version(".".join(parts))
rels.extend(
[
{"name": pkg_name, "version": (">=", deb_version)},
{"name": pkg_name, "version": ("<<", next_maj_deb_version)},
]
)
elif spec[0] == "!=":
rels.extend(
[
{"name": pkg_name, "version": (">>", deb_version)},
{"name": pkg_name, "version": ("<<", deb_version)},
]
)
elif spec[1].endswith(".*") and spec[0] == "==":
s = spec[1].split(".")
s.pop(-1)
n = list(s)
n[-1] = str(int(n[-1]) + 1)
rels.extend([{"name": pkg_name, "version": ('>=', Version('.'.join(s)))},
{"name": pkg_name, "version": ('<<', Version('.'.join(n)))}])
rels.extend(
[
{"name": pkg_name, "version": (">=", Version(".".join(s)))},
{"name": pkg_name, "version": ("<<", Version(".".join(n)))},
]
)
else:
c = {">=": ">=", "<=": "<=", "<": "<<", ">": ">>", "==": "="}[spec[0]]
rels.append([{"name": pkg_name, "version": (c, deb_version)}])
return rels
def get_package_for_python_package(apt_mgr, package, python_version: Optional[str], specs=None):
pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_"))
cpython2_regex = "/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_"))
cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(package.replace("-", "_"))
def get_package_for_python_package(
apt_mgr, package, python_version: Optional[str], specs=None
):
pypy_regex = "/usr/lib/pypy/dist-packages/%s-.*.egg-info" % re.escape(
package.replace("-", "_")
)
cpython2_regex = (
"/usr/lib/python2\\.[0-9]/dist-packages/%s-.*.egg-info"
% re.escape(package.replace("-", "_"))
)
cpython3_regex = "/usr/lib/python3/dist-packages/%s-.*.egg-info" % re.escape(
package.replace("-", "_")
)
if python_version == "pypy":
paths = [pypy_regex]
elif python_version == "cpython2":
@ -177,7 +206,7 @@ def get_package_for_python_package(apt_mgr, package, python_version: Optional[st
elif python_version is None:
paths = [cpython3_regex, cpython2_regex, pypy_regex]
else:
raise NotImplementedError('unsupported python version %s' % python_version)
raise NotImplementedError("unsupported python version %s" % python_version)
names = find_package_names(apt_mgr, paths, regex=True, case_insensitive=True)
return [AptRequirement(python_spec_to_apt_rels(name, specs)) for name in names]
@ -190,7 +219,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
"__init__.py",
),
posixpath.join(
"/usr/lib/python3/dist-packages", re.escape(module.replace(".", "/")) + ".py"
"/usr/lib/python3/dist-packages",
re.escape(module.replace(".", "/")) + ".py",
),
posixpath.join(
"/usr/lib/python3\\.[0-9]+/lib-dynload",
@ -200,7 +230,9 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
"/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")) + ".py"
),
posixpath.join(
"/usr/lib/python3\\.[0-9]+/", re.escape(module.replace(".", "/")), "__init__.py"
"/usr/lib/python3\\.[0-9]+/",
re.escape(module.replace(".", "/")),
"__init__.py",
),
]
cpython2_regexes = [
@ -220,7 +252,9 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
]
pypy_regexes = [
posixpath.join(
"/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")), "__init__.py"
"/usr/lib/pypy/dist-packages",
re.escape(module.replace(".", "/")),
"__init__.py",
),
posixpath.join(
"/usr/lib/pypy/dist-packages", re.escape(module.replace(".", "/")) + ".py"
@ -245,8 +279,8 @@ def get_package_for_python_module(apt_mgr, module, python_version, specs):
vague_map = {
'the Gnu Scientific Library': 'libgsl-dev',
'the required FreeType library': 'libfreetype-dev',
"the Gnu Scientific Library": "libgsl-dev",
"the required FreeType library": "libfreetype-dev",
}
@ -255,7 +289,7 @@ def resolve_vague_dep_req(apt_mgr, req):
options = []
if name in vague_map:
options.append(AptRequirement.simple(vague_map[name]))
if name.startswith('gnu '):
if name.startswith("gnu "):
name = name[4:]
for x in req.expand():
options.extend(resolve_requirement_apt(apt_mgr, x))
@ -273,13 +307,23 @@ def resolve_binary_req(apt_mgr, req):
def resolve_pkg_config_req(apt_mgr, req):
names = find_package_names(apt_mgr,
[posixpath.join("/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc")],
regex=True)
names = find_package_names(
apt_mgr,
[
posixpath.join(
"/usr/lib", ".*", "pkgconfig", re.escape(req.module) + "\\.pc"
)
],
regex=True,
)
if not names:
names = find_package_names(
apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")])
return [AptRequirement.simple(name, minimum_version=req.minimum_version) for name in names]
apt_mgr, [posixpath.join("/usr/lib/pkgconfig", req.module + ".pc")]
)
return [
AptRequirement.simple(name, minimum_version=req.minimum_version)
for name in names
]
def resolve_path_req(apt_mgr, req):
@ -288,12 +332,13 @@ def resolve_path_req(apt_mgr, req):
def resolve_c_header_req(apt_mgr, req):
reqs = find_reqs_simple(
apt_mgr,
[posixpath.join("/usr/include", req.header)], regex=False)
apt_mgr, [posixpath.join("/usr/include", req.header)], regex=False
)
if not reqs:
reqs = find_reqs_simple(
apt_mgr,
[posixpath.join("/usr/include", ".*", re.escape(req.header))], regex=True
[posixpath.join("/usr/include", ".*", re.escape(req.header))],
regex=True,
)
return reqs
@ -314,18 +359,21 @@ def resolve_ruby_gem_req(apt_mgr, req):
"specifications/%s-.*\\.gemspec" % re.escape(req.gem)
)
]
return find_reqs_simple(apt_mgr, paths, regex=True, minimum_version=req.minimum_version)
return find_reqs_simple(
apt_mgr, paths, regex=True, minimum_version=req.minimum_version
)
def resolve_go_package_req(apt_mgr, req):
return find_reqs_simple(
apt_mgr,
[posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")], regex=True
[posixpath.join("/usr/share/gocode/src", re.escape(req.package), ".*")],
regex=True,
)
def resolve_go_req(apt_mgr, req):
return [AptRequirement.simple('golang-go', minimum_version='2:%s' % req.version)]
return [AptRequirement.simple("golang-go", minimum_version="2:%s" % req.version)]
def resolve_dh_addon_req(apt_mgr, req):
@ -339,11 +387,15 @@ def resolve_php_class_req(apt_mgr, req):
def resolve_php_package_req(apt_mgr, req):
return [AptRequirement.simple('php-%s' % req.package, minimum_version=req.min_version)]
return [
AptRequirement.simple("php-%s" % req.package, minimum_version=req.min_version)
]
def resolve_r_package_req(apt_mgr, req):
paths = [posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))]
paths = [
posixpath.join("/usr/lib/R/site-library/.*/R/%s$" % re.escape(req.package))
]
return find_reqs_simple(apt_mgr, paths, regex=True)
@ -441,15 +493,17 @@ def resolve_maven_artifact_req(apt_mgr, req):
else:
version = req.version
regex = False
def escape(x):
return x
kind = req.kind or 'jar'
kind = req.kind or "jar"
path = posixpath.join(
escape("/usr/share/maven-repo"),
escape(req.group_id.replace(".", "/")),
escape(req.artifact_id),
version,
escape("%s-") + version + escape("." + kind)
escape("%s-") + version + escape("." + kind),
)
return find_reqs_simple(apt_mgr, [path], regex=regex)
@ -465,15 +519,15 @@ def resolve_jdk_file_req(apt_mgr, req):
def resolve_jdk_req(apt_mgr, req):
return [AptRequirement.simple('default-jdk')]
return [AptRequirement.simple("default-jdk")]
def resolve_jre_req(apt_mgr, req):
return [AptRequirement.simple('default-jre')]
return [AptRequirement.simple("default-jre")]
def resolve_x11_req(apt_mgr, req):
return [AptRequirement.simple('libx11-dev')]
return [AptRequirement.simple("libx11-dev")]
def resolve_qt_req(apt_mgr, req):
@ -554,13 +608,12 @@ def resolve_python_package_req(apt_mgr, req):
def resolve_cargo_crate_req(apt_mgr, req):
paths = [
'/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml' % re.escape(req.crate)]
paths = ["/usr/share/cargo/registry/%s-[0-9]+.*/Cargo.toml" % re.escape(req.crate)]
return find_reqs_simple(apt_mgr, paths, regex=True)
def resolve_ca_req(apt_mgr, req):
return [AptRequirement.simple('ca-certificates')]
return [AptRequirement.simple("ca-certificates")]
def resolve_apt_req(apt_mgr, req):
@ -670,7 +723,16 @@ class AptResolver(Resolver):
if apt_req is not None:
apt_requirements.append((r, apt_req))
if apt_requirements:
yield (self.apt.satisfy_command([PkgRelation.str(chain(*[r.relations for o, r in apt_requirements]))]), [o for o, r in apt_requirements])
yield (
self.apt.satisfy_command(
[
PkgRelation.str(
chain(*[r.relations for o, r in apt_requirements])
)
]
),
[o for o, r in apt_requirements],
)
def resolve(self, req: Requirement):
ret = resolve_requirement_apt(self.apt, req)
@ -678,14 +740,12 @@ class AptResolver(Resolver):
return None
if len(ret) == 1:
return ret[0]
logging.info('Need to break tie between %r with %r', ret, self.tie_breakers)
logging.info("Need to break tie between %r with %r", ret, self.tie_breakers)
for tie_breaker in self.tie_breakers:
winner = tie_breaker(ret)
if winner is not None:
if not isinstance(winner, AptRequirement):
raise TypeError(winner)
return winner
logging.info(
'Unable to break tie over %r, picking first: %r',
ret, ret[0])
logging.info("Unable to break tie over %r, picking first: %r", ret, ret[0])
return ret[0]

10
ognibuild/session/__init__.py
View file

@ -55,7 +55,7 @@ class Session(object):
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
close_fds: bool = True
close_fds: bool = True,
):
raise NotImplementedError(self.check_call)
@ -90,8 +90,8 @@ class Session(object):
raise NotImplementedError(self.scandir)
def setup_from_vcs(
self, tree, include_controldir: Optional[bool] = None,
subdir="package") -> Tuple[str, str]:
self, tree, include_controldir: Optional[bool] = None, subdir="package"
) -> Tuple[str, str]:
raise NotImplementedError(self.setup_from_vcs)
def setup_from_directory(self, path, subdir="package") -> Tuple[str, str]:
@ -108,8 +108,8 @@ class SessionSetupFailure(Exception):
def run_with_tee(session: Session, args: List[str], **kwargs):
if 'stdin' not in kwargs:
kwargs['stdin'] = subprocess.DEVNULL
if "stdin" not in kwargs:
kwargs["stdin"] = subprocess.DEVNULL
p = session.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
contents = []
while p.poll() is None:

22
ognibuild/session/plain.py
View file

@ -38,6 +38,7 @@ class PlainSession(Session):
raise NoSessionOpen(self)
if user is not None:
import getpass
if user != getpass.getuser():
args = ["sudo", "-u", user] + args
return args
@ -63,19 +64,23 @@ class PlainSession(Session):
pass
def check_call(
self, argv: List[str],
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
close_fds: bool = True):
close_fds: bool = True,
):
argv = self._prepend_user(user, argv)
return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds)
def check_output(
self, argv: List[str],
self,
argv: List[str],
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None) -> bytes:
env: Optional[Dict[str, str]] = None,
) -> bytes:
argv = self._prepend_user(user, argv)
return subprocess.check_output(argv, cwd=cwd, env=env)
@ -95,15 +100,16 @@ class PlainSession(Session):
def external_path(self, path):
return os.path.abspath(path)
def setup_from_vcs(
self, tree, include_controldir=None, subdir="package"):
def setup_from_vcs(self, tree, include_controldir=None, subdir="package"):
from ..vcs import dupe_vcs_tree, export_vcs_tree
if include_controldir is False or (
not hasattr(tree, 'base') and include_controldir is None):
not hasattr(tree, "base") and include_controldir is None
):
td = self.es.enter_context(tempfile.TemporaryDirectory())
export_vcs_tree(tree, td)
return td, td
elif not hasattr(tree, 'base'):
elif not hasattr(tree, "base"):
td = self.es.enter_context(tempfile.TemporaryDirectory())
dupe_vcs_tree(tree, td)
return td, td

25
ognibuild/session/schroot.py
View file

@ -57,12 +57,17 @@ class SchrootSession(Session):
if self.session_id is None:
raise NoSessionOpen(self)
try:
subprocess.check_output(["schroot", "-c", "session:" + self.session_id, "-e"], stderr=subprocess.PIPE)
subprocess.check_output(
["schroot", "-c", "session:" + self.session_id, "-e"],
stderr=subprocess.PIPE,
)
except subprocess.CalledProcessError as e:
for line in e.stderr.splitlines(False):
if line.startswith(b'E: '):
logging.error('%s', line[3:].decode(errors='replace'))
logging.warning('Failed to close schroot session %s, leaving stray.', self.session_id)
if line.startswith(b"E: "):
logging.error("%s", line[3:].decode(errors="replace"))
logging.warning(
"Failed to close schroot session %s, leaving stray.", self.session_id
)
self.session_id = None
return False
self.session_id = None
@ -134,10 +139,12 @@ class SchrootSession(Session):
cwd: Optional[str] = None,
user: Optional[str] = None,
env: Optional[Dict[str, str]] = None,
close_fds: bool = True
close_fds: bool = True,
):
try:
subprocess.check_call(self._run_argv(argv, cwd, user, env=env), close_fds=close_fds)
subprocess.check_call(
self._run_argv(argv, cwd, user, env=env), close_fds=close_fds
)
except subprocess.CalledProcessError as e:
raise subprocess.CalledProcessError(e.returncode, argv)
@ -191,9 +198,10 @@ class SchrootSession(Session):
return os.scandir(fullpath)
def setup_from_vcs(
self, tree, include_controldir: Optional[bool] = None,
subdir="package"):
self, tree, include_controldir: Optional[bool] = None, subdir="package"
):
from ..vcs import dupe_vcs_tree, export_vcs_tree
build_dir = os.path.join(self.location, "build")
directory = tempfile.mkdtemp(dir=build_dir)
@ -209,6 +217,7 @@ class SchrootSession(Session):
def setup_from_directory(self, path, subdir="package"):
import shutil
build_dir = os.path.join(self.location, "build")
directory = tempfile.mkdtemp(dir=build_dir)
reldir = "/" + os.path.relpath(directory, self.location)

7
ognibuild/tests/test_debian_fix_build.py
View file

@ -99,9 +99,12 @@ blah (0.1) UNRELEASED; urgency=medium
apt = AptManager(session)
apt._searchers = [DummyAptSearcher(self._apt_files)]
context = DebianPackagingContext(
self.tree, subpath="", committer="ognibuild <ognibuild@jelmer.uk>",
self.tree,
subpath="",
committer="ognibuild <ognibuild@jelmer.uk>",
update_changelog=True,
commit_reporter=NullCommitReporter())
commit_reporter=NullCommitReporter(),
)
fixers = versioned_package_fixers(session, context) + apt_fixers(apt, context)
return resolve_error(error, ("build",), fixers)