billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More refactoring.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-02-26 01:41:05 +00:00
parent 795bca3a13
commit aa2a3e47fa
11 changed files with 136 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
TODO
View file

@ -1 +1,2 @@
- Need to be able to check up front whether a requirement is satisfied, before attempting to install it (which is more expensive)
- Cache parsed Contents files during test suite runs and/or speed up reading

4
ognibuild/__init__.py
View file

@ -58,8 +58,8 @@ class UpstreamRequirement(object):
def __init__(self, family):
self.family = family
def possible_paths(self):
raise NotImplementedError
def met(self, session):
raise NotImplementedError(self)
class UpstreamOutput(object):

32
ognibuild/__main__.py
View file

@ -20,10 +20,6 @@ import os
import sys
from . import UnidentifiedError
from .buildsystem import NoBuildToolsFound, detect_buildsystems
from .build import run_build
from .clean import run_clean
from .dist import run_dist
from .install import run_install
from .resolver import (
ExplainResolver,
AutoResolver,
@ -31,7 +27,6 @@ from .resolver import (
MissingDependencies,
)
from .resolver.apt import AptResolver
from .test import run_test
def get_necessary_declared_requirements(resolver, requirements, stages):
@ -54,6 +49,7 @@ def install_necessary_declared_requirements(resolver, buildsystem, stages):
STAGE_MAP = {
"dist": [],
"info": [],
"install": ["build"],
"test": ["test", "dev"],
"build": ["build"],
@ -65,9 +61,6 @@ def main(): # noqa: C901
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"subcommand", type=str, choices=["dist", "build", "clean", "test", "install"]
)
parser.add_argument(
"--directory", "-d", type=str, help="Directory for project.", default="."
)
@ -87,6 +80,16 @@ def main(): # noqa: C901
"--verbose",
action="store_true",
help="Be verbose")
subparsers = parser.add_subparsers(dest='subcommand')
subparsers.add_parser('dist')
subparsers.add_parser('build')
subparsers.add_parser('clean')
subparsers.add_parser('test')
subparsers.add_parser('info')
install_parser = subparsers.add_parser('install')
install_parser.add_argument(
'--user', action='store_true', help='Install in local-user directories.')
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
@ -112,21 +115,32 @@ def main(): # noqa: C901
os.chdir(args.directory)
try:
bss = list(detect_buildsystems(args.directory))
logging.info('Detected buildsystems: %r', bss)
if not args.ignore_declared_dependencies:
stages = STAGE_MAP[args.subcommand]
if stages:
for bs in bss:
install_necessary_declared_requirements(resolver, bs, stages)
if args.subcommand == "dist":
from .dist import run_dist
run_dist(session=session, buildsystems=bss, resolver=resolver)
if args.subcommand == "build":
from .build import run_build
run_build(session, buildsystems=bss, resolver=resolver)
if args.subcommand == "clean":
from .clean import run_clean
run_clean(session, buildsystems=bss, resolver=resolver)
if args.subcommand == "install":
run_install(session, buildsystems=bss, resolver=resolver)
from .install import run_install
run_install(
session, buildsystems=bss, resolver=resolver,
user=args.user)
if args.subcommand == "test":
from .test import run_test
run_test(session, buildsystems=bss, resolver=resolver)
if args.subcommand == "info":
from .info import run_info
run_info(session, buildsystems=bss, resolver=resolver)
except UnidentifiedError:
return 1
except NoBuildToolsFound:

38
ognibuild/buildsystem.py
View file

@ -20,6 +20,7 @@
import logging
import os
import re
from typing import Optional
import warnings
from . import shebang_binary, UpstreamOutput, UnidentifiedError
@ -37,6 +38,14 @@ class NoBuildToolsFound(Exception):
"""No supported build tools were found."""
class InstallTarget(object):
# Whether to prefer user-specific installation
user: Optional[bool]
# TODO(jelmer): Add information about target directory, layout, etc.
class BuildSystem(object):
"""A particular buildsystem."""
@ -54,7 +63,7 @@ class BuildSystem(object):
def clean(self, session, resolver):
raise NotImplementedError(self.clean)
def install(self, session, resolver):
def install(self, session, resolver, install_target):
raise NotImplementedError(self.install)
def get_declared_dependencies(self):
@ -84,15 +93,15 @@ class Pear(BuildSystem):
def build(self, session, resolver):
self.setup(resolver)
run_with_build_fixer(session, ["pear", "build"])
run_with_build_fixer(session, ["pear", "build", self.path])
def clean(self, session, resolver):
self.setup(resolver)
# TODO
def install(self, session, resolver):
def install(self, session, resolver, install_target):
self.setup(resolver)
run_with_build_fixer(session, ["pear", "install"])
run_with_build_fixer(session, ["pear", "install", self.path])
class SetupPy(BuildSystem):
@ -104,6 +113,9 @@ class SetupPy(BuildSystem):
from distutils.core import run_setup
self.result = run_setup(os.path.abspath(path), stop_after="init")
def __repr__(self):
return "%s(%r)" % (type(self).__name__, self.path)
def setup(self, resolver):
resolver.install([PythonPackageRequirement('pip')])
with open(self.path, "r") as f:
@ -147,9 +159,12 @@ class SetupPy(BuildSystem):
self.setup(resolver)
self._run_setup(session, resolver, ["clean"])
def install(self, session, resolver):
def install(self, session, resolver, install_target):
self.setup(resolver)
self._run_setup(session, resolver, ["install"])
extra_args = []
if install_target.user:
extra_args.append('--user')
self._run_setup(session, resolver, ["install"] + extra_args)
def _run_setup(self, session, resolver, args):
interpreter = shebang_binary("setup.py")
@ -338,7 +353,12 @@ class Make(BuildSystem):
name = "make"
def __repr__(self):
return "%s()" % type(self).__name__
def setup(self, session, resolver):
resolver.install([BinaryRequirement("make")])
if session.exists("Makefile.PL") and not session.exists("Makefile"):
resolver.install([BinaryRequirement("perl")])
run_with_build_fixer(session, ["perl", "Makefile.PL"])
@ -375,12 +395,14 @@ class Make(BuildSystem):
def build(self, session, resolver):
self.setup(session, resolver)
resolver.install([BinaryRequirement("make")])
run_with_build_fixer(session, ["make", "all"])
def install(self, session, resolver, install_target):
self.setup(session, resolver)
run_with_build_fixer(session, ["make", "install"])
def dist(self, session, resolver):
self.setup(session, resolver)
resolver.install([BinaryRequirement("make")])
try:
run_with_build_fixer(session, ["make", "dist"])
except UnidentifiedError as e:

8
ognibuild/debian/fix_build.py
View file

@ -437,7 +437,7 @@ def fix_missing_python_module(error, context):
return True
def problem_to_upstream_requirement(problem, context):
def problem_to_upstream_requirement(problem):
if isinstance(problem, MissingFile):
return PathRequirement(problem.path)
elif isinstance(problem, MissingCommand):
@ -526,7 +526,11 @@ def problem_to_upstream_requirement(problem, context):
class UpstreamRequirementFixer(BuildFixer):
def fix_missing_requirement(self, error, context):
def can_fix(self, error):
req = problem_to_upstream_requirement(error)
return req is not None
def fix(self, error, context):
req = problem_to_upstream_requirement(error)
if req is None:
return False

6
ognibuild/fix_build.py
View file

@ -167,15 +167,15 @@ def run_with_build_fixer(
def resolve_error(error, context, fixers):
relevant_fixers = []
for error_cls, fixer in fixers:
if isinstance(error, error_cls):
for fixer in fixers:
if fixer.can_fix(error):
relevant_fixers.append(fixer)
if not relevant_fixers:
logging.warning("No fixer found for %r", error)
return False
for fixer in relevant_fixers:
logging.info("Attempting to use fixer %r to address %r", fixer, error)
made_changes = fixer(error, context)
made_changes = fixer.fix(error, context)
if made_changes:
return True
return False

45
ognibuild/info.py Normal file
View file

@ -0,0 +1,45 @@
#!/usr/bin/python3
# Copyright (C) 2020-2021 Jelmer Vernooij <jelmer@jelmer.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
from .buildsystem import NoBuildToolsFound, InstallTarget
def run_info(session, buildsystems, resolver):
for buildsystem in buildsystems:
print('%r:' % buildsystem)
deps = {}
try:
for kind, dep in buildsystem.get_declared_dependencies():
deps.setdefault(kind, []).append(dep)
except NotImplementedError:
print('\tUnable to detect declared dependencies for this type of build system')
if deps:
print('\tDeclared dependencies:')
for kind in deps:
print('\t\t%s:' % kind)
for dep in deps[kind]:
print('\t\t\t%s' % dep)
print('')
try:
outputs = list(buildsystem.get_declared_outputs())
except NotImplementedError:
print('\tUnable to detect declared outputs for this type of build system')
outputs = []
if outputs:
print('\tDeclared outputs:')
for output in outputs:
print('\t\t%s' % output)

9
ognibuild/install.py
View file

@ -15,16 +15,19 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
from .buildsystem import NoBuildToolsFound
from .buildsystem import NoBuildToolsFound, InstallTarget
def run_install(session, buildsystems, resolver):
def run_install(session, buildsystems, resolver, user: bool = False):
# Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache
session.create_home()
install_target = InstallTarget()
install_target.user = user
for buildsystem in buildsystems:
buildsystem.install(session, resolver)
buildsystem.install(session, resolver, install_target)
return
raise NoBuildToolsFound()

5
ognibuild/requirements.py
View file

@ -33,10 +33,13 @@ class PythonPackageRequirement(UpstreamRequirement):
self.minimum_version = minimum_version
def __repr__(self):
return "%s(%r, %r, %r)" % (
return "%s(%r, python_version=%r, minimum_version=%r)" % (
type(self).__name__, self.package, self.python_version,
self.minimum_version)
def __str__(self):
return "python package: %s" % self.package
class BinaryRequirement(UpstreamRequirement):

23
ognibuild/resolver/apt.py
View file

@ -57,7 +57,7 @@ class NoAptPackage(Exception):
"""No apt package."""
def get_package_for_python_package(apt_mgr, package, python_version):
def get_package_for_python_package(apt_mgr, package, python_version, minimum_version=None):
if python_version == "pypy":
return apt_mgr.get_package_for_paths(
["/usr/lib/pypy/dist-packages/%s-.*.egg-info/PKG-INFO" % package],
@ -375,18 +375,18 @@ def resolve_autoconf_macro_req(apt_mgr, req):
def resolve_python_module_req(apt_mgr, req):
if req.python_version == 2:
return get_package_for_python_module(apt_mgr, req.module, "cpython2")
return get_package_for_python_module(apt_mgr, req.module, "cpython2", req.minimum_version)
elif req.python_version in (None, 3):
return get_package_for_python_module(apt_mgr, req.module, "cpython3")
return get_package_for_python_module(apt_mgr, req.module, "cpython3", req.minimum_version)
else:
return None
def resolve_python_package_req(apt_mgr, req):
if req.python_version == 2:
return get_package_for_python_package(apt_mgr, req.package, "cpython2")
return get_package_for_python_package(apt_mgr, req.package, "cpython2", req.minimum_version)
elif req.python_version in (None, 3):
return get_package_for_python_package(apt_mgr, req.package, "cpython3")
return get_package_for_python_package(apt_mgr, req.package, "cpython3", req.minimum_version)
else:
return None
@ -421,6 +421,13 @@ APT_REQUIREMENT_RESOLVERS = [
]
class AptRequirement(object):
def __init__(self, package, minimum_version=None):
self.package = package
self.minimum_version = minimum_version
def resolve_requirement_apt(apt_mgr, req: UpstreamRequirement):
for rr_class, rr_fn in APT_REQUIREMENT_RESOLVERS:
if isinstance(req, rr_class):
@ -440,15 +447,11 @@ class AptResolver(Resolver):
def from_session(cls, session):
return cls(AptManager(session))
def met(self, requirement):
pps = list(requirement.possible_paths())
return any(self.apt.session.exists(p) for p in pps)
def install(self, requirements):
missing = []
for req in requirements:
try:
if not self.met(req):
if not req.met(self.apt.session):
missing.append(req)
except NotImplementedError:
missing.append(req)

5
ognibuild/tests/test_debian_fix_build.py
View file

@ -31,7 +31,7 @@ from buildlog_consultant.common import (
MissingValaPackage,
)
from ..debian import apt
from ..debian.apt import LocalAptManager
from ..debian.apt import AptManager
from ..debian.fix_build import (
resolve_error,
VERSIONED_PACKAGE_FIXERS,
@ -89,7 +89,8 @@ blah (0.1) UNRELEASED; urgency=medium
yield pkg
def resolve(self, error, context=("build",)):
apt = LocalAptManager()
from ..session.plain import PlainSession
apt = AptManager(PlainSession())
context = BuildDependencyContext(
self.tree,
apt,