billchenchina/ognibuild
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More work on resolvers.

Browse source
This commit is contained in:
Jelmer Vernooij 2021-02-08 17:47:39 +00:00
parent 2aab09121d
commit d89426738f
No known key found for this signature in database
GPG key ID: 579C160D4C9E23E8
9 changed files with 267 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

30
notes/structure.md Normal file
View file

@ -0,0 +1,30 @@
Upstream requirements are expressed as objects derived from UpstreamRequirement.
They can either be:
* extracted from the build system
* extracted from errors in build logs
The details of UpstreamRequirements are specific to the kind of requirement,
and otherwise opaque to ognibuild.
When building a package, we first make sure that all declared upstream
requirements are met.
Then we attempt to build.
If any problems are found in the log, buildlog-consultant will report them.
ognibuild can then invoke "fixers" to address Problems.
Problems can be converted to UpstreamRequirements by UpstreamRequirementFixer
Other Fixer can do things like e.g. upgrade configure.ac to a newer version.
UpstreamRequirementFixer uses a UpstreamRequirementResolver object that
can translate UpstreamRequirement objects into apt package names or
e.g. cpan commands.
ognibuild keeps finding problems, resolving them and rebuilding until it finds
a problem it can not resolve or that it thinks it has already resolved
(i.e. seen before).

36
ognibuild/__main__.py
View file

@ -38,10 +38,9 @@ def main():
)
parser.add_argument("--schroot", type=str, help="schroot to run in.")
parser.add_argument(
"--resolve",
choices=["explain", "apt", "native"],
help="What to do about missing dependencies",
)
'--resolve', choices=['explain', 'apt', 'native'],
default='apt',
help='What to do about missing dependencies')
args = parser.parse_args()
if args.schroot:
from .session.schroot import SchrootSession
@ -52,18 +51,27 @@ def main():
session = PlainSession()
with session:
if args.resolve == 'apt':
from .resolver import AptResolver
resolver = AptResolver.from_session(session)
elif args.resolve == 'explain':
from .resolver import ExplainResolver
resolver = ExplainResolver.from_session(session)
elif args.resolve == 'native':
from .resolver import NativeResolver
resolver = NativeResolver.from_session(session)
os.chdir(args.directory)
try:
if args.subcommand == "dist":
run_dist(session)
if args.subcommand == "build":
run_build(session)
if args.subcommand == "clean":
run_clean(session)
if args.subcommand == "install":
run_install(session)
if args.subcommand == "test":
run_test(session)
if args.subcommand == 'dist':
run_dist(session=session, resolver=resolver)
if args.subcommand == 'build':
run_build(session, resolver=resolver)
if args.subcommand == 'clean':
run_clean(session, resolver=resolver)
if args.subcommand == 'install':
run_install(session, resolver=resolver)
if args.subcommand == 'test':
run_test(session, resolver=resolver)
except NoBuildToolsFound:
logging.info("No build tools found.")
return 1

4
ognibuild/build.py
View file

@ -18,13 +18,13 @@
from .buildsystem import detect_buildsystems, NoBuildToolsFound
def run_build(session):
def run_build(session, resolver):
# Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache
session.create_home()
for buildsystem in detect_buildsystems(session):
buildsystem.build()
buildsystem.build(resolver)
return
raise NoBuildToolsFound()

243
ognibuild/buildsystem.py
View file

@ -20,8 +20,8 @@
import logging
import re
from . import shebang_binary
from .apt import AptManager, UnidentifiedError
from . import shebang_binary, UpstreamPackage
from .apt import UnidentifiedError
from .fix_build import run_with_build_fixer
@ -35,105 +35,106 @@ class BuildSystem(object):
def __init__(self, session):
self.session = session
def dist(self):
def dist(self, resolver):
raise NotImplementedError(self.dist)
def test(self):
def test(self, resolver):
raise NotImplementedError(self.test)
def build(self):
def build(self, resolver):
raise NotImplementedError(self.build)
def clean(self):
def clean(self, resolver):
raise NotImplementedError(self.clean)
def install(self):
def install(self, resolver):
raise NotImplementedError(self.install)
class Pear(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["php-pear"])
def dist(self):
self.setup()
run_with_build_fixer(self.session, ["pear", "package"])
def setup(self, resolver):
resolver.install([UpstreamPackage('binary', 'pear')])
def test(self):
def dist(self, resolver):
self.setup(resolver)
run_with_build_fixer(self.session, ['pear', 'package'])
def test(self, resolver):
self.setup()
run_with_build_fixer(self.session, ["pear", "run-tests"])
def build(self):
self.setup()
run_with_build_fixer(self.session, ["pear", "build"])
def build(self, resolver):
self.setup(resolver)
run_with_build_fixer(self.session, ['pear', 'build'])
def clean(self):
self.setup()
def clean(self, resolver):
self.setup(resolver)
# TODO
def install(self):
self.setup()
run_with_build_fixer(self.session, ["pear", "install"])
def install(self, resolver):
self.setup(resolver)
run_with_build_fixer(self.session, ['pear', 'install'])
class SetupPy(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["python3", "python3-pip"])
with open("setup.py", "r") as f:
def setup(self, resolver):
resolver.install([
UpstreamPackage('python3', 'pip'),
UpstreamPackage('binary', 'python3'),
])
with open('setup.py', 'r') as f:
setup_py_contents = f.read()
try:
with open("setup.cfg", "r") as f:
setup_cfg_contents = f.read()
except FileNotFoundError:
setup_cfg_contents = ""
if "setuptools" in setup_py_contents:
logging.info("Reference to setuptools found, installing.")
apt.install(["python3-setuptools"])
if (
"setuptools_scm" in setup_py_contents
or "setuptools_scm" in setup_cfg_contents
):
logging.info("Reference to setuptools-scm found, installing.")
apt.install(["python3-setuptools-scm", "git", "mercurial"])
setup_cfg_contents = ''
if 'setuptools' in setup_py_contents:
logging.info('Reference to setuptools found, installing.')
resolver.install([UpstreamPackage('python3', 'setuptools')])
if ('setuptools_scm' in setup_py_contents or
'setuptools_scm' in setup_cfg_contents):
logging.info('Reference to setuptools-scm found, installing.')
resolver.install([
UpstreamPackage('python3', 'setuptools-scm'),
UpstreamPackage('binary', 'git'),
UpstreamPackage('binary', 'mercurial'),
])
# TODO(jelmer): Install setup_requires
def test(self):
self.setup()
self._run_setup(["test"])
def test(self, resolver):
self.setup(resolver)
self._run_setup(resolver, ['test'])
def dist(self):
self.setup()
self._run_setup(["sdist"])
def dist(self, resolver):
self.setup(resolver)
self._run_setup(resolver, ['sdist'])
def clean(self):
self.setup()
self._run_setup(["clean"])
def clean(self, resolver):
self.setup(resolver)
self._run_setup(resolver, ['clean'])
def install(self):
self.setup()
self._run_setup(["install"])
def install(self, resolver):
self.setup(resolver)
self._run_setup(resolver, ['install'])
def _run_setup(self, args):
apt = AptManager(self.session)
interpreter = shebang_binary("setup.py")
def _run_setup(self, resolver, args):
interpreter = shebang_binary('setup.py')
if interpreter is not None:
if interpreter == "python3":
apt.install(["python3"])
elif interpreter == "python2":
apt.install(["python2"])
elif interpreter == "python":
apt.install(["python"])
if interpreter in ('python3', 'python2', 'python'):
resolver.install([UpstreamPackage('binary', interpreter)])
else:
raise ValueError("Unknown interpreter %r" % interpreter)
apt.install(["python2", "python3"])
run_with_build_fixer(self.session, ["./setup.py"] + args)
raise ValueError('Unknown interpreter %r' % interpreter)
run_with_build_fixer(
self.session, ['./setup.py'] + args)
else:
# Just assume it's Python 3
apt.install(["python3"])
run_with_build_fixer(self.session, ["python3", "./setup.py"] + args)
resolver.install([UpstreamPackage('binary', 'python3')])
run_with_build_fixer(
self.session, ['python3', './setup.py'] + args)
class PyProject(BuildSystem):
@ -143,75 +144,79 @@ class PyProject(BuildSystem):
with open("pyproject.toml", "r") as pf:
return toml.load(pf)
def dist(self):
apt = AptManager(self.session)
def dist(self, resolver):
pyproject = self.load_toml()
if "poetry" in pyproject.get("tool", []):
logging.info(
"Found pyproject.toml with poetry section, " "assuming poetry project."
)
apt.install(["python3-venv", "python3-pip"])
self.session.check_call(["pip3", "install", "poetry"], user="root")
self.session.check_call(["poetry", "build", "-f", "sdist"])
'Found pyproject.toml with poetry section, '
'assuming poetry project.')
resolver.install([
UpstreamPackage('python3', 'venv'),
UpstreamPackage('python3', 'pip'),
])
self.session.check_call(['pip3', 'install', 'poetry'], user='root')
self.session.check_call(['poetry', 'build', '-f', 'sdist'])
return
raise AssertionError("no supported section in pyproject.toml")
class SetupCfg(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["python3-pep517", "python3-pip"])
def dist(self):
self.session.check_call(["python3", "-m", "pep517.build", "-s", "."])
def setup(self, resolver):
resolver.install([
UpstreamPackage('python3', 'pep517'),
UpstreamPackage('python3', 'pip'),
])
def dist(self, resolver):
self.setup(resolver)
self.session.check_call(['python3', '-m', 'pep517.build', '-s', '.'])
class NpmPackage(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["npm"])
def dist(self):
self.setup()
run_with_build_fixer(self.session, ["npm", "pack"])
def setup(self, resolver):
resolver.install([UpstreamPackage('binary', 'npm')])
def dist(self, resolver):
self.setup(resolver)
run_with_build_fixer(self.session, ['npm', 'pack'])
class Waf(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["python3"])
def dist(self):
self.setup()
run_with_build_fixer(self.session, ["./waf", "dist"])
def setup(self, resolver):
resolver.install([UpstreamPackage('binary', 'python3')])
def dist(self, resolver):
self.setup(resolver)
run_with_build_fixer(self.session, ['./waf', 'dist'])
class Gem(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["gem2deb"])
def dist(self):
self.setup()
gemfiles = [
entry.name
for entry in self.session.scandir(".")
if entry.name.endswith(".gem")
]
def setup(self, resolver):
resolver.install([UpstreamPackage('binary', 'gem2deb')])
def dist(self, resolver):
self.setup(resolver)
gemfiles = [entry.name for entry in self.session.scandir('.')
if entry.name.endswith('.gem')]
if len(gemfiles) > 1:
logging.warning("More than one gemfile. Trying the first?")
run_with_build_fixer(self.session, ["gem2tgz", gemfiles[0]])
class DistInkt(BuildSystem):
def setup(self):
apt = AptManager(self.session)
apt.install(["libdist-inkt-perl"])
def dist(self):
self.setup()
apt = AptManager(self.session)
with open("dist.ini", "rb") as f:
def setup(self, resolver):
resolver.install([
UpstreamPackage('perl', 'Dist::Inkt'),
])
def dist(self, resolver):
self.setup(resolver)
with open('dist.ini', 'rb') as f:
for line in f:
if not line.startswith(b";;"):
continue
@ -230,22 +235,23 @@ class DistInkt(BuildSystem):
run_with_build_fixer(self.session, ["distinkt-dist"])
return
# Default to invoking Dist::Zilla
logging.info("Found dist.ini, assuming dist-zilla.")
apt.install(["libdist-zilla-perl"])
run_with_build_fixer(self.session, ["dzil", "build", "--in", ".."])
logging.info('Found dist.ini, assuming dist-zilla.')
resolver.install([UpstreamPackage('perl', 'Dist::Zilla')])
run_with_build_fixer(self.session, ['dzil', 'build', '--in', '..'])
class Make(BuildSystem):
def setup(self):
apt = AptManager(self.session)
if self.session.exists("Makefile.PL") and not self.session.exists("Makefile"):
apt.install(["perl"])
run_with_build_fixer(self.session, ["perl", "Makefile.PL"])
if not self.session.exists("Makefile") and not self.session.exists("configure"):
if self.session.exists("autogen.sh"):
if shebang_binary("autogen.sh") is None:
run_with_build_fixer(self.session, ["/bin/sh", "./autogen.sh"])
def setup(self, resolver):
if self.session.exists('Makefile.PL') and not self.session.exists('Makefile'):
resolver.install([UpstreamPackage('binary', 'perl')])
run_with_build_fixer(self.session, ['perl', 'Makefile.PL'])
if not self.session.exists('Makefile') and not self.session.exists('configure'):
if self.session.exists('autogen.sh'):
if shebang_binary('autogen.sh') is None:
run_with_build_fixer(
self.session, ['/bin/sh', './autogen.sh'])
try:
run_with_build_fixer(self.session, ["./autogen.sh"])
except UnidentifiedError as e:
@ -269,10 +275,9 @@ class Make(BuildSystem):
if not self.session.exists("Makefile") and self.session.exists("configure"):
self.session.check_call(["./configure"])
def dist(self):
self.setup()
apt = AptManager(self.session)
apt.install(["make"])
def dist(self, resolver):
self.setup(resolver)
resolver.install([UpstreamPackage('binary', 'make')])
try:
run_with_build_fixer(self.session, ["make", "dist"])
except UnidentifiedError as e:

4
ognibuild/clean.py
View file

@ -18,13 +18,13 @@
from .buildsystem import detect_buildsystems, NoBuildToolsFound
def run_clean(session):
def run_clean(session, resolver):
# Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache
session.create_home()
for buildsystem in detect_buildsystems(session):
buildsystem.clean()
buildsystem.clean(resolver)
return
raise NoBuildToolsFound()

4
ognibuild/dist.py
View file

@ -62,13 +62,13 @@ class DistNoTarball(Exception):
"""Dist operation did not create a tarball."""
def run_dist(session):
def run_dist(session, resolver):
# Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache
session.create_home()
for buildsystem in detect_buildsystems(session):
buildsystem.dist()
buildsystem.dist(resolver)
return
raise NoBuildToolsFound()

4
ognibuild/install.py
View file

@ -18,13 +18,13 @@
from .buildsystem import detect_buildsystems, NoBuildToolsFound
def run_install(session):
def run_install(session, resolver):
# Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache
session.create_home()
for buildsystem in detect_buildsystems(session):
buildsystem.install()
buildsystem.install(resolver)
return
raise NoBuildToolsFound()

81
ognibuild/resolver.py Normal file
View file

@ -0,0 +1,81 @@
#!/usr/bin/python3
# Copyright (C) 2020 Jelmer Vernooij <jelmer@jelmer.uk>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
class Resolver(object):
def install(self, requirements):
raise NotImplementedError(self.install)
def explain(self, requirements):
raise NotImplementedError(self.explain)
class AptResolver(Resolver):
def __init__(self, apt):
self.apt = apt
@classmethod
def from_session(cls, session):
from .apt import AptManager
return cls(AptManager(session))
def install(self, requirements):
self.apt.install(list(self.resolve(requirements)))
def explain(self, requirements):
raise NotImplementedError(self.explain)
def resolve(self, requirements):
for req in requirements:
if req.family == 'python3':
yield 'python3-%s' % req.name
else:
yield self.apt.find_file('/usr/bin/%s' % req.name)
class NativeResolver(Resolver):
def __init__(self, session):
self.session = session
@classmethod
def from_session(cls, session):
return cls(session)
def install(self, requirements):
raise NotImplementedError(self.install)
def explain(self, requirements):
raise NotImplementedError(self.explain)
class ExplainResolver(Resolver):
def __init__(self, session):
self.session = session
@classmethod
def from_session(cls, session):
return cls(session)
def install(self, requirements):
raise NotImplementedError(self.install)
def explain(self, requirements):
raise NotImplementedError(self.explain)

4
ognibuild/test.py
View file

@ -18,13 +18,13 @@
from .buildsystem import detect_buildsystems, NoBuildToolsFound
def run_test(session):
def run_test(session, resolver):
# Some things want to write to the user's home directory,
# e.g. pip caches in ~/.cache
session.create_home()
for buildsystem in detect_buildsystems(session):
buildsystem.test()
buildsystem.test(resolver)
return
raise NoBuildToolsFound()