diff --git a/ognibuild/__main__.py b/ognibuild/__main__.py index f9a5fc6..47e2f75 100644 --- a/ognibuild/__main__.py +++ b/ognibuild/__main__.py @@ -47,12 +47,12 @@ def get_necessary_declared_requirements(resolver, requirements, stages): return missing -def install_necessary_declared_requirements(session, resolver, buildsystems, stages, explain=False): +def install_necessary_declared_requirements(session, resolver, fixers, buildsystems, stages, explain=False): relevant = [] declared_reqs = [] for buildsystem in buildsystems: try: - declared_reqs.extend(buildsystem.get_declared_dependencies()) + declared_reqs.extend(buildsystem.get_declared_dependencies(session, fixers)) except NotImplementedError: logging.warning( "Unable to determine declared dependencies from %r", buildsystem @@ -169,17 +169,17 @@ def main(): # noqa: C901 bss = list(detect_buildsystems(args.directory)) logging.info( "Detected buildsystems: %s", ', '.join(map(str, bss))) + fixers = determine_fixers(session, resolver, explain=args.explain) if not args.ignore_declared_dependencies: stages = STAGE_MAP[args.subcommand] if stages: logging.info("Checking that declared requirements are present") try: install_necessary_declared_requirements( - session, resolver, bss, stages, explain=args.explain) + session, resolver, fixers, bss, stages, explain=args.explain) except ExplainInstall as e: display_explain_commands(e.commands) return 1 - fixers = determine_fixers(session, resolver, explain=args.explain) if args.subcommand == "dist": from .dist import run_dist diff --git a/ognibuild/buildsystem.py b/ognibuild/buildsystem.py index fe245a9..a3c7a74 100644 --- a/ognibuild/buildsystem.py +++ b/ognibuild/buildsystem.py @@ -73,10 +73,10 @@ class BuildSystem(object): def install(self, session, resolver, fixers, install_target): raise NotImplementedError(self.install) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): raise NotImplementedError(self.get_declared_dependencies) - def get_declared_outputs(self): + def get_declared_outputs(self, session, fixers=None): raise NotImplementedError(self.get_declared_outputs) @classmethod @@ -124,7 +124,6 @@ class Pear(BuildSystem): # run_setup, but setting __name__ # Imported from Python's distutils.core, Copyright (C) PSF - def run_setup(script_name, script_args=None, stop_after="run"): from distutils import core import sys @@ -154,36 +153,64 @@ def run_setup(script_name, script_args=None, stop_after="run"): # (ie. error)? pass - if core._setup_distribution is None: - raise RuntimeError( - ( - "'distutils.core.setup()' was never called -- " - "perhaps '%s' is not a Distutils setup script?" - ) - % script_name - ) - return core._setup_distribution +_setup_wrapper = """\ +from distutils import core +import sys + +script_name = %(script_name)s + +save_argv = sys.argv.copy() +g = {"__file__": script_name, "__name__": "__main__"} +try: + core._setup_stop_after = "init" + sys.argv[0] = script_name + with open(script_name, "rb") as f: + exec(f.read(), g) +except SystemExit: + # Hmm, should we do something if exiting with a non-zero code + # (ie. error)? + pass + +if core._setup_distribution is None: + raise RuntimeError( + ( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" + ) + % script_name + ) + +d = core._setup_distribution +r = { + 'setup_requires': getattr(d, "setup_requires", []), + 'install_requires': getattr(d, "install_requires", []), + 'tests_require': getattr(d, "tests_require", []), + 'scripts': getattr(d, "scripts", []) or [], + 'entry_points': getattr(d, "entry_points", None) or {}, + 'packages': getattr(d, "packages", []) or [], + 'requires': d.get_requires() or [], + } +import os +import json +with open(%(output_path)s, 'w') as f: + json.dump(r, f) +""" + + class SetupPy(BuildSystem): name = "setup.py" + DEFAULT_PYTHON = 'python3' def __init__(self, path): self.path = path if os.path.exists(os.path.join(self.path, 'setup.py')): self.has_setup_py = True - # TODO(jelmer): Perhaps run this in session, so we can install - # missing dependencies? - try: - self.distribution = run_setup(os.path.abspath(os.path.join(self.path, 'setup.py')), stop_after="init") - except RuntimeError as e: - logging.warning("Unable to load setup.py metadata: %s", e) - self.distribution = None else: self.has_setup_py = False - self.distribution = None try: self.pyproject = self.load_toml() @@ -196,28 +223,79 @@ class SetupPy(BuildSystem): with open(os.path.join(self.path, "pyproject.toml"), "r") as pf: return toml.load(pf) + def _extract_setup(self, session=None, fixers=None): + if session is None: + return self._extract_setup_direct() + else: + return self._extract_setup_in_session(session, fixers) + + def _extract_setup_direct(self): + p = os.path.join(self.path, 'setup.py') + try: + d = run_setup(os.path.abspath(p), stop_after="init") + except RuntimeError as e: + logging.warning("Unable to load setup.py metadata: %s", e) + return None + if d is None: + logging.warning( + "'distutils.core.setup()' was never called -- " + "perhaps '%s' is not a Distutils setup script?" % os.path.basename(p)) + return None + + return { + 'setup_requires': getattr(d, "setup_requires", []), + 'install_requires': getattr(d, "install_requires", []), + 'tests_require': getattr(d, "tests_require", []), + 'scripts': getattr(d, "scripts", []), + 'entry_points': getattr(d, "entry_points", None) or {}, + 'packages': getattr(d, "packages", []), + 'requires': d.get_requires() or [], + } + + def _extract_setup_in_session(self, session, fixers=None): + import tempfile + import json + interpreter = shebang_binary(os.path.join(self.path, "setup.py")) + if interpreter is None: + interpreter = self.DEFAULT_PYTHON + output_f = tempfile.NamedTemporaryFile( + dir=os.path.join(session.location, 'tmp'), mode='w+t') + with output_f: + # TODO(jelmer): Perhaps run this in session, so we can install + # missing dependencies? + argv = [interpreter, "-c", + _setup_wrapper + .replace('%(script_name)s', '"setup.py"') + .replace('%(output_path)s', + '"/' + os.path.relpath(output_f.name, session.location) + + '"')] + try: + if fixers is not None: + run_with_build_fixers(session, argv, fixers) + else: + session.check_call(argv, close_fds=False) + except RuntimeError as e: + logging.warning("Unable to load setup.py metadata: %s", e) + return None + output_f.seek(0) + return json.load(output_f) + def __repr__(self): return "%s(%r)" % (type(self).__name__, self.path) - def setup(self, resolver): - pass - def test(self, session, resolver, fixers): - self.setup(resolver) if self.has_setup_py: self._run_setup(session, resolver, ["test"], fixers) else: raise NotImplementedError def build(self, session, resolver, fixers): - self.setup(resolver) if self.has_setup_py: self._run_setup(session, resolver, ["build"], fixers) else: raise NotImplementedError def dist(self, session, resolver, fixers, quiet=False): - self.setup(resolver) if self.has_setup_py: preargs = [] if quiet: @@ -234,14 +312,12 @@ class SetupPy(BuildSystem): raise AssertionError("no supported section in pyproject.toml") def clean(self, session, resolver, fixers): - self.setup(resolver) if self.has_setup_py: self._run_setup(session, resolver, ["clean"], fixers) else: raise NotImplementedError def install(self, session, resolver, fixers, install_target): - self.setup(resolver) if self.has_setup_py: extra_args = [] if install_target.user: @@ -253,44 +329,50 @@ class SetupPy(BuildSystem): def _run_setup(self, session, resolver, args, fixers): interpreter = shebang_binary(os.path.join(self.path, 'setup.py')) if interpreter is not None: - resolver.install([BinaryRequirement(interpreter)]) run_with_build_fixers(session, ["./setup.py"] + args, fixers) else: # Just assume it's Python 3 - resolver.install([BinaryRequirement("python3")]) - run_with_build_fixers(session, ["python3", "./setup.py"] + args, fixers) + run_with_build_fixers(session, [self.DEFAULT_PYTHON, "./setup.py"] + args, fixers) - def get_declared_dependencies(self): - if self.distribution is None: + def get_declared_dependencies(self, session, fixers=None): + distribution = self._extract_setup(session, fixers) + if distribution is None: raise NotImplementedError - for require in self.distribution.get_requires(): + for require in distribution['requires']: yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.distribution, "setup_requires", []): - for require in self.distribution.setup_requires: - yield "build", PythonPackageRequirement.from_requirement_str(require) + for require in distribution['setup_requires']: + yield "build", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.distribution, "install_requires", []): - for require in self.distribution.install_requires: - yield "core", PythonPackageRequirement.from_requirement_str(require) + for require in distribution['install_requires']: + yield "core", PythonPackageRequirement.from_requirement_str(require) # Not present for distutils-only packages - if getattr(self.distribution, "tests_require", []): - for require in self.distribution.tests_require: - yield "test", PythonPackageRequirement.from_requirement_str(require) + for require in distribution['tests_require']: + yield "test", PythonPackageRequirement.from_requirement_str(require) if self.pyproject: if "build-system" in self.pyproject: for require in self.pyproject['build-system'].get("requires", []): yield "build", PythonPackageRequirement.from_requirement_str(require) - def get_declared_outputs(self): - if self.distribution is None: + def get_declared_outputs(self, session, fixers=None): + distribution = self._extract_setup(session, fixers) + if distribution is None: raise NotImplementedError - for script in self.distribution.scripts or []: + for script in distribution['scripts']: yield BinaryOutput(os.path.basename(script)) - entry_points = getattr(self.distribution, "entry_points", None) or {} - for script in entry_points.get("console_scripts", []): + for script in distribution["entry_points"].get("console_scripts", []): yield BinaryOutput(script.split("=")[0]) - for package in self.distribution.packages or []: + packages = set() + for package in sorted(distribution['packages']): + pts = package.split('.') + b = [] + for e in pts: + b.append(e) + if '.'.join(b) in packages: + break + else: + packages.add(package) + for package in packages: yield PythonPackageOutput(package, python_version="cpython3") @classmethod @@ -438,7 +520,7 @@ class Npm(BuildSystem): with open(path, "r") as f: self.package = json.load(f) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): if "devDependencies" in self.package: for name, unused_version in self.package["devDependencies"].items(): # TODO(jelmer): Look at version @@ -555,9 +637,8 @@ class DistInkt(BuildSystem): ): return cls(os.path.join(path, "dist.ini")) - def get_declared_dependencies(self): - import subprocess - out = subprocess.check_output(["dzil", "authordeps"]) + def get_declared_dependencies(self, session, fixers=None): + out = session.check_output(["dzil", "authordeps"]) for entry in out.splitlines(): yield "build", PerlModuleRequirement(entry.decode()) @@ -682,7 +763,7 @@ class Make(BuildSystem): else: raise - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): # TODO(jelmer): Split out the perl-specific stuff? if os.path.exists(os.path.join(self.path, "META.yml")): # See http://module-build.sourceforge.net/META-spec-v1.4.html for @@ -740,7 +821,7 @@ class Cargo(BuildSystem): with open(path, "r") as f: self.cargo = load(f) - def get_declared_dependencies(self): + def get_declared_dependencies(self, session, fixers=None): if "dependencies" in self.cargo: for name, details in self.cargo["dependencies"].items(): if isinstance(details, str): diff --git a/ognibuild/info.py b/ognibuild/info.py index 553b620..0ee1cfd 100644 --- a/ognibuild/info.py +++ b/ognibuild/info.py @@ -21,7 +21,7 @@ def run_info(session, buildsystems): print("%r:" % buildsystem) deps = {} try: - for kind, dep in buildsystem.get_declared_dependencies(): + for kind, dep in buildsystem.get_declared_dependencies(session): deps.setdefault(kind, []).append(dep) except NotImplementedError: print( @@ -35,7 +35,7 @@ def run_info(session, buildsystems): print("\t\t\t%s" % dep) print("") try: - outputs = list(buildsystem.get_declared_outputs()) + outputs = list(buildsystem.get_declared_outputs(session)) except NotImplementedError: print("\tUnable to detect declared outputs for this type of build system") outputs = [] diff --git a/ognibuild/session/__init__.py b/ognibuild/session/__init__.py index aa93f29..1ef59d5 100644 --- a/ognibuild/session/__init__.py +++ b/ognibuild/session/__init__.py @@ -41,6 +41,7 @@ class Session(object): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, + close_fds: bool = True ): raise NotImplementedError(self.check_call) diff --git a/ognibuild/session/plain.py b/ognibuild/session/plain.py index 58cdfb7..924577a 100644 --- a/ognibuild/session/plain.py +++ b/ognibuild/session/plain.py @@ -56,9 +56,10 @@ class PlainSession(Session): self, argv: List[str], cwd: Optional[str] = None, user: Optional[str] = None, - env: Optional[Dict[str, str]] = None): + env: Optional[Dict[str, str]] = None, + close_fds: bool = True): argv = self._prepend_user(user, argv) - return subprocess.check_call(argv, cwd=cwd, env=env) + return subprocess.check_call(argv, cwd=cwd, env=env, close_fds=close_fds) def check_output( self, argv: List[str], diff --git a/ognibuild/session/schroot.py b/ognibuild/session/schroot.py index 6e8c236..760c8e6 100644 --- a/ognibuild/session/schroot.py +++ b/ognibuild/session/schroot.py @@ -114,9 +114,10 @@ class SchrootSession(Session): cwd: Optional[str] = None, user: Optional[str] = None, env: Optional[Dict[str, str]] = None, + close_fds: bool = True ): try: - subprocess.check_call(self._run_argv(argv, cwd, user, env=env)) + subprocess.check_call(self._run_argv(argv, cwd, user, env=env), close_fds=close_fds) except subprocess.CalledProcessError as e: raise subprocess.CalledProcessError(e.returncode, argv)