Capture artifact listing output in-memory instead of via temp files

buildrunner/docker/runner.py

@@ -589,13 +589,11 @@ def save_caches(
                         f"It will not be saved again to `{local_cache_archive_file}`"
                     )
 
-    def run(self, cmd, console=None, stream=True, log=None, workdir=None):
+    def _build_exec_cmdv(self, cmd):
         """
-        Run the given command in the container.
+        Validate the container/shell are ready and build the argv to exec,
+        wrapping string commands in the container's shell.
         """
-        # Unused variable
-        _ = workdir
-
         if isinstance(cmd, str):
             cmdv = [self.shell, "-xc", cmd]
         elif (
@@ -607,14 +605,24 @@ def run(self, cmd, console=None, stream=True, log=None, workdir=None):
             cmdv = cmd
         else:
             raise TypeError(f"Unhandled command type: {type(cmd)}:{cmd}")
-        # if console is None:
-        #    raise Exception('No console!')
         if not self.container:
             raise BuildRunnerContainerError("Container has not been started")
         if not self.shell:
             raise BuildRunnerContainerError(
                 "Cannot call run if container cmd not shell"
             )
+        return cmdv
+
+    def run(self, cmd, console=None, stream=True, log=None, workdir=None):
+        """
+        Run the given command in the container.
+        """
+        # Unused variable
+        _ = workdir
+
+        cmdv = self._build_exec_cmdv(cmd)
+        # if console is None:
+        #    raise Exception('No console!')
 
         self._run_log(log, f"Executing: {cmdv}")
 
@@ -672,6 +680,40 @@ def run(self, cmd, console=None, stream=True, log=None, workdir=None):
             return inspect_res["ExitCode"]
         raise BuildRunnerContainerError("Error running cmd: no exit code")
 
+    def run_and_capture(self, cmd, log=None):
+        """
+        Run the given command in the container and return its exit code and
+        captured stdout as a string, instead of streaming output to a
+        console/log or requiring the caller to write output to a file.
+        """
+        cmdv = self._build_exec_cmdv(cmd)
+
+        self._run_log(log, f"Executing: {cmdv}")
+
+        create_res = self.docker_client.exec_create(
+            self.container["Id"],
+            cmdv,
+            tty=False,
+        )
+        # demux separates stdout from the `-x` trace output (which goes to
+        # stderr), so stdout only contains the command's actual output.
+        stdout, stderr = self.docker_client.exec_start(
+            create_res,
+            stream=False,
+            demux=True,
+        )
+        if stderr:
+            self._run_log(log, stderr)
+
+        inspect_res = self.docker_client.exec_inspect(create_res)
+        if "ExitCode" in inspect_res:
+            if inspect_res["ExitCode"] is None:
+                raise BuildRunnerContainerError(
+                    f"Error running cmd ({cmd}): exit code is None"
+                )
+            return inspect_res["ExitCode"], stdout.decode("utf-8") if stdout else ""
+        raise BuildRunnerContainerError("Error running cmd: no exit code")
+
     def run_script(
         self,
         script,

buildrunner/steprunner/tasks/run.py

@@ -192,24 +192,13 @@ def _retrieve_artifacts(self, console=None):  # pylint: disable=too-many-locals
             for pattern, properties in self.step.artifacts.items():
                 # query files for each artifacts pattern, capturing the output
                 # for parsing
-                stat_output_file = f"{str(uuid.uuid4())}.out"
-                stat_output_file_local = os.path.join(
-                    self.step_runner.results_dir,
-                    stat_output_file,
-                )
-                exit_code = artifact_lister.run(
-                    f'stat -c "%n{FILE_INFO_DELIMITER}%F" {pattern} >/stepresults/{stat_output_file}',
-                    console=console,
-                    stream=True,
+                exit_code, output = artifact_lister.run_and_capture(
+                    f'stat -c "%n{FILE_INFO_DELIMITER}%F" {pattern}',
                     log=self.step_runner.log,
                 )
 
                 # if the command was successful we found something
                 if exit_code == 0:
-                    with open(
-                        stat_output_file_local, "r", encoding="utf-8"
-                    ) as output_fd:
-                        output = output_fd.read()
                     artifact_files = [af.strip() for af in output.split("\n")]
                     for art_info in artifact_files:
                         if not art_info or FILE_INFO_DELIMITER not in art_info:
@@ -263,10 +252,6 @@ def _retrieve_artifacts(self, console=None):  # pylint: disable=too-many-locals
                                 properties,
                             )
 
-                # remove the stat output file
-                if os.path.exists(stat_output_file_local):
-                    os.remove(stat_output_file_local)
-
         finally:
             if artifact_lister:
                 # make sure the current user/group ids of our
@@ -297,19 +282,11 @@ def _archive_dir(
         if properties and properties.get("format", None) == "uncompressed":
             # recursively find all files in dir and add
             # each one, passing any properties
-            find_output_file = f"{str(uuid.uuid4())}.out"
-            find_output_file_local = os.path.join(
-                self.step_runner.results_dir,
-                find_output_file,
-            )
-            find_exit_code = artifact_lister.run(
-                f"find {artifact_file} -type f >/stepresults/{find_output_file}",
-                stream=False,
+            find_exit_code, output = artifact_lister.run_and_capture(
+                f"find {artifact_file} -type f",
                 log=self.step_runner.log,
             )
             if find_exit_code == 0:
-                with open(find_output_file_local, "r", encoding="utf-8") as output_fd:
-                    output = output_fd.read()
                 for _file in [_f.strip() for _f in output.split("\n")]:
                     if not _file:
                         continue
@@ -348,10 +325,6 @@ def _archive_dir(
                         properties,
                     )
 
-                # remove the find output file
-                if os.path.exists(find_output_file_local):
-                    os.remove(find_output_file_local)
-
         else:
             filename = os.path.basename(artifact_file)
 

examples/artifacts/buildrunner.yaml

@@ -0,0 +1,26 @@
+# Description: This example demonstrates how to archive artifacts from a step
+# using the artifacts key, including a plain file and an uncompressed
+# directory, and how to access artifacts from a previous step in a
+# subsequent step using the depends keyword.
+version: 2.0
+steps:
+  make-artifacts:
+    run:
+      image: alpine:latest
+      cmds:
+        - echo "Hello from make-artifacts" > hello.txt
+        - mkdir -p outdir
+        - echo "Hello from outdir" > outdir/nested.txt
+      artifacts:
+        hello.txt:
+          my-property: some-value
+        outdir:
+          format: uncompressed
+  use-artifacts:
+    depends:
+      - make-artifacts
+    run:
+      image: alpine:latest
+      cmds:
+        - cat /artifacts/make-artifacts/hello.txt
+        - cat /artifacts/make-artifacts/outdir/nested.txt

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "buildrunner"
-version = "3.26"
+version = "3.27"
 description = "Docker-based build tool"
 readme = "README.rst"
 requires-python = ">=3.9"

tests/test_docker_runner_run_and_capture.py

@@ -0,0 +1,149 @@
+"""
+Copyright 2026 Adobe
+All Rights Reserved.
+
+NOTICE: Adobe permits you to use, modify, and distribute this file in accordance
+with the terms of the Adobe license agreement accompanying it.
+"""
+
+from unittest import mock
+
+import pytest
+
+from buildrunner.docker import BuildRunnerContainerError
+from buildrunner.docker.runner import DockerRunner
+
+
+@pytest.fixture(name="runner")
+def fixture_runner():
+    """
+    A DockerRunner with __init__ skipped, since it otherwise requires a live
+    docker daemon connection. Only the attributes used by run_and_capture()
+    are populated.
+    """
+    runner = object.__new__(DockerRunner)
+    runner.container = {"Id": "container123"}
+    runner.shell = "/bin/sh"
+    runner.run_log_debug = False
+    runner.docker_client = mock.MagicMock()
+    runner.docker_client.exec_create.return_value = {"Id": "exec123"}
+    return runner
+
+
+def test_run_and_capture_returns_exit_code_and_decoded_stdout(runner):
+    """
+    Regression test for XENG-10989: artifact-listing commands must be able
+    to get their output back directly from the exec call, without writing
+    to a file that a container-root process creates and a host process must
+    later remove.
+    """
+    runner.docker_client.exec_start.return_value = (b"line1\nline2\n", b"")
+    runner.docker_client.exec_inspect.return_value = {"ExitCode": 0}
+
+    exit_code, output = runner.run_and_capture("stat -c '%n' /some/path")
+
+    assert exit_code == 0
+    assert output == "line1\nline2\n"
+
+    # must not stream (would return a generator, not the full buffer) and
+    # must demux so stdout doesn't contain the `-x` shell trace
+    _, kwargs = runner.docker_client.exec_start.call_args
+    assert kwargs["stream"] is False
+    assert kwargs["demux"] is True
+
+
+def test_run_and_capture_no_stdout_returns_empty_string(runner):
+    runner.docker_client.exec_start.return_value = (None, None)
+    runner.docker_client.exec_inspect.return_value = {"ExitCode": 1}
+
+    exit_code, output = runner.run_and_capture("false")
+
+    assert exit_code == 1
+    assert output == ""
+
+
+def test_run_and_capture_passes_iterable_command_through(runner):
+    """
+    A non-string (list/tuple) command must be handed to exec verbatim rather
+    than being wrapped in the shell. run() relies on this for its cp/tar/chown
+    commands, and both now share _build_exec_cmdv.
+    """
+    runner.docker_client.exec_start.return_value = (b"", b"")
+    runner.docker_client.exec_inspect.return_value = {"ExitCode": 0}
+
+    runner.run_and_capture(["find", "/some/dir", "-type", "f"])
+
+    exec_args, _ = runner.docker_client.exec_create.call_args
+    # second positional arg to exec_create is the command vector, passed
+    # through unwrapped (not [shell, "-xc", ...])
+    assert exec_args[1] == ["find", "/some/dir", "-type", "f"]
+
+
+def test_run_and_capture_raises_when_exit_code_is_none(runner):
+    runner.docker_client.exec_start.return_value = (b"", b"")
+    runner.docker_client.exec_inspect.return_value = {"ExitCode": None}
+
+    with pytest.raises(BuildRunnerContainerError):
+        runner.run_and_capture("stat /some/path")
+
+
+def test_run_and_capture_logs_stderr(runner):
+    log = mock.MagicMock()
+    log.clean_output.side_effect = lambda output: (
+        output if isinstance(output, str) else output.decode("utf-8", errors="replace")
+    )
+    runner.docker_client.exec_start.return_value = (b"out", b"+ stat ...\n")
+    runner.docker_client.exec_inspect.return_value = {"ExitCode": 0}
+
+    runner.run_and_capture("stat -c '%n' /some/path", log=log)
+
+    assert log.info.called
+
+
+def test_run_and_capture_requires_started_container():
+    runner = object.__new__(DockerRunner)
+    runner.container = None
+    runner.shell = "/bin/sh"
+
+    with pytest.raises(BuildRunnerContainerError):
+        runner.run_and_capture("stat /some/path")
+
+
+def test_run_and_capture_raises_on_missing_exit_code(runner):
+    runner.docker_client.exec_start.return_value = (b"", b"")
+    runner.docker_client.exec_inspect.return_value = {}
+
+    with pytest.raises(BuildRunnerContainerError):
+        runner.run_and_capture("stat /some/path")
+
+
+def test_run_still_delegates_to_build_exec_cmdv_and_returns_exit_code(runner):
+    """
+    run() was refactored to share command building with run_and_capture via
+    _build_exec_cmdv; verify it still wraps the command and returns the exit
+    code unchanged.
+    """
+    runner.docker_client.exec_start.return_value = b"some output\n"
+    runner.docker_client.exec_inspect.return_value = {"ExitCode": 7}
+
+    exit_code = runner.run("echo hi")
+
+    assert exit_code == 7
+    exec_args, _ = runner.docker_client.exec_create.call_args
+    assert exec_args[1] == ["/bin/sh", "-xc", "echo hi"]
+
+
+def test_build_exec_cmdv_wraps_string_in_shell(runner):
+    assert runner._build_exec_cmdv("stat /x") == ["/bin/sh", "-xc", "stat /x"]
+
+
+def test_build_exec_cmdv_raises_type_error_on_unknown_command(runner):
+    with pytest.raises(TypeError):
+        runner._build_exec_cmdv(12345)
+
+
+def test_build_exec_cmdv_raises_when_shell_not_set(runner):
+    runner.shell = None
+
+    with pytest.raises(BuildRunnerContainerError):
+        runner._build_exec_cmdv(["find", "/x"])