kiln_ai.sandbox.test_sandbox_shared

Tests for sandbox shared helpers — spawn lock identity and call_entrypoint.

 1"""Tests for sandbox shared helpers — spawn lock identity and call_entrypoint."""
 2
 3import asyncio
 4
 5import pytest
 6
 7from kiln_ai.adapters.eval.sandbox_worker import execute_scorer_bridged
 8from kiln_ai.datamodel.project import Project
 9from kiln_ai.sandbox.entrypoint import call_entrypoint
10from kiln_ai.sandbox.spawn import _spawn_lock, start_process_with_light_main
11from kiln_ai.tools.sandbox_bridge import NestedToolServer, run_bridged_child
12
13
14class TestSpawnLockIdentity:
15    def test_spawn_lock_shared_with_eval(self):
16        """Code evals and code tools share the same _spawn_lock (PyInstaller #7410)."""
17        from kiln_ai.sandbox import spawn as spawn_mod
18
19        assert spawn_mod._spawn_lock is _spawn_lock
20
21    def test_bridge_delegates_to_shared_spawn_helper(self):
22        """The shared bridge spawns via start_process_with_light_main from sandbox.spawn."""
23        from kiln_ai.tools import sandbox_bridge
24
25        assert (
26            sandbox_bridge.start_process_with_light_main
27            is start_process_with_light_main
28        )
29
30    @pytest.mark.asyncio
31    async def test_scorer_runs_through_bridge(self):
32        """Regression: a scorer executes through the shared bridge and returns its dict."""
33        code = (
34            "def score(output, trace, reference_data, task_input):\n"
35            "    return {'ok': 1.0}\n"
36        )
37        server = NestedToolServer(
38            allowlist=[], project=Project(name="shared_test"), task=None, context=None
39        )
40        res = await run_bridged_child(
41            target=execute_scorer_bridged,
42            args=(
43                code,
44                {
45                    "output": "x",
46                    "trace": None,
47                    "reference_data": None,
48                    "task_input": "y",
49                },
50            ),
51            timeout_s=10,
52            server=server,
53        )
54        assert res.result_msg is not None
55        assert res.result_msg["ok"] == {"ok": 1.0}
56
57
58class TestCallEntrypoint:
59    def test_sync_function(self):
60        def fn(x):
61            return x * 2
62
63        assert call_entrypoint(fn, {"x": 5}) == 10
64
65    def test_async_function(self):
66        async def fn(x):
67            return x + 1
68
69        assert call_entrypoint(fn, {"x": 5}) == 6
70
71    def test_async_with_gather(self):
72        async def fn(values):
73            async def double(v):
74                return v * 2
75
76            return await asyncio.gather(*(double(v) for v in values))
77
78        result = call_entrypoint(fn, {"values": [1, 2, 3]})
79        assert result == [2, 4, 6]
80
81    def test_sync_returning_non_coroutine(self):
82        def fn():
83            return "plain"
84
85        assert call_entrypoint(fn, {}) == "plain"
86
87    def test_propagates_exception(self):
88        def fn():
89            raise ValueError("boom")
90
91        with pytest.raises(ValueError, match="boom"):
92            call_entrypoint(fn, {})
93
94    def test_async_propagates_exception(self):
95        async def fn():
96            raise RuntimeError("async boom")
97
98        with pytest.raises(RuntimeError, match="async boom"):
99            call_entrypoint(fn, {})
class TestSpawnLockIdentity: 
15class TestSpawnLockIdentity:
16    def test_spawn_lock_shared_with_eval(self):
17        """Code evals and code tools share the same _spawn_lock (PyInstaller #7410)."""
18        from kiln_ai.sandbox import spawn as spawn_mod
19
20        assert spawn_mod._spawn_lock is _spawn_lock
21
22    def test_bridge_delegates_to_shared_spawn_helper(self):
23        """The shared bridge spawns via start_process_with_light_main from sandbox.spawn."""
24        from kiln_ai.tools import sandbox_bridge
25
26        assert (
27            sandbox_bridge.start_process_with_light_main
28            is start_process_with_light_main
29        )
30
31    @pytest.mark.asyncio
32    async def test_scorer_runs_through_bridge(self):
33        """Regression: a scorer executes through the shared bridge and returns its dict."""
34        code = (
35            "def score(output, trace, reference_data, task_input):\n"
36            "    return {'ok': 1.0}\n"
37        )
38        server = NestedToolServer(
39            allowlist=[], project=Project(name="shared_test"), task=None, context=None
40        )
41        res = await run_bridged_child(
42            target=execute_scorer_bridged,
43            args=(
44                code,
45                {
46                    "output": "x",
47                    "trace": None,
48                    "reference_data": None,
49                    "task_input": "y",
50                },
51            ),
52            timeout_s=10,
53            server=server,
54        )
55        assert res.result_msg is not None
56        assert res.result_msg["ok"] == {"ok": 1.0}
def test_spawn_lock_shared_with_eval(self): 
16    def test_spawn_lock_shared_with_eval(self):
17        """Code evals and code tools share the same _spawn_lock (PyInstaller #7410)."""
18        from kiln_ai.sandbox import spawn as spawn_mod
19
20        assert spawn_mod._spawn_lock is _spawn_lock

Code evals and code tools share the same _spawn_lock (PyInstaller #7410).

def test_bridge_delegates_to_shared_spawn_helper(self): 
22    def test_bridge_delegates_to_shared_spawn_helper(self):
23        """The shared bridge spawns via start_process_with_light_main from sandbox.spawn."""
24        from kiln_ai.tools import sandbox_bridge
25
26        assert (
27            sandbox_bridge.start_process_with_light_main
28            is start_process_with_light_main
29        )

The shared bridge spawns via start_process_with_light_main from sandbox.spawn.

@pytest.mark.asyncio
async def test_scorer_runs_through_bridge(self): 
31    @pytest.mark.asyncio
32    async def test_scorer_runs_through_bridge(self):
33        """Regression: a scorer executes through the shared bridge and returns its dict."""
34        code = (
35            "def score(output, trace, reference_data, task_input):\n"
36            "    return {'ok': 1.0}\n"
37        )
38        server = NestedToolServer(
39            allowlist=[], project=Project(name="shared_test"), task=None, context=None
40        )
41        res = await run_bridged_child(
42            target=execute_scorer_bridged,
43            args=(
44                code,
45                {
46                    "output": "x",
47                    "trace": None,
48                    "reference_data": None,
49                    "task_input": "y",
50                },
51            ),
52            timeout_s=10,
53            server=server,
54        )
55        assert res.result_msg is not None
56        assert res.result_msg["ok"] == {"ok": 1.0}

Regression: a scorer executes through the shared bridge and returns its dict.

class TestCallEntrypoint: 
 59class TestCallEntrypoint:
 60    def test_sync_function(self):
 61        def fn(x):
 62            return x * 2
 63
 64        assert call_entrypoint(fn, {"x": 5}) == 10
 65
 66    def test_async_function(self):
 67        async def fn(x):
 68            return x + 1
 69
 70        assert call_entrypoint(fn, {"x": 5}) == 6
 71
 72    def test_async_with_gather(self):
 73        async def fn(values):
 74            async def double(v):
 75                return v * 2
 76
 77            return await asyncio.gather(*(double(v) for v in values))
 78
 79        result = call_entrypoint(fn, {"values": [1, 2, 3]})
 80        assert result == [2, 4, 6]
 81
 82    def test_sync_returning_non_coroutine(self):
 83        def fn():
 84            return "plain"
 85
 86        assert call_entrypoint(fn, {}) == "plain"
 87
 88    def test_propagates_exception(self):
 89        def fn():
 90            raise ValueError("boom")
 91
 92        with pytest.raises(ValueError, match="boom"):
 93            call_entrypoint(fn, {})
 94
 95    def test_async_propagates_exception(self):
 96        async def fn():
 97            raise RuntimeError("async boom")
 98
 99        with pytest.raises(RuntimeError, match="async boom"):
100            call_entrypoint(fn, {})
def test_sync_function(self): 
60    def test_sync_function(self):
61        def fn(x):
62            return x * 2
63
64        assert call_entrypoint(fn, {"x": 5}) == 10
def test_async_function(self): 
66    def test_async_function(self):
67        async def fn(x):
68            return x + 1
69
70        assert call_entrypoint(fn, {"x": 5}) == 6
def test_async_with_gather(self): 
72    def test_async_with_gather(self):
73        async def fn(values):
74            async def double(v):
75                return v * 2
76
77            return await asyncio.gather(*(double(v) for v in values))
78
79        result = call_entrypoint(fn, {"values": [1, 2, 3]})
80        assert result == [2, 4, 6]
def test_sync_returning_non_coroutine(self): 
82    def test_sync_returning_non_coroutine(self):
83        def fn():
84            return "plain"
85
86        assert call_entrypoint(fn, {}) == "plain"
def test_propagates_exception(self): 
88    def test_propagates_exception(self):
89        def fn():
90            raise ValueError("boom")
91
92        with pytest.raises(ValueError, match="boom"):
93            call_entrypoint(fn, {})
def test_async_propagates_exception(self): 
 95    def test_async_propagates_exception(self):
 96        async def fn():
 97            raise RuntimeError("async boom")
 98
 99        with pytest.raises(RuntimeError, match="async boom"):
100            call_entrypoint(fn, {})