main
1# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
3from __future__ import annotations
4
5import os
6from typing import Any, cast
7
8import pytest
9
10from openai import OpenAI, AsyncOpenAI
11from tests.utils import assert_matches_type
12from openai.pagination import SyncCursorPage, AsyncCursorPage
13from openai.types.evals import (
14 RunListResponse,
15 RunCancelResponse,
16 RunCreateResponse,
17 RunDeleteResponse,
18 RunRetrieveResponse,
19)
20
21base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010")
22
23
24class TestRuns:
25 parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"])
26
27 @parametrize
28 def test_method_create(self, client: OpenAI) -> None:
29 run = client.evals.runs.create(
30 eval_id="eval_id",
31 data_source={
32 "source": {
33 "content": [{"item": {"foo": "bar"}}],
34 "type": "file_content",
35 },
36 "type": "jsonl",
37 },
38 )
39 assert_matches_type(RunCreateResponse, run, path=["response"])
40
41 @parametrize
42 def test_method_create_with_all_params(self, client: OpenAI) -> None:
43 run = client.evals.runs.create(
44 eval_id="eval_id",
45 data_source={
46 "source": {
47 "content": [
48 {
49 "item": {"foo": "bar"},
50 "sample": {"foo": "bar"},
51 }
52 ],
53 "type": "file_content",
54 },
55 "type": "jsonl",
56 },
57 metadata={"foo": "string"},
58 name="name",
59 )
60 assert_matches_type(RunCreateResponse, run, path=["response"])
61
62 @parametrize
63 def test_raw_response_create(self, client: OpenAI) -> None:
64 response = client.evals.runs.with_raw_response.create(
65 eval_id="eval_id",
66 data_source={
67 "source": {
68 "content": [{"item": {"foo": "bar"}}],
69 "type": "file_content",
70 },
71 "type": "jsonl",
72 },
73 )
74
75 assert response.is_closed is True
76 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
77 run = response.parse()
78 assert_matches_type(RunCreateResponse, run, path=["response"])
79
80 @parametrize
81 def test_streaming_response_create(self, client: OpenAI) -> None:
82 with client.evals.runs.with_streaming_response.create(
83 eval_id="eval_id",
84 data_source={
85 "source": {
86 "content": [{"item": {"foo": "bar"}}],
87 "type": "file_content",
88 },
89 "type": "jsonl",
90 },
91 ) as response:
92 assert not response.is_closed
93 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
94
95 run = response.parse()
96 assert_matches_type(RunCreateResponse, run, path=["response"])
97
98 assert cast(Any, response.is_closed) is True
99
100 @parametrize
101 def test_path_params_create(self, client: OpenAI) -> None:
102 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
103 client.evals.runs.with_raw_response.create(
104 eval_id="",
105 data_source={
106 "source": {
107 "content": [{"item": {"foo": "bar"}}],
108 "type": "file_content",
109 },
110 "type": "jsonl",
111 },
112 )
113
114 @parametrize
115 def test_method_retrieve(self, client: OpenAI) -> None:
116 run = client.evals.runs.retrieve(
117 run_id="run_id",
118 eval_id="eval_id",
119 )
120 assert_matches_type(RunRetrieveResponse, run, path=["response"])
121
122 @parametrize
123 def test_raw_response_retrieve(self, client: OpenAI) -> None:
124 response = client.evals.runs.with_raw_response.retrieve(
125 run_id="run_id",
126 eval_id="eval_id",
127 )
128
129 assert response.is_closed is True
130 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
131 run = response.parse()
132 assert_matches_type(RunRetrieveResponse, run, path=["response"])
133
134 @parametrize
135 def test_streaming_response_retrieve(self, client: OpenAI) -> None:
136 with client.evals.runs.with_streaming_response.retrieve(
137 run_id="run_id",
138 eval_id="eval_id",
139 ) as response:
140 assert not response.is_closed
141 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
142
143 run = response.parse()
144 assert_matches_type(RunRetrieveResponse, run, path=["response"])
145
146 assert cast(Any, response.is_closed) is True
147
148 @parametrize
149 def test_path_params_retrieve(self, client: OpenAI) -> None:
150 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
151 client.evals.runs.with_raw_response.retrieve(
152 run_id="run_id",
153 eval_id="",
154 )
155
156 with pytest.raises(ValueError, match=r"Expected a non-empty value for `run_id` but received ''"):
157 client.evals.runs.with_raw_response.retrieve(
158 run_id="",
159 eval_id="eval_id",
160 )
161
162 @parametrize
163 def test_method_list(self, client: OpenAI) -> None:
164 run = client.evals.runs.list(
165 eval_id="eval_id",
166 )
167 assert_matches_type(SyncCursorPage[RunListResponse], run, path=["response"])
168
169 @parametrize
170 def test_method_list_with_all_params(self, client: OpenAI) -> None:
171 run = client.evals.runs.list(
172 eval_id="eval_id",
173 after="after",
174 limit=0,
175 order="asc",
176 status="queued",
177 )
178 assert_matches_type(SyncCursorPage[RunListResponse], run, path=["response"])
179
180 @parametrize
181 def test_raw_response_list(self, client: OpenAI) -> None:
182 response = client.evals.runs.with_raw_response.list(
183 eval_id="eval_id",
184 )
185
186 assert response.is_closed is True
187 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
188 run = response.parse()
189 assert_matches_type(SyncCursorPage[RunListResponse], run, path=["response"])
190
191 @parametrize
192 def test_streaming_response_list(self, client: OpenAI) -> None:
193 with client.evals.runs.with_streaming_response.list(
194 eval_id="eval_id",
195 ) as response:
196 assert not response.is_closed
197 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
198
199 run = response.parse()
200 assert_matches_type(SyncCursorPage[RunListResponse], run, path=["response"])
201
202 assert cast(Any, response.is_closed) is True
203
204 @parametrize
205 def test_path_params_list(self, client: OpenAI) -> None:
206 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
207 client.evals.runs.with_raw_response.list(
208 eval_id="",
209 )
210
211 @parametrize
212 def test_method_delete(self, client: OpenAI) -> None:
213 run = client.evals.runs.delete(
214 run_id="run_id",
215 eval_id="eval_id",
216 )
217 assert_matches_type(RunDeleteResponse, run, path=["response"])
218
219 @parametrize
220 def test_raw_response_delete(self, client: OpenAI) -> None:
221 response = client.evals.runs.with_raw_response.delete(
222 run_id="run_id",
223 eval_id="eval_id",
224 )
225
226 assert response.is_closed is True
227 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
228 run = response.parse()
229 assert_matches_type(RunDeleteResponse, run, path=["response"])
230
231 @parametrize
232 def test_streaming_response_delete(self, client: OpenAI) -> None:
233 with client.evals.runs.with_streaming_response.delete(
234 run_id="run_id",
235 eval_id="eval_id",
236 ) as response:
237 assert not response.is_closed
238 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
239
240 run = response.parse()
241 assert_matches_type(RunDeleteResponse, run, path=["response"])
242
243 assert cast(Any, response.is_closed) is True
244
245 @parametrize
246 def test_path_params_delete(self, client: OpenAI) -> None:
247 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
248 client.evals.runs.with_raw_response.delete(
249 run_id="run_id",
250 eval_id="",
251 )
252
253 with pytest.raises(ValueError, match=r"Expected a non-empty value for `run_id` but received ''"):
254 client.evals.runs.with_raw_response.delete(
255 run_id="",
256 eval_id="eval_id",
257 )
258
259 @parametrize
260 def test_method_cancel(self, client: OpenAI) -> None:
261 run = client.evals.runs.cancel(
262 run_id="run_id",
263 eval_id="eval_id",
264 )
265 assert_matches_type(RunCancelResponse, run, path=["response"])
266
267 @parametrize
268 def test_raw_response_cancel(self, client: OpenAI) -> None:
269 response = client.evals.runs.with_raw_response.cancel(
270 run_id="run_id",
271 eval_id="eval_id",
272 )
273
274 assert response.is_closed is True
275 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
276 run = response.parse()
277 assert_matches_type(RunCancelResponse, run, path=["response"])
278
279 @parametrize
280 def test_streaming_response_cancel(self, client: OpenAI) -> None:
281 with client.evals.runs.with_streaming_response.cancel(
282 run_id="run_id",
283 eval_id="eval_id",
284 ) as response:
285 assert not response.is_closed
286 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
287
288 run = response.parse()
289 assert_matches_type(RunCancelResponse, run, path=["response"])
290
291 assert cast(Any, response.is_closed) is True
292
293 @parametrize
294 def test_path_params_cancel(self, client: OpenAI) -> None:
295 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
296 client.evals.runs.with_raw_response.cancel(
297 run_id="run_id",
298 eval_id="",
299 )
300
301 with pytest.raises(ValueError, match=r"Expected a non-empty value for `run_id` but received ''"):
302 client.evals.runs.with_raw_response.cancel(
303 run_id="",
304 eval_id="eval_id",
305 )
306
307
308class TestAsyncRuns:
309 parametrize = pytest.mark.parametrize(
310 "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"]
311 )
312
313 @parametrize
314 async def test_method_create(self, async_client: AsyncOpenAI) -> None:
315 run = await async_client.evals.runs.create(
316 eval_id="eval_id",
317 data_source={
318 "source": {
319 "content": [{"item": {"foo": "bar"}}],
320 "type": "file_content",
321 },
322 "type": "jsonl",
323 },
324 )
325 assert_matches_type(RunCreateResponse, run, path=["response"])
326
327 @parametrize
328 async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> None:
329 run = await async_client.evals.runs.create(
330 eval_id="eval_id",
331 data_source={
332 "source": {
333 "content": [
334 {
335 "item": {"foo": "bar"},
336 "sample": {"foo": "bar"},
337 }
338 ],
339 "type": "file_content",
340 },
341 "type": "jsonl",
342 },
343 metadata={"foo": "string"},
344 name="name",
345 )
346 assert_matches_type(RunCreateResponse, run, path=["response"])
347
348 @parametrize
349 async def test_raw_response_create(self, async_client: AsyncOpenAI) -> None:
350 response = await async_client.evals.runs.with_raw_response.create(
351 eval_id="eval_id",
352 data_source={
353 "source": {
354 "content": [{"item": {"foo": "bar"}}],
355 "type": "file_content",
356 },
357 "type": "jsonl",
358 },
359 )
360
361 assert response.is_closed is True
362 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
363 run = response.parse()
364 assert_matches_type(RunCreateResponse, run, path=["response"])
365
366 @parametrize
367 async def test_streaming_response_create(self, async_client: AsyncOpenAI) -> None:
368 async with async_client.evals.runs.with_streaming_response.create(
369 eval_id="eval_id",
370 data_source={
371 "source": {
372 "content": [{"item": {"foo": "bar"}}],
373 "type": "file_content",
374 },
375 "type": "jsonl",
376 },
377 ) as response:
378 assert not response.is_closed
379 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
380
381 run = await response.parse()
382 assert_matches_type(RunCreateResponse, run, path=["response"])
383
384 assert cast(Any, response.is_closed) is True
385
386 @parametrize
387 async def test_path_params_create(self, async_client: AsyncOpenAI) -> None:
388 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
389 await async_client.evals.runs.with_raw_response.create(
390 eval_id="",
391 data_source={
392 "source": {
393 "content": [{"item": {"foo": "bar"}}],
394 "type": "file_content",
395 },
396 "type": "jsonl",
397 },
398 )
399
400 @parametrize
401 async def test_method_retrieve(self, async_client: AsyncOpenAI) -> None:
402 run = await async_client.evals.runs.retrieve(
403 run_id="run_id",
404 eval_id="eval_id",
405 )
406 assert_matches_type(RunRetrieveResponse, run, path=["response"])
407
408 @parametrize
409 async def test_raw_response_retrieve(self, async_client: AsyncOpenAI) -> None:
410 response = await async_client.evals.runs.with_raw_response.retrieve(
411 run_id="run_id",
412 eval_id="eval_id",
413 )
414
415 assert response.is_closed is True
416 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
417 run = response.parse()
418 assert_matches_type(RunRetrieveResponse, run, path=["response"])
419
420 @parametrize
421 async def test_streaming_response_retrieve(self, async_client: AsyncOpenAI) -> None:
422 async with async_client.evals.runs.with_streaming_response.retrieve(
423 run_id="run_id",
424 eval_id="eval_id",
425 ) as response:
426 assert not response.is_closed
427 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
428
429 run = await response.parse()
430 assert_matches_type(RunRetrieveResponse, run, path=["response"])
431
432 assert cast(Any, response.is_closed) is True
433
434 @parametrize
435 async def test_path_params_retrieve(self, async_client: AsyncOpenAI) -> None:
436 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
437 await async_client.evals.runs.with_raw_response.retrieve(
438 run_id="run_id",
439 eval_id="",
440 )
441
442 with pytest.raises(ValueError, match=r"Expected a non-empty value for `run_id` but received ''"):
443 await async_client.evals.runs.with_raw_response.retrieve(
444 run_id="",
445 eval_id="eval_id",
446 )
447
448 @parametrize
449 async def test_method_list(self, async_client: AsyncOpenAI) -> None:
450 run = await async_client.evals.runs.list(
451 eval_id="eval_id",
452 )
453 assert_matches_type(AsyncCursorPage[RunListResponse], run, path=["response"])
454
455 @parametrize
456 async def test_method_list_with_all_params(self, async_client: AsyncOpenAI) -> None:
457 run = await async_client.evals.runs.list(
458 eval_id="eval_id",
459 after="after",
460 limit=0,
461 order="asc",
462 status="queued",
463 )
464 assert_matches_type(AsyncCursorPage[RunListResponse], run, path=["response"])
465
466 @parametrize
467 async def test_raw_response_list(self, async_client: AsyncOpenAI) -> None:
468 response = await async_client.evals.runs.with_raw_response.list(
469 eval_id="eval_id",
470 )
471
472 assert response.is_closed is True
473 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
474 run = response.parse()
475 assert_matches_type(AsyncCursorPage[RunListResponse], run, path=["response"])
476
477 @parametrize
478 async def test_streaming_response_list(self, async_client: AsyncOpenAI) -> None:
479 async with async_client.evals.runs.with_streaming_response.list(
480 eval_id="eval_id",
481 ) as response:
482 assert not response.is_closed
483 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
484
485 run = await response.parse()
486 assert_matches_type(AsyncCursorPage[RunListResponse], run, path=["response"])
487
488 assert cast(Any, response.is_closed) is True
489
490 @parametrize
491 async def test_path_params_list(self, async_client: AsyncOpenAI) -> None:
492 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
493 await async_client.evals.runs.with_raw_response.list(
494 eval_id="",
495 )
496
497 @parametrize
498 async def test_method_delete(self, async_client: AsyncOpenAI) -> None:
499 run = await async_client.evals.runs.delete(
500 run_id="run_id",
501 eval_id="eval_id",
502 )
503 assert_matches_type(RunDeleteResponse, run, path=["response"])
504
505 @parametrize
506 async def test_raw_response_delete(self, async_client: AsyncOpenAI) -> None:
507 response = await async_client.evals.runs.with_raw_response.delete(
508 run_id="run_id",
509 eval_id="eval_id",
510 )
511
512 assert response.is_closed is True
513 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
514 run = response.parse()
515 assert_matches_type(RunDeleteResponse, run, path=["response"])
516
517 @parametrize
518 async def test_streaming_response_delete(self, async_client: AsyncOpenAI) -> None:
519 async with async_client.evals.runs.with_streaming_response.delete(
520 run_id="run_id",
521 eval_id="eval_id",
522 ) as response:
523 assert not response.is_closed
524 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
525
526 run = await response.parse()
527 assert_matches_type(RunDeleteResponse, run, path=["response"])
528
529 assert cast(Any, response.is_closed) is True
530
531 @parametrize
532 async def test_path_params_delete(self, async_client: AsyncOpenAI) -> None:
533 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
534 await async_client.evals.runs.with_raw_response.delete(
535 run_id="run_id",
536 eval_id="",
537 )
538
539 with pytest.raises(ValueError, match=r"Expected a non-empty value for `run_id` but received ''"):
540 await async_client.evals.runs.with_raw_response.delete(
541 run_id="",
542 eval_id="eval_id",
543 )
544
545 @parametrize
546 async def test_method_cancel(self, async_client: AsyncOpenAI) -> None:
547 run = await async_client.evals.runs.cancel(
548 run_id="run_id",
549 eval_id="eval_id",
550 )
551 assert_matches_type(RunCancelResponse, run, path=["response"])
552
553 @parametrize
554 async def test_raw_response_cancel(self, async_client: AsyncOpenAI) -> None:
555 response = await async_client.evals.runs.with_raw_response.cancel(
556 run_id="run_id",
557 eval_id="eval_id",
558 )
559
560 assert response.is_closed is True
561 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
562 run = response.parse()
563 assert_matches_type(RunCancelResponse, run, path=["response"])
564
565 @parametrize
566 async def test_streaming_response_cancel(self, async_client: AsyncOpenAI) -> None:
567 async with async_client.evals.runs.with_streaming_response.cancel(
568 run_id="run_id",
569 eval_id="eval_id",
570 ) as response:
571 assert not response.is_closed
572 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
573
574 run = await response.parse()
575 assert_matches_type(RunCancelResponse, run, path=["response"])
576
577 assert cast(Any, response.is_closed) is True
578
579 @parametrize
580 async def test_path_params_cancel(self, async_client: AsyncOpenAI) -> None:
581 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
582 await async_client.evals.runs.with_raw_response.cancel(
583 run_id="run_id",
584 eval_id="",
585 )
586
587 with pytest.raises(ValueError, match=r"Expected a non-empty value for `run_id` but received ''"):
588 await async_client.evals.runs.with_raw_response.cancel(
589 run_id="",
590 eval_id="eval_id",
591 )