main
1# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
3from __future__ import annotations
4
5import os
6from typing import Any, cast
7
8import pytest
9
10from openai import OpenAI, AsyncOpenAI
11from tests.utils import assert_matches_type
12from openai.types import (
13 EvalListResponse,
14 EvalCreateResponse,
15 EvalDeleteResponse,
16 EvalUpdateResponse,
17 EvalRetrieveResponse,
18)
19from openai.pagination import SyncCursorPage, AsyncCursorPage
20
21base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010")
22
23
24class TestEvals:
25 parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"])
26
27 @parametrize
28 def test_method_create(self, client: OpenAI) -> None:
29 eval = client.evals.create(
30 data_source_config={
31 "item_schema": {"foo": "bar"},
32 "type": "custom",
33 },
34 testing_criteria=[
35 {
36 "input": [
37 {
38 "content": "content",
39 "role": "role",
40 }
41 ],
42 "labels": ["string"],
43 "model": "model",
44 "name": "name",
45 "passing_labels": ["string"],
46 "type": "label_model",
47 }
48 ],
49 )
50 assert_matches_type(EvalCreateResponse, eval, path=["response"])
51
52 @parametrize
53 def test_method_create_with_all_params(self, client: OpenAI) -> None:
54 eval = client.evals.create(
55 data_source_config={
56 "item_schema": {"foo": "bar"},
57 "type": "custom",
58 "include_sample_schema": True,
59 },
60 testing_criteria=[
61 {
62 "input": [
63 {
64 "content": "content",
65 "role": "role",
66 }
67 ],
68 "labels": ["string"],
69 "model": "model",
70 "name": "name",
71 "passing_labels": ["string"],
72 "type": "label_model",
73 }
74 ],
75 metadata={"foo": "string"},
76 name="name",
77 )
78 assert_matches_type(EvalCreateResponse, eval, path=["response"])
79
80 @parametrize
81 def test_raw_response_create(self, client: OpenAI) -> None:
82 response = client.evals.with_raw_response.create(
83 data_source_config={
84 "item_schema": {"foo": "bar"},
85 "type": "custom",
86 },
87 testing_criteria=[
88 {
89 "input": [
90 {
91 "content": "content",
92 "role": "role",
93 }
94 ],
95 "labels": ["string"],
96 "model": "model",
97 "name": "name",
98 "passing_labels": ["string"],
99 "type": "label_model",
100 }
101 ],
102 )
103
104 assert response.is_closed is True
105 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
106 eval = response.parse()
107 assert_matches_type(EvalCreateResponse, eval, path=["response"])
108
109 @parametrize
110 def test_streaming_response_create(self, client: OpenAI) -> None:
111 with client.evals.with_streaming_response.create(
112 data_source_config={
113 "item_schema": {"foo": "bar"},
114 "type": "custom",
115 },
116 testing_criteria=[
117 {
118 "input": [
119 {
120 "content": "content",
121 "role": "role",
122 }
123 ],
124 "labels": ["string"],
125 "model": "model",
126 "name": "name",
127 "passing_labels": ["string"],
128 "type": "label_model",
129 }
130 ],
131 ) as response:
132 assert not response.is_closed
133 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
134
135 eval = response.parse()
136 assert_matches_type(EvalCreateResponse, eval, path=["response"])
137
138 assert cast(Any, response.is_closed) is True
139
140 @parametrize
141 def test_method_retrieve(self, client: OpenAI) -> None:
142 eval = client.evals.retrieve(
143 "eval_id",
144 )
145 assert_matches_type(EvalRetrieveResponse, eval, path=["response"])
146
147 @parametrize
148 def test_raw_response_retrieve(self, client: OpenAI) -> None:
149 response = client.evals.with_raw_response.retrieve(
150 "eval_id",
151 )
152
153 assert response.is_closed is True
154 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
155 eval = response.parse()
156 assert_matches_type(EvalRetrieveResponse, eval, path=["response"])
157
158 @parametrize
159 def test_streaming_response_retrieve(self, client: OpenAI) -> None:
160 with client.evals.with_streaming_response.retrieve(
161 "eval_id",
162 ) as response:
163 assert not response.is_closed
164 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
165
166 eval = response.parse()
167 assert_matches_type(EvalRetrieveResponse, eval, path=["response"])
168
169 assert cast(Any, response.is_closed) is True
170
171 @parametrize
172 def test_path_params_retrieve(self, client: OpenAI) -> None:
173 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
174 client.evals.with_raw_response.retrieve(
175 "",
176 )
177
178 @parametrize
179 def test_method_update(self, client: OpenAI) -> None:
180 eval = client.evals.update(
181 eval_id="eval_id",
182 )
183 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
184
185 @parametrize
186 def test_method_update_with_all_params(self, client: OpenAI) -> None:
187 eval = client.evals.update(
188 eval_id="eval_id",
189 metadata={"foo": "string"},
190 name="name",
191 )
192 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
193
194 @parametrize
195 def test_raw_response_update(self, client: OpenAI) -> None:
196 response = client.evals.with_raw_response.update(
197 eval_id="eval_id",
198 )
199
200 assert response.is_closed is True
201 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
202 eval = response.parse()
203 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
204
205 @parametrize
206 def test_streaming_response_update(self, client: OpenAI) -> None:
207 with client.evals.with_streaming_response.update(
208 eval_id="eval_id",
209 ) as response:
210 assert not response.is_closed
211 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
212
213 eval = response.parse()
214 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
215
216 assert cast(Any, response.is_closed) is True
217
218 @parametrize
219 def test_path_params_update(self, client: OpenAI) -> None:
220 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
221 client.evals.with_raw_response.update(
222 eval_id="",
223 )
224
225 @parametrize
226 def test_method_list(self, client: OpenAI) -> None:
227 eval = client.evals.list()
228 assert_matches_type(SyncCursorPage[EvalListResponse], eval, path=["response"])
229
230 @parametrize
231 def test_method_list_with_all_params(self, client: OpenAI) -> None:
232 eval = client.evals.list(
233 after="after",
234 limit=0,
235 order="asc",
236 order_by="created_at",
237 )
238 assert_matches_type(SyncCursorPage[EvalListResponse], eval, path=["response"])
239
240 @parametrize
241 def test_raw_response_list(self, client: OpenAI) -> None:
242 response = client.evals.with_raw_response.list()
243
244 assert response.is_closed is True
245 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
246 eval = response.parse()
247 assert_matches_type(SyncCursorPage[EvalListResponse], eval, path=["response"])
248
249 @parametrize
250 def test_streaming_response_list(self, client: OpenAI) -> None:
251 with client.evals.with_streaming_response.list() as response:
252 assert not response.is_closed
253 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
254
255 eval = response.parse()
256 assert_matches_type(SyncCursorPage[EvalListResponse], eval, path=["response"])
257
258 assert cast(Any, response.is_closed) is True
259
260 @parametrize
261 def test_method_delete(self, client: OpenAI) -> None:
262 eval = client.evals.delete(
263 "eval_id",
264 )
265 assert_matches_type(EvalDeleteResponse, eval, path=["response"])
266
267 @parametrize
268 def test_raw_response_delete(self, client: OpenAI) -> None:
269 response = client.evals.with_raw_response.delete(
270 "eval_id",
271 )
272
273 assert response.is_closed is True
274 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
275 eval = response.parse()
276 assert_matches_type(EvalDeleteResponse, eval, path=["response"])
277
278 @parametrize
279 def test_streaming_response_delete(self, client: OpenAI) -> None:
280 with client.evals.with_streaming_response.delete(
281 "eval_id",
282 ) as response:
283 assert not response.is_closed
284 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
285
286 eval = response.parse()
287 assert_matches_type(EvalDeleteResponse, eval, path=["response"])
288
289 assert cast(Any, response.is_closed) is True
290
291 @parametrize
292 def test_path_params_delete(self, client: OpenAI) -> None:
293 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
294 client.evals.with_raw_response.delete(
295 "",
296 )
297
298
299class TestAsyncEvals:
300 parametrize = pytest.mark.parametrize(
301 "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"]
302 )
303
304 @parametrize
305 async def test_method_create(self, async_client: AsyncOpenAI) -> None:
306 eval = await async_client.evals.create(
307 data_source_config={
308 "item_schema": {"foo": "bar"},
309 "type": "custom",
310 },
311 testing_criteria=[
312 {
313 "input": [
314 {
315 "content": "content",
316 "role": "role",
317 }
318 ],
319 "labels": ["string"],
320 "model": "model",
321 "name": "name",
322 "passing_labels": ["string"],
323 "type": "label_model",
324 }
325 ],
326 )
327 assert_matches_type(EvalCreateResponse, eval, path=["response"])
328
329 @parametrize
330 async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> None:
331 eval = await async_client.evals.create(
332 data_source_config={
333 "item_schema": {"foo": "bar"},
334 "type": "custom",
335 "include_sample_schema": True,
336 },
337 testing_criteria=[
338 {
339 "input": [
340 {
341 "content": "content",
342 "role": "role",
343 }
344 ],
345 "labels": ["string"],
346 "model": "model",
347 "name": "name",
348 "passing_labels": ["string"],
349 "type": "label_model",
350 }
351 ],
352 metadata={"foo": "string"},
353 name="name",
354 )
355 assert_matches_type(EvalCreateResponse, eval, path=["response"])
356
357 @parametrize
358 async def test_raw_response_create(self, async_client: AsyncOpenAI) -> None:
359 response = await async_client.evals.with_raw_response.create(
360 data_source_config={
361 "item_schema": {"foo": "bar"},
362 "type": "custom",
363 },
364 testing_criteria=[
365 {
366 "input": [
367 {
368 "content": "content",
369 "role": "role",
370 }
371 ],
372 "labels": ["string"],
373 "model": "model",
374 "name": "name",
375 "passing_labels": ["string"],
376 "type": "label_model",
377 }
378 ],
379 )
380
381 assert response.is_closed is True
382 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
383 eval = response.parse()
384 assert_matches_type(EvalCreateResponse, eval, path=["response"])
385
386 @parametrize
387 async def test_streaming_response_create(self, async_client: AsyncOpenAI) -> None:
388 async with async_client.evals.with_streaming_response.create(
389 data_source_config={
390 "item_schema": {"foo": "bar"},
391 "type": "custom",
392 },
393 testing_criteria=[
394 {
395 "input": [
396 {
397 "content": "content",
398 "role": "role",
399 }
400 ],
401 "labels": ["string"],
402 "model": "model",
403 "name": "name",
404 "passing_labels": ["string"],
405 "type": "label_model",
406 }
407 ],
408 ) as response:
409 assert not response.is_closed
410 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
411
412 eval = await response.parse()
413 assert_matches_type(EvalCreateResponse, eval, path=["response"])
414
415 assert cast(Any, response.is_closed) is True
416
417 @parametrize
418 async def test_method_retrieve(self, async_client: AsyncOpenAI) -> None:
419 eval = await async_client.evals.retrieve(
420 "eval_id",
421 )
422 assert_matches_type(EvalRetrieveResponse, eval, path=["response"])
423
424 @parametrize
425 async def test_raw_response_retrieve(self, async_client: AsyncOpenAI) -> None:
426 response = await async_client.evals.with_raw_response.retrieve(
427 "eval_id",
428 )
429
430 assert response.is_closed is True
431 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
432 eval = response.parse()
433 assert_matches_type(EvalRetrieveResponse, eval, path=["response"])
434
435 @parametrize
436 async def test_streaming_response_retrieve(self, async_client: AsyncOpenAI) -> None:
437 async with async_client.evals.with_streaming_response.retrieve(
438 "eval_id",
439 ) as response:
440 assert not response.is_closed
441 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
442
443 eval = await response.parse()
444 assert_matches_type(EvalRetrieveResponse, eval, path=["response"])
445
446 assert cast(Any, response.is_closed) is True
447
448 @parametrize
449 async def test_path_params_retrieve(self, async_client: AsyncOpenAI) -> None:
450 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
451 await async_client.evals.with_raw_response.retrieve(
452 "",
453 )
454
455 @parametrize
456 async def test_method_update(self, async_client: AsyncOpenAI) -> None:
457 eval = await async_client.evals.update(
458 eval_id="eval_id",
459 )
460 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
461
462 @parametrize
463 async def test_method_update_with_all_params(self, async_client: AsyncOpenAI) -> None:
464 eval = await async_client.evals.update(
465 eval_id="eval_id",
466 metadata={"foo": "string"},
467 name="name",
468 )
469 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
470
471 @parametrize
472 async def test_raw_response_update(self, async_client: AsyncOpenAI) -> None:
473 response = await async_client.evals.with_raw_response.update(
474 eval_id="eval_id",
475 )
476
477 assert response.is_closed is True
478 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
479 eval = response.parse()
480 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
481
482 @parametrize
483 async def test_streaming_response_update(self, async_client: AsyncOpenAI) -> None:
484 async with async_client.evals.with_streaming_response.update(
485 eval_id="eval_id",
486 ) as response:
487 assert not response.is_closed
488 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
489
490 eval = await response.parse()
491 assert_matches_type(EvalUpdateResponse, eval, path=["response"])
492
493 assert cast(Any, response.is_closed) is True
494
495 @parametrize
496 async def test_path_params_update(self, async_client: AsyncOpenAI) -> None:
497 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
498 await async_client.evals.with_raw_response.update(
499 eval_id="",
500 )
501
502 @parametrize
503 async def test_method_list(self, async_client: AsyncOpenAI) -> None:
504 eval = await async_client.evals.list()
505 assert_matches_type(AsyncCursorPage[EvalListResponse], eval, path=["response"])
506
507 @parametrize
508 async def test_method_list_with_all_params(self, async_client: AsyncOpenAI) -> None:
509 eval = await async_client.evals.list(
510 after="after",
511 limit=0,
512 order="asc",
513 order_by="created_at",
514 )
515 assert_matches_type(AsyncCursorPage[EvalListResponse], eval, path=["response"])
516
517 @parametrize
518 async def test_raw_response_list(self, async_client: AsyncOpenAI) -> None:
519 response = await async_client.evals.with_raw_response.list()
520
521 assert response.is_closed is True
522 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
523 eval = response.parse()
524 assert_matches_type(AsyncCursorPage[EvalListResponse], eval, path=["response"])
525
526 @parametrize
527 async def test_streaming_response_list(self, async_client: AsyncOpenAI) -> None:
528 async with async_client.evals.with_streaming_response.list() as response:
529 assert not response.is_closed
530 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
531
532 eval = await response.parse()
533 assert_matches_type(AsyncCursorPage[EvalListResponse], eval, path=["response"])
534
535 assert cast(Any, response.is_closed) is True
536
537 @parametrize
538 async def test_method_delete(self, async_client: AsyncOpenAI) -> None:
539 eval = await async_client.evals.delete(
540 "eval_id",
541 )
542 assert_matches_type(EvalDeleteResponse, eval, path=["response"])
543
544 @parametrize
545 async def test_raw_response_delete(self, async_client: AsyncOpenAI) -> None:
546 response = await async_client.evals.with_raw_response.delete(
547 "eval_id",
548 )
549
550 assert response.is_closed is True
551 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
552 eval = response.parse()
553 assert_matches_type(EvalDeleteResponse, eval, path=["response"])
554
555 @parametrize
556 async def test_streaming_response_delete(self, async_client: AsyncOpenAI) -> None:
557 async with async_client.evals.with_streaming_response.delete(
558 "eval_id",
559 ) as response:
560 assert not response.is_closed
561 assert response.http_request.headers.get("X-Stainless-Lang") == "python"
562
563 eval = await response.parse()
564 assert_matches_type(EvalDeleteResponse, eval, path=["response"])
565
566 assert cast(Any, response.is_closed) is True
567
568 @parametrize
569 async def test_path_params_delete(self, async_client: AsyncOpenAI) -> None:
570 with pytest.raises(ValueError, match=r"Expected a non-empty value for `eval_id` but received ''"):
571 await async_client.evals.with_raw_response.delete(
572 "",
573 )