main
1# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
2
3from __future__ import annotations
4
5from typing import Optional
6from typing_extensions import Literal
7
8import httpx
9
10from .... import _legacy_response
11from ...._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
12from ...._utils import maybe_transform, async_maybe_transform
13from ...._compat import cached_property
14from ...._resource import SyncAPIResource, AsyncAPIResource
15from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper
16from .output_items import (
17 OutputItems,
18 AsyncOutputItems,
19 OutputItemsWithRawResponse,
20 AsyncOutputItemsWithRawResponse,
21 OutputItemsWithStreamingResponse,
22 AsyncOutputItemsWithStreamingResponse,
23)
24from ....pagination import SyncCursorPage, AsyncCursorPage
25from ....types.evals import run_list_params, run_create_params
26from ...._base_client import AsyncPaginator, make_request_options
27from ....types.shared_params.metadata import Metadata
28from ....types.evals.run_list_response import RunListResponse
29from ....types.evals.run_cancel_response import RunCancelResponse
30from ....types.evals.run_create_response import RunCreateResponse
31from ....types.evals.run_delete_response import RunDeleteResponse
32from ....types.evals.run_retrieve_response import RunRetrieveResponse
33
34__all__ = ["Runs", "AsyncRuns"]
35
36
37class Runs(SyncAPIResource):
38 @cached_property
39 def output_items(self) -> OutputItems:
40 return OutputItems(self._client)
41
42 @cached_property
43 def with_raw_response(self) -> RunsWithRawResponse:
44 """
45 This property can be used as a prefix for any HTTP method call to return
46 the raw response object instead of the parsed content.
47
48 For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
49 """
50 return RunsWithRawResponse(self)
51
52 @cached_property
53 def with_streaming_response(self) -> RunsWithStreamingResponse:
54 """
55 An alternative to `.with_raw_response` that doesn't eagerly read the response body.
56
57 For more information, see https://www.github.com/openai/openai-python#with_streaming_response
58 """
59 return RunsWithStreamingResponse(self)
60
61 def create(
62 self,
63 eval_id: str,
64 *,
65 data_source: run_create_params.DataSource,
66 metadata: Optional[Metadata] | Omit = omit,
67 name: str | Omit = omit,
68 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
69 # The extra values given here take precedence over values defined on the client or passed to this method.
70 extra_headers: Headers | None = None,
71 extra_query: Query | None = None,
72 extra_body: Body | None = None,
73 timeout: float | httpx.Timeout | None | NotGiven = not_given,
74 ) -> RunCreateResponse:
75 """
76 Kicks off a new run for a given evaluation, specifying the data source, and what
77 model configuration to use to test. The datasource will be validated against the
78 schema specified in the config of the evaluation.
79
80 Args:
81 data_source: Details about the run's data source.
82
83 metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
84 for storing additional information about the object in a structured format, and
85 querying for objects via API or the dashboard.
86
87 Keys are strings with a maximum length of 64 characters. Values are strings with
88 a maximum length of 512 characters.
89
90 name: The name of the run.
91
92 extra_headers: Send extra headers
93
94 extra_query: Add additional query parameters to the request
95
96 extra_body: Add additional JSON properties to the request
97
98 timeout: Override the client-level default timeout for this request, in seconds
99 """
100 if not eval_id:
101 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
102 return self._post(
103 f"/evals/{eval_id}/runs",
104 body=maybe_transform(
105 {
106 "data_source": data_source,
107 "metadata": metadata,
108 "name": name,
109 },
110 run_create_params.RunCreateParams,
111 ),
112 options=make_request_options(
113 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
114 ),
115 cast_to=RunCreateResponse,
116 )
117
118 def retrieve(
119 self,
120 run_id: str,
121 *,
122 eval_id: str,
123 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
124 # The extra values given here take precedence over values defined on the client or passed to this method.
125 extra_headers: Headers | None = None,
126 extra_query: Query | None = None,
127 extra_body: Body | None = None,
128 timeout: float | httpx.Timeout | None | NotGiven = not_given,
129 ) -> RunRetrieveResponse:
130 """
131 Get an evaluation run by ID.
132
133 Args:
134 extra_headers: Send extra headers
135
136 extra_query: Add additional query parameters to the request
137
138 extra_body: Add additional JSON properties to the request
139
140 timeout: Override the client-level default timeout for this request, in seconds
141 """
142 if not eval_id:
143 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
144 if not run_id:
145 raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
146 return self._get(
147 f"/evals/{eval_id}/runs/{run_id}",
148 options=make_request_options(
149 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
150 ),
151 cast_to=RunRetrieveResponse,
152 )
153
154 def list(
155 self,
156 eval_id: str,
157 *,
158 after: str | Omit = omit,
159 limit: int | Omit = omit,
160 order: Literal["asc", "desc"] | Omit = omit,
161 status: Literal["queued", "in_progress", "completed", "canceled", "failed"] | Omit = omit,
162 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
163 # The extra values given here take precedence over values defined on the client or passed to this method.
164 extra_headers: Headers | None = None,
165 extra_query: Query | None = None,
166 extra_body: Body | None = None,
167 timeout: float | httpx.Timeout | None | NotGiven = not_given,
168 ) -> SyncCursorPage[RunListResponse]:
169 """
170 Get a list of runs for an evaluation.
171
172 Args:
173 after: Identifier for the last run from the previous pagination request.
174
175 limit: Number of runs to retrieve.
176
177 order: Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for
178 descending order. Defaults to `asc`.
179
180 status: Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed`
181 | `canceled`.
182
183 extra_headers: Send extra headers
184
185 extra_query: Add additional query parameters to the request
186
187 extra_body: Add additional JSON properties to the request
188
189 timeout: Override the client-level default timeout for this request, in seconds
190 """
191 if not eval_id:
192 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
193 return self._get_api_list(
194 f"/evals/{eval_id}/runs",
195 page=SyncCursorPage[RunListResponse],
196 options=make_request_options(
197 extra_headers=extra_headers,
198 extra_query=extra_query,
199 extra_body=extra_body,
200 timeout=timeout,
201 query=maybe_transform(
202 {
203 "after": after,
204 "limit": limit,
205 "order": order,
206 "status": status,
207 },
208 run_list_params.RunListParams,
209 ),
210 ),
211 model=RunListResponse,
212 )
213
214 def delete(
215 self,
216 run_id: str,
217 *,
218 eval_id: str,
219 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
220 # The extra values given here take precedence over values defined on the client or passed to this method.
221 extra_headers: Headers | None = None,
222 extra_query: Query | None = None,
223 extra_body: Body | None = None,
224 timeout: float | httpx.Timeout | None | NotGiven = not_given,
225 ) -> RunDeleteResponse:
226 """
227 Delete an eval run.
228
229 Args:
230 extra_headers: Send extra headers
231
232 extra_query: Add additional query parameters to the request
233
234 extra_body: Add additional JSON properties to the request
235
236 timeout: Override the client-level default timeout for this request, in seconds
237 """
238 if not eval_id:
239 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
240 if not run_id:
241 raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
242 return self._delete(
243 f"/evals/{eval_id}/runs/{run_id}",
244 options=make_request_options(
245 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
246 ),
247 cast_to=RunDeleteResponse,
248 )
249
250 def cancel(
251 self,
252 run_id: str,
253 *,
254 eval_id: str,
255 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
256 # The extra values given here take precedence over values defined on the client or passed to this method.
257 extra_headers: Headers | None = None,
258 extra_query: Query | None = None,
259 extra_body: Body | None = None,
260 timeout: float | httpx.Timeout | None | NotGiven = not_given,
261 ) -> RunCancelResponse:
262 """
263 Cancel an ongoing evaluation run.
264
265 Args:
266 extra_headers: Send extra headers
267
268 extra_query: Add additional query parameters to the request
269
270 extra_body: Add additional JSON properties to the request
271
272 timeout: Override the client-level default timeout for this request, in seconds
273 """
274 if not eval_id:
275 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
276 if not run_id:
277 raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
278 return self._post(
279 f"/evals/{eval_id}/runs/{run_id}",
280 options=make_request_options(
281 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
282 ),
283 cast_to=RunCancelResponse,
284 )
285
286
287class AsyncRuns(AsyncAPIResource):
288 @cached_property
289 def output_items(self) -> AsyncOutputItems:
290 return AsyncOutputItems(self._client)
291
292 @cached_property
293 def with_raw_response(self) -> AsyncRunsWithRawResponse:
294 """
295 This property can be used as a prefix for any HTTP method call to return
296 the raw response object instead of the parsed content.
297
298 For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
299 """
300 return AsyncRunsWithRawResponse(self)
301
302 @cached_property
303 def with_streaming_response(self) -> AsyncRunsWithStreamingResponse:
304 """
305 An alternative to `.with_raw_response` that doesn't eagerly read the response body.
306
307 For more information, see https://www.github.com/openai/openai-python#with_streaming_response
308 """
309 return AsyncRunsWithStreamingResponse(self)
310
311 async def create(
312 self,
313 eval_id: str,
314 *,
315 data_source: run_create_params.DataSource,
316 metadata: Optional[Metadata] | Omit = omit,
317 name: str | Omit = omit,
318 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
319 # The extra values given here take precedence over values defined on the client or passed to this method.
320 extra_headers: Headers | None = None,
321 extra_query: Query | None = None,
322 extra_body: Body | None = None,
323 timeout: float | httpx.Timeout | None | NotGiven = not_given,
324 ) -> RunCreateResponse:
325 """
326 Kicks off a new run for a given evaluation, specifying the data source, and what
327 model configuration to use to test. The datasource will be validated against the
328 schema specified in the config of the evaluation.
329
330 Args:
331 data_source: Details about the run's data source.
332
333 metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
334 for storing additional information about the object in a structured format, and
335 querying for objects via API or the dashboard.
336
337 Keys are strings with a maximum length of 64 characters. Values are strings with
338 a maximum length of 512 characters.
339
340 name: The name of the run.
341
342 extra_headers: Send extra headers
343
344 extra_query: Add additional query parameters to the request
345
346 extra_body: Add additional JSON properties to the request
347
348 timeout: Override the client-level default timeout for this request, in seconds
349 """
350 if not eval_id:
351 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
352 return await self._post(
353 f"/evals/{eval_id}/runs",
354 body=await async_maybe_transform(
355 {
356 "data_source": data_source,
357 "metadata": metadata,
358 "name": name,
359 },
360 run_create_params.RunCreateParams,
361 ),
362 options=make_request_options(
363 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
364 ),
365 cast_to=RunCreateResponse,
366 )
367
368 async def retrieve(
369 self,
370 run_id: str,
371 *,
372 eval_id: str,
373 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
374 # The extra values given here take precedence over values defined on the client or passed to this method.
375 extra_headers: Headers | None = None,
376 extra_query: Query | None = None,
377 extra_body: Body | None = None,
378 timeout: float | httpx.Timeout | None | NotGiven = not_given,
379 ) -> RunRetrieveResponse:
380 """
381 Get an evaluation run by ID.
382
383 Args:
384 extra_headers: Send extra headers
385
386 extra_query: Add additional query parameters to the request
387
388 extra_body: Add additional JSON properties to the request
389
390 timeout: Override the client-level default timeout for this request, in seconds
391 """
392 if not eval_id:
393 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
394 if not run_id:
395 raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
396 return await self._get(
397 f"/evals/{eval_id}/runs/{run_id}",
398 options=make_request_options(
399 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
400 ),
401 cast_to=RunRetrieveResponse,
402 )
403
404 def list(
405 self,
406 eval_id: str,
407 *,
408 after: str | Omit = omit,
409 limit: int | Omit = omit,
410 order: Literal["asc", "desc"] | Omit = omit,
411 status: Literal["queued", "in_progress", "completed", "canceled", "failed"] | Omit = omit,
412 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
413 # The extra values given here take precedence over values defined on the client or passed to this method.
414 extra_headers: Headers | None = None,
415 extra_query: Query | None = None,
416 extra_body: Body | None = None,
417 timeout: float | httpx.Timeout | None | NotGiven = not_given,
418 ) -> AsyncPaginator[RunListResponse, AsyncCursorPage[RunListResponse]]:
419 """
420 Get a list of runs for an evaluation.
421
422 Args:
423 after: Identifier for the last run from the previous pagination request.
424
425 limit: Number of runs to retrieve.
426
427 order: Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for
428 descending order. Defaults to `asc`.
429
430 status: Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed`
431 | `canceled`.
432
433 extra_headers: Send extra headers
434
435 extra_query: Add additional query parameters to the request
436
437 extra_body: Add additional JSON properties to the request
438
439 timeout: Override the client-level default timeout for this request, in seconds
440 """
441 if not eval_id:
442 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
443 return self._get_api_list(
444 f"/evals/{eval_id}/runs",
445 page=AsyncCursorPage[RunListResponse],
446 options=make_request_options(
447 extra_headers=extra_headers,
448 extra_query=extra_query,
449 extra_body=extra_body,
450 timeout=timeout,
451 query=maybe_transform(
452 {
453 "after": after,
454 "limit": limit,
455 "order": order,
456 "status": status,
457 },
458 run_list_params.RunListParams,
459 ),
460 ),
461 model=RunListResponse,
462 )
463
464 async def delete(
465 self,
466 run_id: str,
467 *,
468 eval_id: str,
469 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
470 # The extra values given here take precedence over values defined on the client or passed to this method.
471 extra_headers: Headers | None = None,
472 extra_query: Query | None = None,
473 extra_body: Body | None = None,
474 timeout: float | httpx.Timeout | None | NotGiven = not_given,
475 ) -> RunDeleteResponse:
476 """
477 Delete an eval run.
478
479 Args:
480 extra_headers: Send extra headers
481
482 extra_query: Add additional query parameters to the request
483
484 extra_body: Add additional JSON properties to the request
485
486 timeout: Override the client-level default timeout for this request, in seconds
487 """
488 if not eval_id:
489 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
490 if not run_id:
491 raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
492 return await self._delete(
493 f"/evals/{eval_id}/runs/{run_id}",
494 options=make_request_options(
495 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
496 ),
497 cast_to=RunDeleteResponse,
498 )
499
500 async def cancel(
501 self,
502 run_id: str,
503 *,
504 eval_id: str,
505 # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
506 # The extra values given here take precedence over values defined on the client or passed to this method.
507 extra_headers: Headers | None = None,
508 extra_query: Query | None = None,
509 extra_body: Body | None = None,
510 timeout: float | httpx.Timeout | None | NotGiven = not_given,
511 ) -> RunCancelResponse:
512 """
513 Cancel an ongoing evaluation run.
514
515 Args:
516 extra_headers: Send extra headers
517
518 extra_query: Add additional query parameters to the request
519
520 extra_body: Add additional JSON properties to the request
521
522 timeout: Override the client-level default timeout for this request, in seconds
523 """
524 if not eval_id:
525 raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
526 if not run_id:
527 raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
528 return await self._post(
529 f"/evals/{eval_id}/runs/{run_id}",
530 options=make_request_options(
531 extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
532 ),
533 cast_to=RunCancelResponse,
534 )
535
536
537class RunsWithRawResponse:
538 def __init__(self, runs: Runs) -> None:
539 self._runs = runs
540
541 self.create = _legacy_response.to_raw_response_wrapper(
542 runs.create,
543 )
544 self.retrieve = _legacy_response.to_raw_response_wrapper(
545 runs.retrieve,
546 )
547 self.list = _legacy_response.to_raw_response_wrapper(
548 runs.list,
549 )
550 self.delete = _legacy_response.to_raw_response_wrapper(
551 runs.delete,
552 )
553 self.cancel = _legacy_response.to_raw_response_wrapper(
554 runs.cancel,
555 )
556
557 @cached_property
558 def output_items(self) -> OutputItemsWithRawResponse:
559 return OutputItemsWithRawResponse(self._runs.output_items)
560
561
562class AsyncRunsWithRawResponse:
563 def __init__(self, runs: AsyncRuns) -> None:
564 self._runs = runs
565
566 self.create = _legacy_response.async_to_raw_response_wrapper(
567 runs.create,
568 )
569 self.retrieve = _legacy_response.async_to_raw_response_wrapper(
570 runs.retrieve,
571 )
572 self.list = _legacy_response.async_to_raw_response_wrapper(
573 runs.list,
574 )
575 self.delete = _legacy_response.async_to_raw_response_wrapper(
576 runs.delete,
577 )
578 self.cancel = _legacy_response.async_to_raw_response_wrapper(
579 runs.cancel,
580 )
581
582 @cached_property
583 def output_items(self) -> AsyncOutputItemsWithRawResponse:
584 return AsyncOutputItemsWithRawResponse(self._runs.output_items)
585
586
587class RunsWithStreamingResponse:
588 def __init__(self, runs: Runs) -> None:
589 self._runs = runs
590
591 self.create = to_streamed_response_wrapper(
592 runs.create,
593 )
594 self.retrieve = to_streamed_response_wrapper(
595 runs.retrieve,
596 )
597 self.list = to_streamed_response_wrapper(
598 runs.list,
599 )
600 self.delete = to_streamed_response_wrapper(
601 runs.delete,
602 )
603 self.cancel = to_streamed_response_wrapper(
604 runs.cancel,
605 )
606
607 @cached_property
608 def output_items(self) -> OutputItemsWithStreamingResponse:
609 return OutputItemsWithStreamingResponse(self._runs.output_items)
610
611
612class AsyncRunsWithStreamingResponse:
613 def __init__(self, runs: AsyncRuns) -> None:
614 self._runs = runs
615
616 self.create = async_to_streamed_response_wrapper(
617 runs.create,
618 )
619 self.retrieve = async_to_streamed_response_wrapper(
620 runs.retrieve,
621 )
622 self.list = async_to_streamed_response_wrapper(
623 runs.list,
624 )
625 self.delete = async_to_streamed_response_wrapper(
626 runs.delete,
627 )
628 self.cancel = async_to_streamed_response_wrapper(
629 runs.cancel,
630 )
631
632 @cached_property
633 def output_items(self) -> AsyncOutputItemsWithStreamingResponse:
634 return AsyncOutputItemsWithStreamingResponse(self._runs.output_items)