1# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
  2
  3from typing import Dict, List, Union, Optional
  4from typing_extensions import Literal, Annotated, TypeAlias
  5
  6from pydantic import Field as FieldInfo
  7
  8from .._utils import PropertyInfo
  9from .._models import BaseModel
 10from .shared.metadata import Metadata
 11from .graders.python_grader import PythonGrader
 12from .graders.label_model_grader import LabelModelGrader
 13from .graders.score_model_grader import ScoreModelGrader
 14from .graders.string_check_grader import StringCheckGrader
 15from .eval_custom_data_source_config import EvalCustomDataSourceConfig
 16from .graders.text_similarity_grader import TextSimilarityGrader
 17from .eval_stored_completions_data_source_config import EvalStoredCompletionsDataSourceConfig
 18
 19__all__ = [
 20    "EvalRetrieveResponse",
 21    "DataSourceConfig",
 22    "DataSourceConfigLogs",
 23    "TestingCriterion",
 24    "TestingCriterionEvalGraderTextSimilarity",
 25    "TestingCriterionEvalGraderPython",
 26    "TestingCriterionEvalGraderScoreModel",
 27]
 28
 29
 30class DataSourceConfigLogs(BaseModel):
 31    schema_: Dict[str, object] = FieldInfo(alias="schema")
 32    """
 33    The json schema for the run data source items. Learn how to build JSON schemas
 34    [here](https://json-schema.org/).
 35    """
 36
 37    type: Literal["logs"]
 38    """The type of data source. Always `logs`."""
 39
 40    metadata: Optional[Metadata] = None
 41    """Set of 16 key-value pairs that can be attached to an object.
 42
 43    This can be useful for storing additional information about the object in a
 44    structured format, and querying for objects via API or the dashboard.
 45
 46    Keys are strings with a maximum length of 64 characters. Values are strings with
 47    a maximum length of 512 characters.
 48    """
 49
 50
 51DataSourceConfig: TypeAlias = Annotated[
 52    Union[EvalCustomDataSourceConfig, DataSourceConfigLogs, EvalStoredCompletionsDataSourceConfig],
 53    PropertyInfo(discriminator="type"),
 54]
 55
 56
 57class TestingCriterionEvalGraderTextSimilarity(TextSimilarityGrader):
 58    __test__ = False
 59    pass_threshold: float
 60    """The threshold for the score."""
 61
 62
 63class TestingCriterionEvalGraderPython(PythonGrader):
 64    __test__ = False
 65    pass_threshold: Optional[float] = None
 66    """The threshold for the score."""
 67
 68
 69class TestingCriterionEvalGraderScoreModel(ScoreModelGrader):
 70    __test__ = False
 71    pass_threshold: Optional[float] = None
 72    """The threshold for the score."""
 73
 74
 75TestingCriterion: TypeAlias = Union[
 76    LabelModelGrader,
 77    StringCheckGrader,
 78    TestingCriterionEvalGraderTextSimilarity,
 79    TestingCriterionEvalGraderPython,
 80    TestingCriterionEvalGraderScoreModel,
 81]
 82
 83
 84class EvalRetrieveResponse(BaseModel):
 85    id: str
 86    """Unique identifier for the evaluation."""
 87
 88    created_at: int
 89    """The Unix timestamp (in seconds) for when the eval was created."""
 90
 91    data_source_config: DataSourceConfig
 92    """Configuration of data sources used in runs of the evaluation."""
 93
 94    metadata: Optional[Metadata] = None
 95    """Set of 16 key-value pairs that can be attached to an object.
 96
 97    This can be useful for storing additional information about the object in a
 98    structured format, and querying for objects via API or the dashboard.
 99
100    Keys are strings with a maximum length of 64 characters. Values are strings with
101    a maximum length of 512 characters.
102    """
103
104    name: str
105    """The name of the evaluation."""
106
107    object: Literal["eval"]
108    """The object type."""
109
110    testing_criteria: List[TestingCriterion]
111    """A list of testing criteria."""