diff --git a/app/models/schemas.py b/app/models/schemas.py index 40e988e9..2cde5431 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -1630,8 +1630,9 @@ class RunEvaluatorsResponse(BaseModel): MetricScope = Literal["workspace", "organization"] # Max length for metric rubric text (description / example) accepted by -# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. -METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 +# the API (~1 MB of text). DB columns are TEXT or unbounded VARCHAR; this +# cap is validation-only. +METRIC_RUBRIC_TEXT_MAX_LENGTH = 1_000_000 @@ -1763,18 +1764,6 @@ class ChooseNextCombinationResponse(BaseModel): next_index: int -# Metric Schemas -SelectionMode = Literal["single_choice", "multi_label"] - - -MetricScope = Literal["workspace", "organization"] - -# Max length for metric rubric text (description / example) accepted by -# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. -METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 - - - class MetricCreate(BaseModel): """Schema for creating a metric. @@ -1793,7 +1782,7 @@ class MetricCreate(BaseModel): ignored server-side. """ name: str - description: Optional[str] = None + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) # Optional illustrative example surfaced alongside ``description`` # in the LLM judge's rubric. Today this is mainly populated on # child sub-labels (one example per categorization label) but diff --git a/tests/test_models/test_metric_rubric_max_length.py b/tests/test_models/test_metric_rubric_max_length.py index bfe82762..c90521cd 100644 --- a/tests/test_models/test_metric_rubric_max_length.py +++ b/tests/test_models/test_metric_rubric_max_length.py @@ -5,7 +5,9 @@ from app.models.schemas import ( METRIC_RUBRIC_TEXT_MAX_LENGTH, MetricChildDraft, + MetricCreate, MetricCreateWithChildren, + MetricUpdate, ) @@ -23,13 +25,49 @@ def test_metric_rubric_accepts_text_over_legacy_4000_limit(): assert len(payload.children[0].example) == 5000 +def test_metric_rubric_accepts_text_at_max_length(): + max_text = "x" * METRIC_RUBRIC_TEXT_MAX_LENGTH + + create_payload = MetricCreate( + name="Standalone", + description=max_text, + example=max_text, + ) + assert len(create_payload.description) == METRIC_RUBRIC_TEXT_MAX_LENGTH + assert len(create_payload.example) == METRIC_RUBRIC_TEXT_MAX_LENGTH + + with_children_payload = MetricCreateWithChildren( + name="Parent", + description=max_text, + selection_mode="single_choice", + children=[ + MetricChildDraft(name="Label1", description=max_text, example=max_text) + ], + ) + assert len(with_children_payload.description) == METRIC_RUBRIC_TEXT_MAX_LENGTH + assert len(with_children_payload.children[0].description) == METRIC_RUBRIC_TEXT_MAX_LENGTH + assert len(with_children_payload.children[0].example) == METRIC_RUBRIC_TEXT_MAX_LENGTH + + update_payload = MetricUpdate(description=max_text, example=max_text) + assert len(update_payload.description) == METRIC_RUBRIC_TEXT_MAX_LENGTH + assert len(update_payload.example) == METRIC_RUBRIC_TEXT_MAX_LENGTH + + def test_metric_rubric_rejects_text_over_max_length(): - try: - MetricCreateWithChildren( - name="Test", - description="x" * (METRIC_RUBRIC_TEXT_MAX_LENGTH + 1), - selection_mode="single_choice", - ) - except ValidationError: - return - raise AssertionError("expected ValidationError for over-limit description") + over_limit = "x" * (METRIC_RUBRIC_TEXT_MAX_LENGTH + 1) + + for factory, kwargs in ( + (MetricCreate, {"name": "Standalone", "description": over_limit}), + (MetricCreate, {"name": "Standalone", "example": over_limit}), + ( + MetricCreateWithChildren, + {"name": "Test", "description": over_limit, "selection_mode": "single_choice"}, + ), + (MetricUpdate, {"description": over_limit}), + (MetricUpdate, {"example": over_limit}), + ): + try: + factory(**kwargs) + except ValidationError: + continue + raise AssertionError(f"expected ValidationError for over-limit {factory.__name__}")