From ff9e485e4b2617e0fa54c238eae80a7d4d57213f Mon Sep 17 00:00:00 2001 From: Steve Messick Date: Wed, 21 Oct 2026 20:43:37 +0000 Subject: [PATCH] Release 0.1.38 --- .../agents/services/agents_api_service.py | 19 +- kagglesdk/agents/types/agent_service.py | 20 + .../services/benchmark_tasks_api_service.py | 39 +- .../services/benchmarks_api_service.py | 199 ++- kagglesdk/benchmarks/types/benchmark_enums.py | 140 +- .../benchmarks/types/benchmark_service.py | 59 - .../types/benchmark_task_run_service.py | 476 ++++- .../types/benchmark_tasks_api_service.py | 1508 ++++++++++++---- kagglesdk/benchmarks/types/benchmark_types.py | 1590 +++++++++++++++-- .../types/benchmarks_api_service.py | 1178 +++++++++++- .../services/competition_api_service.py | 14 +- .../types/competition_api_service.py | 113 +- kagglesdk/competitions/types/host_service.py | 60 +- kagglesdk/search/types/search_enums.py | 1 + kagglesdk/security/types/security_types.py | 1 + kagglesdk/users/types/account_service.py | 10 +- 16 files changed, 4810 insertions(+), 617 deletions(-) delete mode 100644 kagglesdk/benchmarks/types/benchmark_service.py diff --git a/kagglesdk/agents/services/agents_api_service.py b/kagglesdk/agents/services/agents_api_service.py index f7bfac3..e6c2ec6 100644 --- a/kagglesdk/agents/services/agents_api_service.py +++ b/kagglesdk/agents/services/agents_api_service.py @@ -9,7 +9,7 @@ def __init__(self, client: KaggleHttpClient): def create_harness(self, request: ApiCreateHarnessRequest = None) -> ApiHarness: r""" Create a Harness together with its (required) first HarnessVersion. - Admin-only. + Requires the agentic-candidates feature flag. Example: curl -sSL -u andrewmingwang:local_api_token \ @@ -45,7 +45,7 @@ def create_harness(self, request: ApiCreateHarnessRequest = None) -> ApiHarness: def get_harness(self, request: ApiGetHarnessRequest = None) -> ApiHarness: r""" - Get a Harness by id. Admin-only. + Get a Harness by id. Requires the agentic-candidates feature flag. Example: curl -sSL -u andrewmingwang:local_api_token \ @@ -64,7 +64,7 @@ def get_harness(self, request: ApiGetHarnessRequest = None) -> ApiHarness: def create_harness_version(self, request: ApiCreateHarnessVersionRequest = None) -> ApiHarnessVersion: r""" Create an additional HarnessVersion under an existing Harness. - Admin-only. + Requires the agentic-candidates feature flag. Example: curl -sSL -u andrewmingwang:local_api_token \ @@ -95,7 +95,7 @@ def create_harness_version(self, request: ApiCreateHarnessVersionRequest = None) def get_harness_version(self, request: ApiGetHarnessVersionRequest = None) -> ApiHarnessVersion: r""" - Get a HarnessVersion by id. Admin-only. + Get a HarnessVersion by id. Requires the agentic-candidates feature flag. Example: curl -sSL -u andrewmingwang:local_api_token \ @@ -115,7 +115,8 @@ def list_harnesses(self, request: ApiListHarnessesRequest = None) -> ApiListHarn r""" List Harnesses. Returns one row per HarnessVersion visible to the caller (a Harness with N versions appears N times, each row with version - populated for that version). Admin-only. + populated for that version). Requires the agentic-candidates feature + flag. Example: curl -sSL -u andrewmingwang:local_api_token -G \ @@ -135,7 +136,8 @@ def list_harnesses(self, request: ApiListHarnessesRequest = None) -> ApiListHarn def create_agent(self, request: ApiCreateAgentRequest = None) -> ApiAgent: r""" Create an Agent (pairing of BenchmarkModelVersionConfig + HarnessVersion). - Admin-only, plus the caller must be able to read the referenced + Requires the agentic-candidates feature flag, plus the caller must be + able to read the referenced BenchmarkModelVersionConfig and HarnessVersion. Example: @@ -164,7 +166,7 @@ def create_agent(self, request: ApiCreateAgentRequest = None) -> ApiAgent: def get_agent(self, request: ApiGetAgentRequest = None) -> ApiAgent: r""" - Get an Agent by id. Admin-only. + Get an Agent by id. Requires the agentic-candidates feature flag. Example: curl -sSL -u andrewmingwang:local_api_token \ @@ -182,7 +184,8 @@ def get_agent(self, request: ApiGetAgentRequest = None) -> ApiAgent: def list_agents(self, request: ApiListAgentsRequest = None) -> ApiListAgentsResponse: r""" - List Agents visible to the caller. Admin-only. + List Agents visible to the caller. Requires the agentic-candidates + feature flag. Example: curl -sSL -u andrewmingwang:local_api_token -G \ diff --git a/kagglesdk/agents/types/agent_service.py b/kagglesdk/agents/types/agent_service.py index b27e8e3..453fd11 100644 --- a/kagglesdk/agents/types/agent_service.py +++ b/kagglesdk/agents/types/agent_service.py @@ -10,12 +10,15 @@ class ListAgentsFilter(KaggleObject): Restrict to Agents whose BenchmarkModelVersionConfig has the given ids. harness_version_ids (int) Restrict to Agents whose HarnessVersion has the given ids. + slugs (str) + Restrict to Agents with the given slugs. """ def __init__(self): self._agent_ids = [] self._benchmark_model_version_config_ids = [] self._harness_version_ids = [] + self._slugs = [] self._freeze() @property @@ -66,6 +69,22 @@ def harness_version_ids(self, harness_version_ids: Optional[List[int]]): raise TypeError('harness_version_ids must contain only items of type int') self._harness_version_ids = harness_version_ids + @property + def slugs(self) -> Optional[List[str]]: + """Restrict to Agents with the given slugs.""" + return self._slugs + + @slugs.setter + def slugs(self, slugs: Optional[List[str]]): + if slugs is None: + del self.slugs + return + if not isinstance(slugs, list): + raise TypeError('slugs must be of type list') + if not all([isinstance(t, str) for t in slugs]): + raise TypeError('slugs must contain only items of type str') + self._slugs = slugs + class ListHarnessesFilter(KaggleObject): r""" @@ -122,6 +141,7 @@ def latest_version_per_harness_only(self, latest_version_per_harness_only: Optio FieldMetadata("agentIds", "agent_ids", "_agent_ids", int, [], ListSerializer(PredefinedSerializer())), FieldMetadata("benchmarkModelVersionConfigIds", "benchmark_model_version_config_ids", "_benchmark_model_version_config_ids", int, [], ListSerializer(PredefinedSerializer())), FieldMetadata("harnessVersionIds", "harness_version_ids", "_harness_version_ids", int, [], ListSerializer(PredefinedSerializer())), + FieldMetadata("slugs", "slugs", "_slugs", str, [], ListSerializer(PredefinedSerializer())), ] ListHarnessesFilter._fields = [ diff --git a/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py b/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py index 33ed5e8..c8cc245 100644 --- a/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py +++ b/kagglesdk/benchmarks/services/benchmark_tasks_api_service.py @@ -1,4 +1,5 @@ -from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBatchScheduleBenchmarkTaskRunsRequest, ApiBatchScheduleBenchmarkTaskRunsResponse, ApiBenchmarkTask, ApiCreateBenchmarkTaskRequest, ApiDownloadBenchmarkTaskRunOutputRequest, ApiGetBenchmarkTaskQuotaRequest, ApiGetBenchmarkTaskQuotaResponse, ApiGetBenchmarkTaskRequest, ApiGetBenchmarkTaskRunLogsRequest, ApiListBenchmarkTaskRunsRequest, ApiListBenchmarkTaskRunsResponse, ApiListBenchmarkTasksRequest, ApiListBenchmarkTasksResponse, ApiPublishBenchmarkTaskRequest, ApiUpdateBenchmarkTaskRequest, ApiUpdateBenchmarkTaskResponse, ApiUpdateBenchmarkTaskVersionRequest +from kagglesdk.benchmarks.types.benchmark_task_run_service import GetBenchmarkSchedulerStatusResponse +from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBatchScheduleBenchmarkTaskRunsRequest, ApiBatchScheduleBenchmarkTaskRunsResponse, ApiBenchmarkTask, ApiCancelBenchmarkTaskRunsRequest, ApiCancelBenchmarkTaskRunsResponse, ApiCreateBenchmarkTaskRequest, ApiDownloadBenchmarkTaskRunOutputRequest, ApiGetBenchmarkSchedulerStatusRequest, ApiGetBenchmarkTaskQuotaRequest, ApiGetBenchmarkTaskQuotaResponse, ApiGetBenchmarkTaskRequest, ApiGetBenchmarkTaskRunLogsRequest, ApiListBenchmarkTaskRunsRequest, ApiListBenchmarkTaskRunsResponse, ApiListBenchmarkTasksRequest, ApiListBenchmarkTasksResponse, ApiPublishBenchmarkTaskRequest, ApiUpdateBenchmarkTaskRequest, ApiUpdateBenchmarkTaskResponse, ApiUpdateBenchmarkTaskVersionRequest from kagglesdk.common.types.file_download import FileDownload from kagglesdk.kaggle_http_client import KaggleHttpClient @@ -37,6 +38,22 @@ def batch_schedule_benchmark_task_runs(self, request: ApiBatchScheduleBenchmarkT return self._client.call("benchmarks.BenchmarkTasksApiService", "BatchScheduleBenchmarkTaskRuns", request, ApiBatchScheduleBenchmarkTaskRunsResponse) + def api_cancel_benchmark_task_runs(self, request: ApiCancelBenchmarkTaskRunsRequest = None) -> ApiCancelBenchmarkTaskRunsResponse: + r""" + Cancels the queued (and optionally the running) runs of a set of tasks, + optionally narrowed to one candidate. Only ever cancels the caller's own + runs; the flag-gated fleet-wide scope is only offered by the internal RPC. + + Args: + request (ApiCancelBenchmarkTaskRunsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiCancelBenchmarkTaskRunsRequest() + + return self._client.call("benchmarks.BenchmarkTasksApiService", "ApiCancelBenchmarkTaskRuns", request, ApiCancelBenchmarkTaskRunsResponse) + def list_benchmark_tasks(self, request: ApiListBenchmarkTasksRequest = None) -> ApiListBenchmarkTasksResponse: r""" List tasks for the current user. @@ -127,10 +144,28 @@ def get_benchmark_task_quota(self, request: ApiGetBenchmarkTaskQuotaRequest = No return self._client.call("benchmarks.BenchmarkTasksApiService", "GetBenchmarkTaskQuota", request, ApiGetBenchmarkTaskQuotaResponse) + def api_get_benchmark_scheduler_status(self, request: ApiGetBenchmarkSchedulerStatusRequest = None) -> GetBenchmarkSchedulerStatusResponse: + r""" + Scheduler readout covering the calling user's own enqueued runs: queue + depth, recent outcomes, and stats per candidate model and per benchmark. + Always self-scoped; the flag-gated fleet-wide view is only offered by the + internal RPC. + + Args: + request (ApiGetBenchmarkSchedulerStatusRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiGetBenchmarkSchedulerStatusRequest() + + return self._client.call("benchmarks.BenchmarkTasksApiService", "ApiGetBenchmarkSchedulerStatus", request, GetBenchmarkSchedulerStatusResponse) + def publish_benchmark_task(self, request: ApiPublishBenchmarkTaskRequest = None) -> ApiBenchmarkTask: r""" Publish a particular task for a user. Optionally publishes the backing - notebook (`source_kernel_id`) in the same request. + notebook (`source_kernel_id`) and the task's run artifacts in the same + request. Args: request (ApiPublishBenchmarkTaskRequest): diff --git a/kagglesdk/benchmarks/services/benchmarks_api_service.py b/kagglesdk/benchmarks/services/benchmarks_api_service.py index 438512c..040215f 100644 --- a/kagglesdk/benchmarks/services/benchmarks_api_service.py +++ b/kagglesdk/benchmarks/services/benchmarks_api_service.py @@ -1,4 +1,5 @@ -from kagglesdk.benchmarks.types.benchmarks_api_service import ApiAddTasksToBenchmarkRequest, ApiAddTasksToBenchmarkResponse, ApiBenchmarkLeaderboard, ApiBenchmarkModelVersionConfig, ApiCreateBenchmarkModelVersionConfigRequest, ApiCreateBenchmarkVersionAgentMappingsRequest, ApiCreateBenchmarkVersionAgentMappingsResponse, ApiDeleteBenchmarkVersionAgentMappingsRequest, ApiDeleteBenchmarkVersionAgentMappingsResponse, ApiGetBenchmarkLeaderboardRequest, ApiGetBenchmarkModelVersionConfigRequest, ApiListBenchmarkModelsRequest, ApiListBenchmarkModelsResponse, ApiListBenchmarkModelVersionConfigsRequest, ApiListBenchmarkModelVersionConfigsResponse, ApiListBenchmarkVersionAgentMappingsRequest, ApiListBenchmarkVersionAgentMappingsResponse +from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaSettings +from kagglesdk.benchmarks.types.benchmarks_api_service import ApiAddTasksToBenchmarkRequest, ApiAddTasksToBenchmarkResponse, ApiBenchmarkLeaderboard, ApiBenchmarkModelVersionConfig, ApiCreateBenchmarkArenaSettingsRequest, ApiCreateBenchmarkModelVersionConfigRequest, ApiCreateBenchmarkVersionAgentMappingsRequest, ApiCreateBenchmarkVersionAgentMappingsResponse, ApiDeleteBenchmarkVersionAgentMappingsRequest, ApiDeleteBenchmarkVersionAgentMappingsResponse, ApiGetBenchmarkArenaSettingsRequest, ApiGetBenchmarkLeaderboardRequest, ApiGetBenchmarkModelVersionConfigRequest, ApiListBenchmarkArenaCombinationsRequest, ApiListBenchmarkArenaCombinationsResponse, ApiListBenchmarkModelsRequest, ApiListBenchmarkModelsResponse, ApiListBenchmarkModelVersionConfigsRequest, ApiListBenchmarkModelVersionConfigsResponse, ApiListBenchmarkVersionAgentMappingsRequest, ApiListBenchmarkVersionAgentMappingsResponse, ApiRegisterAuxiliaryModelVersionConfigsRequest, ApiRemoveTasksFromBenchmarkRequest, ApiRemoveTasksFromBenchmarkResponse, ApiUnregisterAuxiliaryModelVersionConfigsRequest, ApiUpdateBenchmarkArenaCombinationsRequest, ApiUpdateBenchmarkArenaSettingsRequest from kagglesdk.kaggle_http_client import KaggleHttpClient class BenchmarksApiClient(object): @@ -189,11 +190,10 @@ def add_tasks_to_benchmark(self, request: ApiAddTasksToBenchmarkRequest = None) r""" Map one or more BenchmarkTasks into one of the current user's own benchmarks, making them part of that benchmark's task tree and - leaderboard. Only supported for benchmarks of type Personal. Requires - update access to the benchmark version and read access to each task. Tasks - already mapped to the benchmark are skipped and echoed back in - `skipped_tasks` rather than failing the request, so a partially-applied - bulk import can be safely re-run. + leaderboard. Requires update access to the benchmark version and read + access to each task. Tasks already mapped to the benchmark are skipped and + echoed back in `skipped_tasks` rather than failing the request, so a + partially-applied bulk import can be safely re-run. Example: curl -sSL -u fickleone:local_api_token \ @@ -216,3 +216,190 @@ def add_tasks_to_benchmark(self, request: ApiAddTasksToBenchmarkRequest = None) request = ApiAddTasksToBenchmarkRequest() return self._client.call("benchmarks.BenchmarksApiService", "AddTasksToBenchmark", request, ApiAddTasksToBenchmarkResponse) + + def remove_tasks_from_benchmark(self, request: ApiRemoveTasksFromBenchmarkRequest = None) -> ApiRemoveTasksFromBenchmarkResponse: + r""" + Unmap one or more BenchmarkTasks from one of the current user's own + benchmarks, removing them from that benchmark's task tree and leaderboard. + Requires update access to the benchmark version and read access to each + task. Tasks not currently mapped to the benchmark are skipped and echoed + back in `skipped_tasks` rather than failing the request, so a + partially-applied bulk removal can be safely re-run. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/my-benchmark/tasks/remove \ + -H 'Content-Type: application/json' \ + -d '{ + 'taskSlugs': [ + {'taskSlug': 'count-letters'}, + {'ownerSlug': 'bob', 'taskSlug': 'sort-list', 'versionNumber': 2} + ] + }' + + Args: + request (ApiRemoveTasksFromBenchmarkRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiRemoveTasksFromBenchmarkRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "RemoveTasksFromBenchmark", request, ApiRemoveTasksFromBenchmarkResponse) + + def register_auxiliary_model_version_configs(self, request: ApiRegisterAuxiliaryModelVersionConfigsRequest = None): + r""" + Retroactively records auxiliary BenchmarkModelVersionConfigs a task version + invokes beyond the candidate (e.g. LLM judges), for authors who did not + declare them at creation time. The task is resolved under the current user; + configs are addressed by slug. Additive and idempotent: configs already + registered on the version are skipped. + + Args: + request (ApiRegisterAuxiliaryModelVersionConfigsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiRegisterAuxiliaryModelVersionConfigsRequest() + + self._client.call("benchmarks.BenchmarksApiService", "RegisterAuxiliaryModelVersionConfigs", request, None) + + def unregister_auxiliary_model_version_configs(self, request: ApiUnregisterAuxiliaryModelVersionConfigsRequest = None): + r""" + Removes previously-registered auxiliary BenchmarkModelVersionConfigs from a + task version. The task is resolved under the current user; configs are + addressed by slug. Idempotent: configs not currently registered are + ignored. + + Args: + request (ApiUnregisterAuxiliaryModelVersionConfigsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiUnregisterAuxiliaryModelVersionConfigsRequest() + + self._client.call("benchmarks.BenchmarksApiService", "UnregisterAuxiliaryModelVersionConfigs", request, None) + + def get_benchmark_arena_settings(self, request: ApiGetBenchmarkArenaSettingsRequest = None) -> BenchmarkArenaSettings: + r""" + Get a benchmark version's arena settings. Requires update access to the + benchmark version. Returns NOT_FOUND if the version has no arena. + + Example: + curl -sSL -u fickleone:local_api_token \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/settings + + Args: + request (ApiGetBenchmarkArenaSettingsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiGetBenchmarkArenaSettingsRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "GetBenchmarkArenaSettings", request, BenchmarkArenaSettings) + + def create_benchmark_arena_settings(self, request: ApiCreateBenchmarkArenaSettingsRequest = None) -> BenchmarkArenaSettings: + r""" + Create a benchmark version's arena. Requires update access to the + benchmark version. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/settings/create + \ + -H 'Content-Type: application/json' \ + -d '{ + 'settings': { + 'artifactPath': '*.mp4', + 'artifactType': 'BENCHMARK_ARENA_ARTIFACT_TYPE_VIDEO', + 'revealCandidates': true + } + }' + + Args: + request (ApiCreateBenchmarkArenaSettingsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiCreateBenchmarkArenaSettingsRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "CreateBenchmarkArenaSettings", request, BenchmarkArenaSettings) + + def update_benchmark_arena_settings(self, request: ApiUpdateBenchmarkArenaSettingsRequest = None) -> BenchmarkArenaSettings: + r""" + Update a benchmark version's arena settings. Requires update access to the + benchmark version. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/settings/update + \ + -H 'Content-Type: application/json' \ + -d '{ + 'settings': {'votingEnabled': true}, + 'updateMask': 'votingEnabled' + }' + + Args: + request (ApiUpdateBenchmarkArenaSettingsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiUpdateBenchmarkArenaSettingsRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "UpdateBenchmarkArenaSettings", request, BenchmarkArenaSettings) + + def list_benchmark_arena_combinations(self, request: ApiListBenchmarkArenaCombinationsRequest = None) -> ApiListBenchmarkArenaCombinationsResponse: + r""" + List the (task version, candidate) combinations opted into a benchmark + version's arena. Requires update access to the benchmark version. + + Example: + curl -sSL -u fickleone:local_api_token \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/combinations + + Args: + request (ApiListBenchmarkArenaCombinationsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiListBenchmarkArenaCombinationsRequest() + + return self._client.call("benchmarks.BenchmarksApiService", "ListBenchmarkArenaCombinations", request, ApiListBenchmarkArenaCombinationsResponse) + + def update_benchmark_arena_combinations(self, request: ApiUpdateBenchmarkArenaCombinationsRequest = None): + r""" + Opt (task version, candidate) combinations into or out of a benchmark + version's arena. Requires update access to the benchmark version and read + access to each candidate added. + + Example: + curl -sSL -u fickleone:local_api_token \ + -X POST \ + http://localhost/api/v1/benchmarks/fickleone/my-benchmark/arena/combinations/update + \ + -H 'Content-Type: application/json' \ + -d '{ + 'add': [ + {'taskVersionId': 1, 'candidate': {'benchmarkModelVersionId': 2}} + ] + }' + + Args: + request (ApiUpdateBenchmarkArenaCombinationsRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiUpdateBenchmarkArenaCombinationsRequest() + + self._client.call("benchmarks.BenchmarksApiService", "UpdateBenchmarkArenaCombinations", request, None) diff --git a/kagglesdk/benchmarks/types/benchmark_enums.py b/kagglesdk/benchmarks/types/benchmark_enums.py index f6edd16..a588c2f 100644 --- a/kagglesdk/benchmarks/types/benchmark_enums.py +++ b/kagglesdk/benchmarks/types/benchmark_enums.py @@ -1,5 +1,14 @@ import enum +class BenchmarkArenaArtifactType(enum.Enum): + """How an arena renders a run's artifact.""" + BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED = 0 + BENCHMARK_ARENA_ARTIFACT_TYPE_VIDEO = 1 + BENCHMARK_ARENA_ARTIFACT_TYPE_IMAGE = 2 + BENCHMARK_ARENA_ARTIFACT_TYPE_HTML = 3 + """Rendered in a sandboxed iframe.""" + BENCHMARK_ARENA_ARTIFACT_TYPE_TEXT = 4 + class BenchmarkCandidateType(enum.Enum): r""" Saved to the DB. Do not modify existing values. @@ -8,6 +17,39 @@ class BenchmarkCandidateType(enum.Enum): BENCHMARK_CANDIDATE_TYPE_UNSPECIFIED = 0 BENCHMARK_CANDIDATE_TYPE_MODEL_VERSIONS = 1 BENCHMARK_CANDIDATE_TYPE_AGENTS = 2 + BENCHMARK_CANDIDATE_TYPE_MODEL_VERSION_CONFIGS = 3 + r""" + TODO(bml): Support this candidate type as part of migrating off of + ModelVersions. In the meantime, DO NOT PERSIST THIS VALUE IN THE DB + """ + +class BenchmarkEnqueueSkipPolicy(enum.Enum): + r""" + Which (task version, candidate) pairs an enqueue queues, widening in the + order below. All but ALL skip pairs whose run is still in flight. + """ + BENCHMARK_ENQUEUE_SKIP_POLICY_UNSPECIFIED = 0 + BENCHMARK_ENQUEUE_SKIP_POLICY_ALL = 1 + """Queue every pair, existing runs included.""" + BENCHMARK_ENQUEUE_SKIP_POLICY_MISSING_ONLY = 2 + """Queue only pairs with no run at all.""" + BENCHMARK_ENQUEUE_SKIP_POLICY_MISSING_OR_ERRORED = 3 + """... plus pairs whose latest run errored.""" + BENCHMARK_ENQUEUE_SKIP_POLICY_STALER_THAN = 4 + r""" + ... plus pairs whose run completed before stale_after. A widening rather + than a stale-only filter: refreshing a sweep fills its gaps too. + """ + +class BenchmarkGalleryCardSize(enum.Enum): + """How large the gallery draws each card.""" + BENCHMARK_GALLERY_CARD_SIZE_UNSPECIFIED = 0 + """Same as MEDIUM.""" + BENCHMARK_GALLERY_CARD_SIZE_EXTRA_SMALL = 1 + BENCHMARK_GALLERY_CARD_SIZE_SMALL = 2 + BENCHMARK_GALLERY_CARD_SIZE_MEDIUM = 3 + BENCHMARK_GALLERY_CARD_SIZE_LARGE = 4 + """One card per row, for outputs such as web pages that need room.""" class BenchmarkModelImportanceLevel(enum.Enum): r""" @@ -28,12 +70,23 @@ class BenchmarkTaskVersionCreationState(enum.Enum): LINT.IfChange(BenchmarkTaskVersionCreationState) """ BENCHMARK_TASK_VERSION_CREATION_STATE_UNSPECIFIED = 0 + BENCHMARK_TASK_VERSION_CREATION_STATE_VALIDATING = 8 + """Validation stage runs are in progress.""" + BENCHMARK_TASK_VERSION_CREATION_STATE_VALIDATION_FAILED = 6 + r""" + The created task failed validation, e.g. a validation stage run's output + didn't match its expected output. + """ + BENCHMARK_TASK_VERSION_CREATION_STATE_ERRORED = 4 + """The creation session, or any validation stage run, errored.""" + BENCHMARK_TASK_VERSION_CREATION_STATE_COMPLETED = 3 BENCHMARK_TASK_VERSION_CREATION_STATE_QUEUED = 1 + r""" + Deprecated: new task versions shouldn't use these states. Retained because + existing rows still hold them. + """ BENCHMARK_TASK_VERSION_CREATION_STATE_RUNNING = 2 - BENCHMARK_TASK_VERSION_CREATION_STATE_COMPLETED = 3 - BENCHMARK_TASK_VERSION_CREATION_STATE_ERRORED = 4 BENCHMARK_TASK_VERSION_CREATION_STATE_KERNEL_WITHOUT_RUN = 5 - BENCHMARK_TASK_VERSION_CREATION_STATE_VALIDATION_FAILED = 6 BENCHMARK_TASK_VERSION_CREATION_STATE_NO_MODEL_SPECIFIED = 7 class BenchmarkTaskVersionSource(enum.Enum): @@ -62,6 +115,87 @@ class Modality(enum.Enum): MODALITY_VIDEO = 3 MODALITY_AUDIO = 4 +class BenchmarkCandidateClass(enum.Enum): + r""" + The kind of contestant a benchmark candidate is, derived from which of a + BenchmarkRun's candidate ID columns are populated, or from which mapping + table selected a candidate onto a leaderboard. + + Distinct from BenchmarkCandidateType, which is a declared column on the task + version and is stamped inconsistently across write paths: a bulk-import path + stamps MODEL_VERSIONS on runs that carry an AgentId. This enum is always + derived, never stored, so it cannot drift the same way. + + What kind of contestant a leaderboard row or a run belongs to. The classes + are disjoint predicates over the same columns and together cover every run, + so every candidate falls into exactly one. Rows of different classes rank + together on one board, which is why 'Opus' and 'Claude Code on Opus' can + appear as separate results rather than one absorbing the other's runs. + """ + BENCHMARK_CANDIDATE_CLASS_UNSPECIFIED = 0 + BENCHMARK_CANDIDATE_CLASS_MODEL_VERSION = 1 + r""" + A bare model version: neither a config nor an agent. Also covers legacy + rows written before the configs table existed. + """ + BENCHMARK_CANDIDATE_CLASS_MODEL_VERSION_CONFIG = 2 + """A model version pinned to a config (e.g. a reasoning effort), no harness.""" + BENCHMARK_CANDIDATE_CLASS_AGENT = 3 + r""" + A harness paired with a config. Identified by the agent alone, since an + agent pins its own config and model version. + """ + +class BenchmarkLeaderboardDisplayType(enum.Enum): + r""" + Saved to the DB. Do not modify existing values. + This enum specifies how task version numeric results should be interpreted + and displayed on leaderboards. + LINT.IfChange + """ + BENCHMARK_LEADERBOARD_DISPLAY_TYPE_UNSPECIFIED = 0 + PERCENTAGES = 1 + r""" + Displays 0.XX values as a percentage, rounded to 1 sigfig, ex: + 0.12345 -> 12.3%, 45600.0 -> 4560000.0%, 0.75 -> 75.0% + """ + COUNTS = 2 + r""" + Displays value as an absolute count, rounded to nearest integer, ex: + 123, 456, 790 + """ + DOLLARS = 3 + r""" + Displays value as a dollar and cents amount, rounded to 2 sigfigs, ex: + $123.45, $456.00, $790.57 + """ + CENTS = 4 + r""" + Displays value as a cents amount, rounded to 2 sigfigs, ex: + ¢123.45, ¢456.00, ¢790.57 + """ + RAW_PERCENTAGES = 5 + r""" + Displays XX values as a percentage, rounded to 1 sigfig, ex: + 75 -> 75.0%, 123 -> 123.0%, 4.56 -> 4.6% + """ + FLOAT_COUNTS = 6 + r""" + Displays value as an absolute count, rounded to 1 sigfig, ex: + 123.4, 456.0, 789.6 + """ + RAW_VALUES = 7 + r""" + Displays the value with 2 decimal places, with no scaling or unit, ex: + 123.44, 456.00, 789.57 + """ + WHOLE_DOLLARS = 8 + r""" + Displays value as a dollar amount rounded to the nearest whole dollar with + thousands separators, ex: + $2,601,845, $456, $790 + """ + class BenchmarkTaskRunState(enum.Enum): BENCHMARK_TASK_RUN_STATE_UNSPECIFIED = 0 BENCHMARK_TASK_RUN_STATE_QUEUED = 1 diff --git a/kagglesdk/benchmarks/types/benchmark_service.py b/kagglesdk/benchmarks/types/benchmark_service.py deleted file mode 100644 index 2abb59f..0000000 --- a/kagglesdk/benchmarks/types/benchmark_service.py +++ /dev/null @@ -1,59 +0,0 @@ -from kagglesdk.kaggle_object import * -from typing import List, Optional - -class ListBenchmarkVersionAgentMappingsFilter(KaggleObject): - r""" - Attributes: - parent_benchmark_version_ids (int) - If set, only return mappings whose parent BenchmarkVersionId is in this - list. - child_agent_ids (int) - If set, only return mappings whose child AgentId is in this list. - """ - - def __init__(self): - self._parent_benchmark_version_ids = [] - self._child_agent_ids = [] - self._freeze() - - @property - def parent_benchmark_version_ids(self) -> Optional[List[int]]: - r""" - If set, only return mappings whose parent BenchmarkVersionId is in this - list. - """ - return self._parent_benchmark_version_ids - - @parent_benchmark_version_ids.setter - def parent_benchmark_version_ids(self, parent_benchmark_version_ids: Optional[List[int]]): - if parent_benchmark_version_ids is None: - del self.parent_benchmark_version_ids - return - if not isinstance(parent_benchmark_version_ids, list): - raise TypeError('parent_benchmark_version_ids must be of type list') - if not all([isinstance(t, int) for t in parent_benchmark_version_ids]): - raise TypeError('parent_benchmark_version_ids must contain only items of type int') - self._parent_benchmark_version_ids = parent_benchmark_version_ids - - @property - def child_agent_ids(self) -> Optional[List[int]]: - """If set, only return mappings whose child AgentId is in this list.""" - return self._child_agent_ids - - @child_agent_ids.setter - def child_agent_ids(self, child_agent_ids: Optional[List[int]]): - if child_agent_ids is None: - del self.child_agent_ids - return - if not isinstance(child_agent_ids, list): - raise TypeError('child_agent_ids must be of type list') - if not all([isinstance(t, int) for t in child_agent_ids]): - raise TypeError('child_agent_ids must contain only items of type int') - self._child_agent_ids = child_agent_ids - - -ListBenchmarkVersionAgentMappingsFilter._fields = [ - FieldMetadata("parentBenchmarkVersionIds", "parent_benchmark_version_ids", "_parent_benchmark_version_ids", int, [], ListSerializer(PredefinedSerializer())), - FieldMetadata("childAgentIds", "child_agent_ids", "_child_agent_ids", int, [], ListSerializer(PredefinedSerializer())), -] - diff --git a/kagglesdk/benchmarks/types/benchmark_task_run_service.py b/kagglesdk/benchmarks/types/benchmark_task_run_service.py index 49cebc9..35ce498 100644 --- a/kagglesdk/benchmarks/types/benchmark_task_run_service.py +++ b/kagglesdk/benchmarks/types/benchmark_task_run_service.py @@ -1,6 +1,7 @@ -from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkCandidateType +from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkCandidateType, BenchmarkTaskRunState +from kagglesdk.benchmarks.types.benchmark_types import BenchmarkSchedulerSettings from kagglesdk.kaggle_object import * -from typing import Optional +from typing import Optional, List class BatchScheduleBenchmarkModelVersionResult(KaggleObject): r""" @@ -18,14 +19,22 @@ class BatchScheduleBenchmarkModelVersionResult(KaggleObject): One of the values provided in BatchScheduleBenchmarkTaskRunsRequest.benchmark_task_version_ids parent_task_version_id (int) - When the requested benchmark_task_version_id is a child of another task - version, scheduling is redirected to the parent and this field reports the - parent's id. Unset when the requested task version was executed directly. + If the request task version was a leaf, this field indicates the parent + task version that should be executed instead. agent_id (int) One of the values provided in BatchScheduleBenchmarkTaskRunsRequest.benchmark_agent_ids candidate_type (BenchmarkCandidateType) Specifies the candidate type that was scheduled (ModelVersion vs. Agent) + benchmark_model_version_config_id (int) + The BenchmarkModelVersionConfig the run was scheduled against, resolved + from benchmark_model_version_id plus the request's reasoning_effort. Unset + when the request named no reasoning effort. + TODO(bml): eventually deprecate benchmark_model_version_id in favor of + this field, since the config identifies the model version too. + run_id (int) + The id of the BenchmarkRun that was scheduled. Set only when run_scheduled + is true. """ def __init__(self): @@ -36,6 +45,8 @@ def __init__(self): self._parent_task_version_id = None self._agent_id = 0 self._candidate_type = BenchmarkCandidateType.BENCHMARK_CANDIDATE_TYPE_UNSPECIFIED + self._benchmark_model_version_config_id = None + self._run_id = None self._freeze() @property @@ -89,6 +100,26 @@ def agent_id(self, agent_id: int): raise TypeError('agent_id must be of type int') self._agent_id = agent_id + @property + def benchmark_model_version_config_id(self) -> int: + r""" + The BenchmarkModelVersionConfig the run was scheduled against, resolved + from benchmark_model_version_id plus the request's reasoning_effort. Unset + when the request named no reasoning effort. + TODO(bml): eventually deprecate benchmark_model_version_id in favor of + this field, since the config identifies the model version too. + """ + return self._benchmark_model_version_config_id or 0 + + @benchmark_model_version_config_id.setter + def benchmark_model_version_config_id(self, benchmark_model_version_config_id: Optional[int]): + if benchmark_model_version_config_id is None: + del self.benchmark_model_version_config_id + return + if not isinstance(benchmark_model_version_config_id, int): + raise TypeError('benchmark_model_version_config_id must be of type int') + self._benchmark_model_version_config_id = benchmark_model_version_config_id + @property def candidate_type(self) -> 'BenchmarkCandidateType': """Specifies the candidate type that was scheduled (ModelVersion vs. Agent)""" @@ -120,6 +151,23 @@ def run_scheduled(self, run_scheduled: bool): raise TypeError('run_scheduled must be of type bool') self._run_scheduled = run_scheduled + @property + def run_id(self) -> int: + r""" + The id of the BenchmarkRun that was scheduled. Set only when run_scheduled + is true. + """ + return self._run_id or 0 + + @run_id.setter + def run_id(self, run_id: Optional[int]): + if run_id is None: + del self.run_id + return + if not isinstance(run_id, int): + raise TypeError('run_id must be of type int') + self._run_id = run_id + @property def run_skipped_reason(self) -> str: r""" @@ -140,9 +188,8 @@ def run_skipped_reason(self, run_skipped_reason: Optional[str]): @property def parent_task_version_id(self) -> int: r""" - When the requested benchmark_task_version_id is a child of another task - version, scheduling is redirected to the parent and this field reports the - parent's id. Unset when the requested task version was executed directly. + If the request task version was a leaf, this field indicates the parent + task version that should be executed instead. """ return self._parent_task_version_id or 0 @@ -156,6 +203,388 @@ def parent_task_version_id(self, parent_task_version_id: Optional[int]): self._parent_task_version_id = parent_task_version_id +class BenchmarkSchedulerBenchmarkStats(KaggleObject): + r""" + Attributes: + benchmark_id (int) + Unset for the row grouping runs of unmapped tasks, and for the withheld + row below. + benchmark_name (str) + withheld (bool) + Groups runs on benchmarks the caller can't read, so their counts still add + up without disclosing which benchmarks they are. + queued_count (int) + running_count (int) + recent_state_counts (BenchmarkSchedulerStateCount) + """ + + def __init__(self): + self._benchmark_id = None + self._benchmark_name = "" + self._withheld = False + self._queued_count = 0 + self._running_count = 0 + self._recent_state_counts = [] + self._freeze() + + @property + def benchmark_id(self) -> int: + r""" + Unset for the row grouping runs of unmapped tasks, and for the withheld + row below. + """ + return self._benchmark_id or 0 + + @benchmark_id.setter + def benchmark_id(self, benchmark_id: Optional[int]): + if benchmark_id is None: + del self.benchmark_id + return + if not isinstance(benchmark_id, int): + raise TypeError('benchmark_id must be of type int') + self._benchmark_id = benchmark_id + + @property + def benchmark_name(self) -> str: + return self._benchmark_name + + @benchmark_name.setter + def benchmark_name(self, benchmark_name: str): + if benchmark_name is None: + del self.benchmark_name + return + if not isinstance(benchmark_name, str): + raise TypeError('benchmark_name must be of type str') + self._benchmark_name = benchmark_name + + @property + def withheld(self) -> bool: + r""" + Groups runs on benchmarks the caller can't read, so their counts still add + up without disclosing which benchmarks they are. + """ + return self._withheld + + @withheld.setter + def withheld(self, withheld: bool): + if withheld is None: + del self.withheld + return + if not isinstance(withheld, bool): + raise TypeError('withheld must be of type bool') + self._withheld = withheld + + @property + def queued_count(self) -> int: + return self._queued_count + + @queued_count.setter + def queued_count(self, queued_count: int): + if queued_count is None: + del self.queued_count + return + if not isinstance(queued_count, int): + raise TypeError('queued_count must be of type int') + self._queued_count = queued_count + + @property + def running_count(self) -> int: + return self._running_count + + @running_count.setter + def running_count(self, running_count: int): + if running_count is None: + del self.running_count + return + if not isinstance(running_count, int): + raise TypeError('running_count must be of type int') + self._running_count = running_count + + @property + def recent_state_counts(self) -> Optional[List[Optional['BenchmarkSchedulerStateCount']]]: + return self._recent_state_counts + + @recent_state_counts.setter + def recent_state_counts(self, recent_state_counts: Optional[List[Optional['BenchmarkSchedulerStateCount']]]): + if recent_state_counts is None: + del self.recent_state_counts + return + if not isinstance(recent_state_counts, list): + raise TypeError('recent_state_counts must be of type list') + if not all([isinstance(t, BenchmarkSchedulerStateCount) for t in recent_state_counts]): + raise TypeError('recent_state_counts must contain only items of type BenchmarkSchedulerStateCount') + self._recent_state_counts = recent_state_counts + + +class BenchmarkSchedulerModelVersionStats(KaggleObject): + r""" + Agent runs count under their backing model version, which is the identity + the per-candidate cap and Model Proxy capacity are tracked against. + + Attributes: + benchmark_model_version_id (int) + queued_count (int) + running_count (int) + recent_state_counts (BenchmarkSchedulerStateCount) + Same 24-hour window as the fleet-wide recent_state_counts, so one failing + candidate can be told apart from a fleet-wide problem. + """ + + def __init__(self): + self._benchmark_model_version_id = 0 + self._queued_count = 0 + self._running_count = 0 + self._recent_state_counts = [] + self._freeze() + + @property + def benchmark_model_version_id(self) -> int: + return self._benchmark_model_version_id + + @benchmark_model_version_id.setter + def benchmark_model_version_id(self, benchmark_model_version_id: int): + if benchmark_model_version_id is None: + del self.benchmark_model_version_id + return + if not isinstance(benchmark_model_version_id, int): + raise TypeError('benchmark_model_version_id must be of type int') + self._benchmark_model_version_id = benchmark_model_version_id + + @property + def queued_count(self) -> int: + return self._queued_count + + @queued_count.setter + def queued_count(self, queued_count: int): + if queued_count is None: + del self.queued_count + return + if not isinstance(queued_count, int): + raise TypeError('queued_count must be of type int') + self._queued_count = queued_count + + @property + def running_count(self) -> int: + return self._running_count + + @running_count.setter + def running_count(self, running_count: int): + if running_count is None: + del self.running_count + return + if not isinstance(running_count, int): + raise TypeError('running_count must be of type int') + self._running_count = running_count + + @property + def recent_state_counts(self) -> Optional[List[Optional['BenchmarkSchedulerStateCount']]]: + r""" + Same 24-hour window as the fleet-wide recent_state_counts, so one failing + candidate can be told apart from a fleet-wide problem. + """ + return self._recent_state_counts + + @recent_state_counts.setter + def recent_state_counts(self, recent_state_counts: Optional[List[Optional['BenchmarkSchedulerStateCount']]]): + if recent_state_counts is None: + del self.recent_state_counts + return + if not isinstance(recent_state_counts, list): + raise TypeError('recent_state_counts must be of type list') + if not all([isinstance(t, BenchmarkSchedulerStateCount) for t in recent_state_counts]): + raise TypeError('recent_state_counts must contain only items of type BenchmarkSchedulerStateCount') + self._recent_state_counts = recent_state_counts + + +class BenchmarkSchedulerStateCount(KaggleObject): + r""" + Attributes: + state (BenchmarkTaskRunState) + count (int) + """ + + def __init__(self): + self._state = BenchmarkTaskRunState.BENCHMARK_TASK_RUN_STATE_UNSPECIFIED + self._count = 0 + self._freeze() + + @property + def state(self) -> 'BenchmarkTaskRunState': + return self._state + + @state.setter + def state(self, state: 'BenchmarkTaskRunState'): + if state is None: + del self.state + return + if not isinstance(state, BenchmarkTaskRunState): + raise TypeError('state must be of type BenchmarkTaskRunState') + self._state = state + + @property + def count(self) -> int: + return self._count + + @count.setter + def count(self, count: int): + if count is None: + del self.count + return + if not isinstance(count, int): + raise TypeError('count must be of type int') + self._count = count + + +class GetBenchmarkSchedulerStatusResponse(KaggleObject): + r""" + Point-in-time readout of the scheduler. Under FLEET, every count covers only + runs the scheduler enqueued (non-null EnqueueTime): that scope reports on the + drain, and synchronously started runs are work it never saw and cannot + influence. SELF counts the caller's runs however they were started, since the + surfaces they schedule through start runs synchronously. Counts always + exclude removed runs and leaf runs of a multi-metric parent, which share the + parent's session. + + Attributes: + state_counts (BenchmarkSchedulerStateCount) + Queue depth per live state (Queued, Running, ScorePending); a state with + no runs is omitted. Not a map because proto3 forbids enum keys. + recent_state_counts (BenchmarkSchedulerStateCount) + Completed / Errored counts for runs that ended in the last 24 hours. + Excludes cancellations, which are stamped Errored and removed. + model_version_stats (BenchmarkSchedulerModelVersionStats) + One entry per model version with live work or a recent outcome, ordered + by running count descending, then id ascending so ties stay stable. + settings (BenchmarkSchedulerSettings) + The settings these counts were measured against. The caps are enforced over + the enqueued-only population, so headroom is their difference from the + FLEET counts; under SELF the counts cover a wider population than the caps + govern. Readable by any caller of this RPC, writes stay admin-only. + benchmark_stats (BenchmarkSchedulerBenchmarkStats) + The same counts as model_version_stats, grouped by the run's benchmark + instead, with the same inclusion rule and order. + """ + + def __init__(self): + self._state_counts = [] + self._recent_state_counts = [] + self._model_version_stats = [] + self._settings = None + self._benchmark_stats = [] + self._freeze() + + @property + def state_counts(self) -> Optional[List[Optional['BenchmarkSchedulerStateCount']]]: + r""" + Queue depth per live state (Queued, Running, ScorePending); a state with + no runs is omitted. Not a map because proto3 forbids enum keys. + """ + return self._state_counts + + @state_counts.setter + def state_counts(self, state_counts: Optional[List[Optional['BenchmarkSchedulerStateCount']]]): + if state_counts is None: + del self.state_counts + return + if not isinstance(state_counts, list): + raise TypeError('state_counts must be of type list') + if not all([isinstance(t, BenchmarkSchedulerStateCount) for t in state_counts]): + raise TypeError('state_counts must contain only items of type BenchmarkSchedulerStateCount') + self._state_counts = state_counts + + @property + def recent_state_counts(self) -> Optional[List[Optional['BenchmarkSchedulerStateCount']]]: + r""" + Completed / Errored counts for runs that ended in the last 24 hours. + Excludes cancellations, which are stamped Errored and removed. + """ + return self._recent_state_counts + + @recent_state_counts.setter + def recent_state_counts(self, recent_state_counts: Optional[List[Optional['BenchmarkSchedulerStateCount']]]): + if recent_state_counts is None: + del self.recent_state_counts + return + if not isinstance(recent_state_counts, list): + raise TypeError('recent_state_counts must be of type list') + if not all([isinstance(t, BenchmarkSchedulerStateCount) for t in recent_state_counts]): + raise TypeError('recent_state_counts must contain only items of type BenchmarkSchedulerStateCount') + self._recent_state_counts = recent_state_counts + + @property + def model_version_stats(self) -> Optional[List[Optional['BenchmarkSchedulerModelVersionStats']]]: + r""" + One entry per model version with live work or a recent outcome, ordered + by running count descending, then id ascending so ties stay stable. + """ + return self._model_version_stats + + @model_version_stats.setter + def model_version_stats(self, model_version_stats: Optional[List[Optional['BenchmarkSchedulerModelVersionStats']]]): + if model_version_stats is None: + del self.model_version_stats + return + if not isinstance(model_version_stats, list): + raise TypeError('model_version_stats must be of type list') + if not all([isinstance(t, BenchmarkSchedulerModelVersionStats) for t in model_version_stats]): + raise TypeError('model_version_stats must contain only items of type BenchmarkSchedulerModelVersionStats') + self._model_version_stats = model_version_stats + + @property + def settings(self) -> Optional['BenchmarkSchedulerSettings']: + r""" + The settings these counts were measured against. The caps are enforced over + the enqueued-only population, so headroom is their difference from the + FLEET counts; under SELF the counts cover a wider population than the caps + govern. Readable by any caller of this RPC, writes stay admin-only. + """ + return self._settings + + @settings.setter + def settings(self, settings: Optional['BenchmarkSchedulerSettings']): + if settings is None: + del self.settings + return + if not isinstance(settings, BenchmarkSchedulerSettings): + raise TypeError('settings must be of type BenchmarkSchedulerSettings') + self._settings = settings + + @property + def benchmark_stats(self) -> Optional[List[Optional['BenchmarkSchedulerBenchmarkStats']]]: + r""" + The same counts as model_version_stats, grouped by the run's benchmark + instead, with the same inclusion rule and order. + """ + return self._benchmark_stats + + @benchmark_stats.setter + def benchmark_stats(self, benchmark_stats: Optional[List[Optional['BenchmarkSchedulerBenchmarkStats']]]): + if benchmark_stats is None: + del self.benchmark_stats + return + if not isinstance(benchmark_stats, list): + raise TypeError('benchmark_stats must be of type list') + if not all([isinstance(t, BenchmarkSchedulerBenchmarkStats) for t in benchmark_stats]): + raise TypeError('benchmark_stats must contain only items of type BenchmarkSchedulerBenchmarkStats') + self._benchmark_stats = benchmark_stats + + @property + def stateCounts(self): + return self.state_counts + + @property + def recentStateCounts(self): + return self.recent_state_counts + + @property + def modelVersionStats(self): + return self.model_version_stats + + @property + def benchmarkStats(self): + return self.benchmark_stats + + BatchScheduleBenchmarkModelVersionResult._fields = [ FieldMetadata("benchmarkModelVersionId", "benchmark_model_version_id", "_benchmark_model_version_id", int, 0, PredefinedSerializer()), FieldMetadata("runScheduled", "run_scheduled", "_run_scheduled", bool, False, PredefinedSerializer()), @@ -164,5 +593,36 @@ def parent_task_version_id(self, parent_task_version_id: Optional[int]): FieldMetadata("parentTaskVersionId", "parent_task_version_id", "_parent_task_version_id", int, None, PredefinedSerializer(), optional=True), FieldMetadata("agentId", "agent_id", "_agent_id", int, 0, PredefinedSerializer()), FieldMetadata("candidateType", "candidate_type", "_candidate_type", BenchmarkCandidateType, BenchmarkCandidateType.BENCHMARK_CANDIDATE_TYPE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("benchmarkModelVersionConfigId", "benchmark_model_version_config_id", "_benchmark_model_version_config_id", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("runId", "run_id", "_run_id", int, None, PredefinedSerializer(), optional=True), +] + +BenchmarkSchedulerBenchmarkStats._fields = [ + FieldMetadata("benchmarkId", "benchmark_id", "_benchmark_id", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("benchmarkName", "benchmark_name", "_benchmark_name", str, "", PredefinedSerializer()), + FieldMetadata("withheld", "withheld", "_withheld", bool, False, PredefinedSerializer()), + FieldMetadata("queuedCount", "queued_count", "_queued_count", int, 0, PredefinedSerializer()), + FieldMetadata("runningCount", "running_count", "_running_count", int, 0, PredefinedSerializer()), + FieldMetadata("recentStateCounts", "recent_state_counts", "_recent_state_counts", BenchmarkSchedulerStateCount, [], ListSerializer(KaggleObjectSerializer())), +] + +BenchmarkSchedulerModelVersionStats._fields = [ + FieldMetadata("benchmarkModelVersionId", "benchmark_model_version_id", "_benchmark_model_version_id", int, 0, PredefinedSerializer()), + FieldMetadata("queuedCount", "queued_count", "_queued_count", int, 0, PredefinedSerializer()), + FieldMetadata("runningCount", "running_count", "_running_count", int, 0, PredefinedSerializer()), + FieldMetadata("recentStateCounts", "recent_state_counts", "_recent_state_counts", BenchmarkSchedulerStateCount, [], ListSerializer(KaggleObjectSerializer())), +] + +BenchmarkSchedulerStateCount._fields = [ + FieldMetadata("state", "state", "_state", BenchmarkTaskRunState, BenchmarkTaskRunState.BENCHMARK_TASK_RUN_STATE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("count", "count", "_count", int, 0, PredefinedSerializer()), +] + +GetBenchmarkSchedulerStatusResponse._fields = [ + FieldMetadata("stateCounts", "state_counts", "_state_counts", BenchmarkSchedulerStateCount, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("recentStateCounts", "recent_state_counts", "_recent_state_counts", BenchmarkSchedulerStateCount, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("modelVersionStats", "model_version_stats", "_model_version_stats", BenchmarkSchedulerModelVersionStats, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("settings", "settings", "_settings", BenchmarkSchedulerSettings, None, KaggleObjectSerializer()), + FieldMetadata("benchmarkStats", "benchmark_stats", "_benchmark_stats", BenchmarkSchedulerBenchmarkStats, [], ListSerializer(KaggleObjectSerializer())), ] diff --git a/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py b/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py index 85eb27b..74338be 100644 --- a/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py +++ b/kagglesdk/benchmarks/types/benchmark_tasks_api_service.py @@ -1,8 +1,8 @@ -from datetime import datetime +from datetime import datetime, timedelta from google.protobuf.field_mask_pb2 import FieldMask -from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkTaskRunState, BenchmarkTaskVersionCreationState, BenchmarkTaskVersionSource +from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkArenaArtifactType, BenchmarkCandidateType, BenchmarkEnqueueSkipPolicy, BenchmarkTaskRunState, BenchmarkTaskVersionCreationState, BenchmarkTaskVersionSource from kagglesdk.benchmarks.types.benchmark_task_run_service import BatchScheduleBenchmarkModelVersionResult -from kagglesdk.benchmarks.types.benchmark_types import BenchmarkTaskKaggleDatasetsDefinition, BenchmarkTaskOptions, EnvVariable, UserSecret +from kagglesdk.benchmarks.types.benchmark_types import BenchmarkTaskDefinition, BenchmarkTaskOptions, BenchmarkTaskValidationStage from kagglesdk.kaggle_object import * from typing import List, Optional @@ -17,12 +17,44 @@ class ApiBatchScheduleBenchmarkTaskRunsRequest(KaggleObject): Canonical Agent.Slug values (e.g. 'claude-code-2.1.216-opus-4.8-high'). Agents are backed by model versions and follow the same validation checks. + reasoning_effort (str) + Reasoning effort to evaluate each model version at (e.g. 'high'). Each + model version must have a config declaring this effort, matched as an exact + string; one that doesn't is rejected with InvalidArgument. Without this the + run inherits the default agent's effort. Setting it alongside agent_slugs + is rejected with InvalidArgument, since an agent already pins its own + effort. + enqueue (bool) + When true, the runs are appended to the async scheduler queue via the + internal EnqueueBenchmarkTaskRuns path instead of being scheduled + synchronously. Requires the BENCHMARK_SCHEDULER feature, enforced by that + nested handler; a caller without it gets NOT_FOUND. A task slug that pins + version_number queues that version. + skip_policy (BenchmarkEnqueueSkipPolicy) + Which pairs with existing runs to leave alone when enqueuing; rejected + without enqueue. Unspecified means MISSING_OR_ERRORED, so a pair whose + latest run completed, or that has a run in flight, is not queued again: set + ALL to re-run it. A pair left alone is still reported in results, with + run_skipped_reason saying so. + stale_after (timedelta) + Required by, and only valid with, STALER_THAN. + benchmark (ApiBenchmarkSlug) + The benchmark to attribute the queued runs to; rejected without enqueue. + Attributed runs take the benchmark's scheduler priority, Model Proxy quota + bucket and dispatch pause, so the caller must be able to edit it. Every + task must be mapped under the resolved benchmark version, or the request + is rejected with InvalidArgument. """ def __init__(self): self._task_slugs = [] self._model_version_slugs = [] self._agent_slugs = [] + self._reasoning_effort = None + self._enqueue = None + self._skip_policy = BenchmarkEnqueueSkipPolicy.BENCHMARK_ENQUEUE_SKIP_POLICY_UNSPECIFIED + self._stale_after = None + self._benchmark = None self._freeze() @property @@ -79,6 +111,101 @@ def agent_slugs(self, agent_slugs: Optional[List[str]]): raise TypeError('agent_slugs must contain only items of type str') self._agent_slugs = agent_slugs + @property + def reasoning_effort(self) -> str: + r""" + Reasoning effort to evaluate each model version at (e.g. 'high'). Each + model version must have a config declaring this effort, matched as an exact + string; one that doesn't is rejected with InvalidArgument. Without this the + run inherits the default agent's effort. Setting it alongside agent_slugs + is rejected with InvalidArgument, since an agent already pins its own + effort. + """ + return self._reasoning_effort or "" + + @reasoning_effort.setter + def reasoning_effort(self, reasoning_effort: Optional[str]): + if reasoning_effort is None: + del self.reasoning_effort + return + if not isinstance(reasoning_effort, str): + raise TypeError('reasoning_effort must be of type str') + self._reasoning_effort = reasoning_effort + + @property + def enqueue(self) -> bool: + r""" + When true, the runs are appended to the async scheduler queue via the + internal EnqueueBenchmarkTaskRuns path instead of being scheduled + synchronously. Requires the BENCHMARK_SCHEDULER feature, enforced by that + nested handler; a caller without it gets NOT_FOUND. A task slug that pins + version_number queues that version. + """ + return self._enqueue or False + + @enqueue.setter + def enqueue(self, enqueue: Optional[bool]): + if enqueue is None: + del self.enqueue + return + if not isinstance(enqueue, bool): + raise TypeError('enqueue must be of type bool') + self._enqueue = enqueue + + @property + def skip_policy(self) -> 'BenchmarkEnqueueSkipPolicy': + r""" + Which pairs with existing runs to leave alone when enqueuing; rejected + without enqueue. Unspecified means MISSING_OR_ERRORED, so a pair whose + latest run completed, or that has a run in flight, is not queued again: set + ALL to re-run it. A pair left alone is still reported in results, with + run_skipped_reason saying so. + """ + return self._skip_policy + + @skip_policy.setter + def skip_policy(self, skip_policy: 'BenchmarkEnqueueSkipPolicy'): + if skip_policy is None: + del self.skip_policy + return + if not isinstance(skip_policy, BenchmarkEnqueueSkipPolicy): + raise TypeError('skip_policy must be of type BenchmarkEnqueueSkipPolicy') + self._skip_policy = skip_policy + + @property + def stale_after(self) -> timedelta: + """Required by, and only valid with, STALER_THAN.""" + return self._stale_after or None + + @stale_after.setter + def stale_after(self, stale_after: Optional[timedelta]): + if stale_after is None: + del self.stale_after + return + if not isinstance(stale_after, timedelta): + raise TypeError('stale_after must be of type timedelta') + self._stale_after = stale_after + + @property + def benchmark(self) -> Optional['ApiBenchmarkSlug']: + r""" + The benchmark to attribute the queued runs to; rejected without enqueue. + Attributed runs take the benchmark's scheduler priority, Model Proxy quota + bucket and dispatch pause, so the caller must be able to edit it. Every + task must be mapped under the resolved benchmark version, or the request + is rejected with InvalidArgument. + """ + return self._benchmark + + @benchmark.setter + def benchmark(self, benchmark: Optional['ApiBenchmarkSlug']): + if benchmark is None: + del self.benchmark + return + if not isinstance(benchmark, ApiBenchmarkSlug): + raise TypeError('benchmark must be of type ApiBenchmarkSlug') + self._benchmark = benchmark + def endpoint(self): path = '/api/v1/benchmarks/tasks/schedule' return path.format_map(self.to_field_map(self)) @@ -119,6 +246,68 @@ def results(self, results: Optional[List[Optional['BatchScheduleBenchmarkModelVe self._results = results +class ApiBenchmarkSlug(KaggleObject): + r""" + Attributes: + owner_slug (str) + If omitted: use the current user's slug + benchmark_slug (str) + version_number (int) + If omitted: the current published version, or the draft when there is + none. + """ + + def __init__(self): + self._owner_slug = None + self._benchmark_slug = "" + self._version_number = None + self._freeze() + + @property + def owner_slug(self) -> str: + """If omitted: use the current user's slug""" + return self._owner_slug or "" + + @owner_slug.setter + def owner_slug(self, owner_slug: Optional[str]): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + If omitted: the current published version, or the draft when there is + none. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + class ApiBenchmarkTask(KaggleObject): r""" API equivalent of the BenchmarkTask + BenchmarkTaskVersion from @@ -132,8 +321,8 @@ class ApiBenchmarkTask(KaggleObject): error (str) Optional error string. creation_state (BenchmarkTaskVersionCreationState) - Creation state, for now this is essentially the state of the first - underlying kernel session (SourceKernelSessionId). + Creation state of the task version. See + BenchmarkTaskVersion.creation_state. create_time (datetime) When this task version was created. creation_error_message (str) @@ -165,6 +354,53 @@ class ApiBenchmarkTask(KaggleObject): longer values are rejected with InvalidArgument on update. citations (ApiBenchmarkTaskCitation) Citations on the task's current version. + candidate_type (BenchmarkCandidateType) + The kind of candidate this task version runs against (model versions vs. + agents). Mirrors BenchmarkTaskVersion.candidate_type from the internal + proto. + definition (BenchmarkTaskDefinition) + What backs this task version, returned in the same shape + CreateBenchmarkTask accepts, so a client can compare the task it declared + against what the server stored. Populated by GetBenchmarkTask and + ListBenchmarkTasks only when `include_definition` is set, since it costs + several extra reads. + + Three fields will not match what was sent, so a client comparing the two + should skip them: + * `secrets` is always empty. Secret values are stored in the caller's + UserSecrets, not on the task version. The references do come back, as + `env_variables` entries with `is_secret_name` set. + * `nested_virtualization_required` is always false. It is applied to the + session when the task runs and is not stored on the task version. + * `title` is always set. It holds the stored name, which is trimmed and + which defaults to the task slug when the client sent no title, so a + task's first version can report a title the client never sent. + + The first two are empty for everyone, so `definition_partial` stays false + for them -- nothing was hidden from this particular caller. + + Separately, slugs naming a candidate, auxiliary config or dataset the + reader cannot read are omitted, and `env_variables` is omitted wholesale + unless the caller could update the task. `definition_partial` is set + whenever anything is omitted for one of those reasons. + definition_partial (bool) + True when `definition` is missing something the version declares. + Separates 'declares no default candidate' from 'declares one that did not + come back', which otherwise look identical and would have a client push a + spurious change. False when `definition` is unset. + + Usually this means the caller may not read what was named. It is also set + when a referenced dataset cannot be named -- one still being created, say + -- so a client cannot read it as a statement about its own permissions. + artifact_path (str) + Glob, relative to a run's output root, naming the file the task's preview + shows, e.g. *.mp4. Unset if the task has none. + artifact_type (BenchmarkArenaArtifactType) + How the artifact is rendered. Set iff artifact_path is. + are_run_artifacts_public (bool) + Whether everyone who can read the task can also read its raw run artifacts + (run.json, logs, output files, trajectories), rather than only the task's + writers. """ def __init__(self): @@ -183,6 +419,12 @@ def __init__(self): self._tags = [] self._provenance_sources = None self._citations = [] + self._candidate_type = BenchmarkCandidateType.BENCHMARK_CANDIDATE_TYPE_UNSPECIFIED + self._definition = None + self._definition_partial = False + self._artifact_path = None + self._artifact_type = None + self._are_run_artifacts_public = None self._freeze() @property @@ -230,8 +472,8 @@ def error(self, error: Optional[str]): @property def creation_state(self) -> 'BenchmarkTaskVersionCreationState': r""" - Creation state, for now this is essentially the state of the first - underlying kernel session (SourceKernelSessionId). + Creation state of the task version. See + BenchmarkTaskVersion.creation_state. """ return self._creation_state @@ -317,6 +559,24 @@ def is_backing_notebook_published(self, is_backing_notebook_published: Optional[ raise TypeError('is_backing_notebook_published must be of type bool') self._is_backing_notebook_published = is_backing_notebook_published + @property + def are_run_artifacts_public(self) -> bool: + r""" + Whether everyone who can read the task can also read its raw run artifacts + (run.json, logs, output files, trajectories), rather than only the task's + writers. + """ + return self._are_run_artifacts_public or False + + @are_run_artifacts_public.setter + def are_run_artifacts_public(self, are_run_artifacts_public: Optional[bool]): + if are_run_artifacts_public is None: + del self.are_run_artifacts_public + return + if not isinstance(are_run_artifacts_public, bool): + raise TypeError('are_run_artifacts_public must be of type bool') + self._are_run_artifacts_public = are_run_artifacts_public + @property def options(self) -> Optional['BenchmarkTaskOptions']: r""" @@ -419,6 +679,117 @@ def citations(self, citations: Optional[List[Optional['ApiBenchmarkTaskCitation' raise TypeError('citations must contain only items of type ApiBenchmarkTaskCitation') self._citations = citations + @property + def candidate_type(self) -> 'BenchmarkCandidateType': + r""" + The kind of candidate this task version runs against (model versions vs. + agents). Mirrors BenchmarkTaskVersion.candidate_type from the internal + proto. + """ + return self._candidate_type + + @candidate_type.setter + def candidate_type(self, candidate_type: 'BenchmarkCandidateType'): + if candidate_type is None: + del self.candidate_type + return + if not isinstance(candidate_type, BenchmarkCandidateType): + raise TypeError('candidate_type must be of type BenchmarkCandidateType') + self._candidate_type = candidate_type + + @property + def definition(self) -> Optional['BenchmarkTaskDefinition']: + r""" + What backs this task version, returned in the same shape + CreateBenchmarkTask accepts, so a client can compare the task it declared + against what the server stored. Populated by GetBenchmarkTask and + ListBenchmarkTasks only when `include_definition` is set, since it costs + several extra reads. + + Three fields will not match what was sent, so a client comparing the two + should skip them: + * `secrets` is always empty. Secret values are stored in the caller's + UserSecrets, not on the task version. The references do come back, as + `env_variables` entries with `is_secret_name` set. + * `nested_virtualization_required` is always false. It is applied to the + session when the task runs and is not stored on the task version. + * `title` is always set. It holds the stored name, which is trimmed and + which defaults to the task slug when the client sent no title, so a + task's first version can report a title the client never sent. + + The first two are empty for everyone, so `definition_partial` stays false + for them -- nothing was hidden from this particular caller. + + Separately, slugs naming a candidate, auxiliary config or dataset the + reader cannot read are omitted, and `env_variables` is omitted wholesale + unless the caller could update the task. `definition_partial` is set + whenever anything is omitted for one of those reasons. + """ + return self._definition or None + + @definition.setter + def definition(self, definition: Optional[Optional['BenchmarkTaskDefinition']]): + if definition is None: + del self.definition + return + if not isinstance(definition, BenchmarkTaskDefinition): + raise TypeError('definition must be of type BenchmarkTaskDefinition') + self._definition = definition + + @property + def definition_partial(self) -> bool: + r""" + True when `definition` is missing something the version declares. + Separates 'declares no default candidate' from 'declares one that did not + come back', which otherwise look identical and would have a client push a + spurious change. False when `definition` is unset. + + Usually this means the caller may not read what was named. It is also set + when a referenced dataset cannot be named -- one still being created, say + -- so a client cannot read it as a statement about its own permissions. + """ + return self._definition_partial + + @definition_partial.setter + def definition_partial(self, definition_partial: bool): + if definition_partial is None: + del self.definition_partial + return + if not isinstance(definition_partial, bool): + raise TypeError('definition_partial must be of type bool') + self._definition_partial = definition_partial + + @property + def artifact_path(self) -> str: + r""" + Glob, relative to a run's output root, naming the file the task's preview + shows, e.g. *.mp4. Unset if the task has none. + """ + return self._artifact_path or "" + + @artifact_path.setter + def artifact_path(self, artifact_path: Optional[str]): + if artifact_path is None: + del self.artifact_path + return + if not isinstance(artifact_path, str): + raise TypeError('artifact_path must be of type str') + self._artifact_path = artifact_path + + @property + def artifact_type(self) -> 'BenchmarkArenaArtifactType': + """How the artifact is rendered. Set iff artifact_path is.""" + return self._artifact_type or BenchmarkArenaArtifactType.BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED + + @artifact_type.setter + def artifact_type(self, artifact_type: Optional['BenchmarkArenaArtifactType']): + if artifact_type is None: + del self.artifact_type + return + if not isinstance(artifact_type, BenchmarkArenaArtifactType): + raise TypeError('artifact_type must be of type BenchmarkArenaArtifactType') + self._artifact_type = artifact_type + class ApiBenchmarkTaskCitation(KaggleObject): r""" @@ -490,6 +861,20 @@ class ApiBenchmarkTaskRun(KaggleObject): End time of the run error_message (str) Error message if the run failed + agent_slug (str) + Agent candidate against the task, when the run was scheduled against an + agent rather than a bare model version. This is the Agent.Slug; unset for + model-version runs. + model_version_config_slug (str) + The resolved BenchmarkModelVersionConfig the run was dispatched against. + This is the BenchmarkModelVersionConfig.Slug. During the current transition + to config-based dispatch, this is only set when the run was scheduled with + a reasoning effort; runs against a bare model version (and historical runs + that predate the configs table) leave it unset. + result (ApiBenchmarkTaskRunResult) + Result the run produced (score or pass/fail). Unset for runs that have not + produced a result (e.g. still queued/running, errored, or no result + recorded). """ def __init__(self): @@ -500,6 +885,9 @@ def __init__(self): self._start_time = None self._end_time = None self._error_message = None + self._agent_slug = None + self._model_version_config_slug = None + self._result = None self._freeze() @property @@ -604,6 +992,135 @@ def error_message(self, error_message: Optional[str]): raise TypeError('error_message must be of type str') self._error_message = error_message + @property + def agent_slug(self) -> str: + r""" + Agent candidate against the task, when the run was scheduled against an + agent rather than a bare model version. This is the Agent.Slug; unset for + model-version runs. + """ + return self._agent_slug or "" + + @agent_slug.setter + def agent_slug(self, agent_slug: Optional[str]): + if agent_slug is None: + del self.agent_slug + return + if not isinstance(agent_slug, str): + raise TypeError('agent_slug must be of type str') + self._agent_slug = agent_slug + + @property + def model_version_config_slug(self) -> str: + r""" + The resolved BenchmarkModelVersionConfig the run was dispatched against. + This is the BenchmarkModelVersionConfig.Slug. During the current transition + to config-based dispatch, this is only set when the run was scheduled with + a reasoning effort; runs against a bare model version (and historical runs + that predate the configs table) leave it unset. + """ + return self._model_version_config_slug or "" + + @model_version_config_slug.setter + def model_version_config_slug(self, model_version_config_slug: Optional[str]): + if model_version_config_slug is None: + del self.model_version_config_slug + return + if not isinstance(model_version_config_slug, str): + raise TypeError('model_version_config_slug must be of type str') + self._model_version_config_slug = model_version_config_slug + + @property + def result(self) -> Optional['ApiBenchmarkTaskRunResult']: + r""" + Result the run produced (score or pass/fail). Unset for runs that have not + produced a result (e.g. still queued/running, errored, or no result + recorded). + """ + return self._result or None + + @result.setter + def result(self, result: Optional[Optional['ApiBenchmarkTaskRunResult']]): + if result is None: + del self.result + return + if not isinstance(result, ApiBenchmarkTaskRunResult): + raise TypeError('result must be of type ApiBenchmarkTaskRunResult') + self._result = result + + +class ApiBenchmarkTaskRunIds(KaggleObject): + r""" + A repeated field cannot live in a oneof directly, hence the wrapper. + + Attributes: + ids (int) + """ + + def __init__(self): + self._ids = [] + self._freeze() + + @property + def ids(self) -> Optional[List[int]]: + return self._ids + + @ids.setter + def ids(self, ids: Optional[List[int]]): + if ids is None: + del self.ids + return + if not isinstance(ids, list): + raise TypeError('ids must be of type list') + if not all([isinstance(t, int) for t in ids]): + raise TypeError('ids must contain only items of type int') + self._ids = ids + + +class ApiBenchmarkTaskRunResult(KaggleObject): + r""" + Minimal public view of a run's result. Mirrors the numeric/boolean value the + task UI renders; the internal BenchmarkTaskRunResult carries additional + metrics/assertions that are intentionally not exposed here for now. + + Attributes: + numeric_result (float) + boolean_result (bool) + """ + + def __init__(self): + self._numeric_result = None + self._boolean_result = None + self._freeze() + + @property + def numeric_result(self) -> float: + return self._numeric_result or 0.0 + + @numeric_result.setter + def numeric_result(self, numeric_result: float): + if numeric_result is None: + del self.numeric_result + return + if not isinstance(numeric_result, float): + raise TypeError('numeric_result must be of type float') + del self.boolean_result + self._numeric_result = numeric_result + + @property + def boolean_result(self) -> bool: + return self._boolean_result or False + + @boolean_result.setter + def boolean_result(self, boolean_result: bool): + if boolean_result is None: + del self.boolean_result + return + if not isinstance(boolean_result, bool): + raise TypeError('boolean_result must be of type bool') + del self.numeric_result + self._boolean_result = boolean_result + class ApiBenchmarkTaskSlug(KaggleObject): r""" @@ -675,6 +1192,208 @@ def version_number(self, version_number: Optional[int]): self._version_number = version_number +class ApiBenchmarkTaskSlugs(KaggleObject): + r""" + A repeated field cannot live in a oneof directly, hence the wrapper. + + Attributes: + slugs (ApiBenchmarkTaskSlug) + """ + + def __init__(self): + self._slugs = [] + self._freeze() + + @property + def slugs(self) -> Optional[List[Optional['ApiBenchmarkTaskSlug']]]: + return self._slugs + + @slugs.setter + def slugs(self, slugs: Optional[List[Optional['ApiBenchmarkTaskSlug']]]): + if slugs is None: + del self.slugs + return + if not isinstance(slugs, list): + raise TypeError('slugs must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskSlug) for t in slugs]): + raise TypeError('slugs must contain only items of type ApiBenchmarkTaskSlug') + self._slugs = slugs + + +class ApiCancelBenchmarkTaskRunsRequest(KaggleObject): + r""" + Attributes: + task_slugs (ApiBenchmarkTaskSlugs) + Every run of these tasks. + model_version (ApiModelVersionCandidate) + agent_slug (str) + A canonical Agent.Slug (e.g. 'claude-code-2.1.216-opus-4.8-high'). + include_active (bool) + When false, only queued runs are cancelled, leaving in-flight work alone. + When true, running runs are also terminated. + run_ids (ApiBenchmarkTaskRunIds) + Exactly these runs, addressed by the ids ApiBenchmarkTaskRun reports. + """ + + def __init__(self): + self._task_slugs = None + self._model_version = None + self._agent_slug = None + self._include_active = False + self._run_ids = None + self._freeze() + + @property + def task_slugs(self) -> Optional['ApiBenchmarkTaskSlugs']: + """Every run of these tasks.""" + return self._task_slugs or None + + @task_slugs.setter + def task_slugs(self, task_slugs: Optional['ApiBenchmarkTaskSlugs']): + if task_slugs is None: + del self.task_slugs + return + if not isinstance(task_slugs, ApiBenchmarkTaskSlugs): + raise TypeError('task_slugs must be of type ApiBenchmarkTaskSlugs') + del self.run_ids + self._task_slugs = task_slugs + + @property + def run_ids(self) -> Optional['ApiBenchmarkTaskRunIds']: + """Exactly these runs, addressed by the ids ApiBenchmarkTaskRun reports.""" + return self._run_ids or None + + @run_ids.setter + def run_ids(self, run_ids: Optional['ApiBenchmarkTaskRunIds']): + if run_ids is None: + del self.run_ids + return + if not isinstance(run_ids, ApiBenchmarkTaskRunIds): + raise TypeError('run_ids must be of type ApiBenchmarkTaskRunIds') + del self.task_slugs + self._run_ids = run_ids + + @property + def model_version(self) -> Optional['ApiModelVersionCandidate']: + return self._model_version or None + + @model_version.setter + def model_version(self, model_version: Optional['ApiModelVersionCandidate']): + if model_version is None: + del self.model_version + return + if not isinstance(model_version, ApiModelVersionCandidate): + raise TypeError('model_version must be of type ApiModelVersionCandidate') + del self.agent_slug + self._model_version = model_version + + @property + def agent_slug(self) -> str: + """A canonical Agent.Slug (e.g. 'claude-code-2.1.216-opus-4.8-high').""" + return self._agent_slug or "" + + @agent_slug.setter + def agent_slug(self, agent_slug: str): + if agent_slug is None: + del self.agent_slug + return + if not isinstance(agent_slug, str): + raise TypeError('agent_slug must be of type str') + del self.model_version + self._agent_slug = agent_slug + + @property + def include_active(self) -> bool: + r""" + When false, only queued runs are cancelled, leaving in-flight work alone. + When true, running runs are also terminated. + """ + return self._include_active + + @include_active.setter + def include_active(self, include_active: bool): + if include_active is None: + del self.include_active + return + if not isinstance(include_active, bool): + raise TypeError('include_active must be of type bool') + self._include_active = include_active + + def endpoint(self): + path = '/api/v1/benchmarks/tasks/runs/cancel' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiCancelBenchmarkTaskRunsResponse(KaggleObject): + r""" + Attributes: + queued_cancelled_count (int) + Number of queued runs cancelled. + active_terminated_count (int) + Number of running runs whose sessions were successfully terminated. Only + ever non-zero when include_active was set. A run whose session could not + be reached is left running and excluded from this count, so a caller that + needs everything stopped should re-issue the request until it stops + reporting fewer than it expects. + """ + + def __init__(self): + self._queued_cancelled_count = 0 + self._active_terminated_count = 0 + self._freeze() + + @property + def queued_cancelled_count(self) -> int: + """Number of queued runs cancelled.""" + return self._queued_cancelled_count + + @queued_cancelled_count.setter + def queued_cancelled_count(self, queued_cancelled_count: int): + if queued_cancelled_count is None: + del self.queued_cancelled_count + return + if not isinstance(queued_cancelled_count, int): + raise TypeError('queued_cancelled_count must be of type int') + self._queued_cancelled_count = queued_cancelled_count + + @property + def active_terminated_count(self) -> int: + r""" + Number of running runs whose sessions were successfully terminated. Only + ever non-zero when include_active was set. A run whose session could not + be reached is left running and excluded from this count, so a caller that + needs everything stopped should re-issue the request until it stops + reporting fewer than it expects. + """ + return self._active_terminated_count + + @active_terminated_count.setter + def active_terminated_count(self, active_terminated_count: int): + if active_terminated_count is None: + del self.active_terminated_count + return + if not isinstance(active_terminated_count, int): + raise TypeError('active_terminated_count must be of type int') + self._active_terminated_count = active_terminated_count + + @property + def queuedCancelledCount(self): + return self.queued_cancelled_count + + @property + def activeTerminatedCount(self): + return self.active_terminated_count + + class ApiCreateBenchmarkTaskRequest(KaggleObject): r""" Attributes: @@ -696,9 +1415,11 @@ class ApiCreateBenchmarkTaskRequest(KaggleObject): preserve the previous version's options must echo them back on each push. definition (BenchmarkTaskDefinition) - What backs this task (notebook / docker image / Harbor git repo) and - any per-type configuration. Pairs with the BenchmarkTaskDefinitionType + What backs this task (notebook / docker image / Harbor Kaggle datasets) + and any per-type configuration. Pairs with the BenchmarkTaskDefinitionType enum stored on the entity. + validation_config (ApiTaskCreationValidationConfig) + Optional. How the created task version is validated. """ def __init__(self): @@ -706,6 +1427,7 @@ def __init__(self): self._text = "" self._options = None self._definition = None + self._validation_config = None self._freeze() @property @@ -768,8 +1490,8 @@ def options(self, options: Optional[Optional['BenchmarkTaskOptions']]): @property def definition(self) -> Optional['BenchmarkTaskDefinition']: r""" - What backs this task (notebook / docker image / Harbor git repo) and - any per-type configuration. Pairs with the BenchmarkTaskDefinitionType + What backs this task (notebook / docker image / Harbor Kaggle datasets) + and any per-type configuration. Pairs with the BenchmarkTaskDefinitionType enum stored on the entity. """ return self._definition or None @@ -783,6 +1505,20 @@ def definition(self, definition: Optional[Optional['BenchmarkTaskDefinition']]): raise TypeError('definition must be of type BenchmarkTaskDefinition') self._definition = definition + @property + def validation_config(self) -> Optional['ApiTaskCreationValidationConfig']: + """Optional. How the created task version is validated.""" + return self._validation_config or None + + @validation_config.setter + def validation_config(self, validation_config: Optional[Optional['ApiTaskCreationValidationConfig']]): + if validation_config is None: + del self.validation_config + return + if not isinstance(validation_config, ApiTaskCreationValidationConfig): + raise TypeError('validation_config must be of type ApiTaskCreationValidationConfig') + self._validation_config = validation_config + def endpoint(self): path = '/api/v1/benchmarks/tasks/push' return path.format_map(self.to_field_map(self)) @@ -852,6 +1588,18 @@ def endpoint_path(): return '/api/v1/benchmarks/tasks/runs/{run_id}/output' +class ApiGetBenchmarkSchedulerStatusRequest(KaggleObject): + r""" + No scope field: this endpoint always reports the caller's own runs. + + """ + + pass + def endpoint(self): + path = '/api/v1/benchmarks/scheduler/status' + return path.format_map(self.to_field_map(self)) + + class ApiGetBenchmarkTaskQuotaRequest(KaggleObject): r""" """ @@ -917,10 +1665,14 @@ class ApiGetBenchmarkTaskRequest(KaggleObject): r""" Attributes: slug (ApiBenchmarkTaskSlug) + include_definition (bool) + Populate `definition` and `definition_partial`. Off by default since it + costs extra reads. """ def __init__(self): self._slug = None + self._include_definition = False self._freeze() @property @@ -936,6 +1688,23 @@ def slug(self, slug: Optional['ApiBenchmarkTaskSlug']): raise TypeError('slug must be of type ApiBenchmarkTaskSlug') self._slug = slug + @property + def include_definition(self) -> bool: + r""" + Populate `definition` and `definition_partial`. Off by default since it + costs extra reads. + """ + return self._include_definition + + @include_definition.setter + def include_definition(self, include_definition: bool): + if include_definition is None: + del self.include_definition + return + if not isinstance(include_definition, bool): + raise TypeError('include_definition must be of type bool') + self._include_definition = include_definition + def endpoint(self): path = '/api/v1/benchmarks/tasks/{slug.task_slug}' return path.format_map(self.to_field_map(self)) @@ -1151,6 +1920,22 @@ class ApiListBenchmarkTasksRequest(KaggleObject): page_size (int) page_token (str) skip (int) + owner_slug (str) + Filter to tasks owned by this user (their username / profile slug). Only + tasks the caller is allowed to see are returned, so pass your own slug to + list just the tasks you created. When omitted, returns tasks the caller + owns or collaborates on. + benchmark_slug (str) + Filter to the tasks belonging to this benchmark, identified by its slug + (paired with owner_slug, which then names the benchmark's owner rather than + the task's owner). Resolves to the benchmark version's mapped child tasks. + When omitted, the listing is not scoped to a benchmark. + version_number (int) + Optional benchmark version to target when benchmark_slug is set. When + omitted, the benchmark's current published (or draft) version is used. + include_definition (bool) + Populate `definition` and `definition_partial` on each task. Off by + default since it costs extra reads. """ def __init__(self): @@ -1159,6 +1944,10 @@ def __init__(self): self._page_size = 0 self._page_token = "" self._skip = 0 + self._owner_slug = None + self._benchmark_slug = None + self._version_number = None + self._include_definition = False self._freeze() @property @@ -1228,6 +2017,78 @@ def skip(self, skip: int): raise TypeError('skip must be of type int') self._skip = skip + @property + def owner_slug(self) -> str: + r""" + Filter to tasks owned by this user (their username / profile slug). Only + tasks the caller is allowed to see are returned, so pass your own slug to + list just the tasks you created. When omitted, returns tasks the caller + owns or collaborates on. + """ + return self._owner_slug or "" + + @owner_slug.setter + def owner_slug(self, owner_slug: Optional[str]): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + r""" + Filter to the tasks belonging to this benchmark, identified by its slug + (paired with owner_slug, which then names the benchmark's owner rather than + the task's owner). Resolves to the benchmark version's mapped child tasks. + When omitted, the listing is not scoped to a benchmark. + """ + return self._benchmark_slug or "" + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: Optional[str]): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Optional benchmark version to target when benchmark_slug is set. When + omitted, the benchmark's current published (or draft) version is used. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def include_definition(self) -> bool: + r""" + Populate `definition` and `definition_partial` on each task. Off by + default since it costs extra reads. + """ + return self._include_definition + + @include_definition.setter + def include_definition(self, include_definition: bool): + if include_definition is None: + del self.include_definition + return + if not isinstance(include_definition, bool): + raise TypeError('include_definition must be of type bool') + self._include_definition = include_definition + def endpoint(self): path = '/api/v1/benchmarks/tasks/list' return path.format_map(self.to_field_map(self)) @@ -1297,6 +2158,57 @@ def nextPageToken(self): return self.next_page_token +class ApiModelVersionCandidate(KaggleObject): + r""" + Slug-keyed counterpart of ModelVersionCandidate. + + Attributes: + model_version_slug (str) + A canonical BenchmarkModelVersion.Slug (e.g. 'claude-sonnet-4-6-default'). + reasoning_effort (str) + Narrow to the runs evaluating the model version at this reasoning effort + (e.g. 'high'), matched as an exact string against the model version's + configs. Unset means 'any effort'. + """ + + def __init__(self): + self._model_version_slug = "" + self._reasoning_effort = None + self._freeze() + + @property + def model_version_slug(self) -> str: + """A canonical BenchmarkModelVersion.Slug (e.g. 'claude-sonnet-4-6-default').""" + return self._model_version_slug + + @model_version_slug.setter + def model_version_slug(self, model_version_slug: str): + if model_version_slug is None: + del self.model_version_slug + return + if not isinstance(model_version_slug, str): + raise TypeError('model_version_slug must be of type str') + self._model_version_slug = model_version_slug + + @property + def reasoning_effort(self) -> str: + r""" + Narrow to the runs evaluating the model version at this reasoning effort + (e.g. 'high'), matched as an exact string against the model version's + configs. Unset means 'any effort'. + """ + return self._reasoning_effort or "" + + @reasoning_effort.setter + def reasoning_effort(self, reasoning_effort: Optional[str]): + if reasoning_effort is None: + del self.reasoning_effort + return + if not isinstance(reasoning_effort, str): + raise TypeError('reasoning_effort must be of type str') + self._reasoning_effort = reasoning_effort + + class ApiPublishBenchmarkTaskRequest(KaggleObject): r""" Attributes: @@ -1306,11 +2218,17 @@ class ApiPublishBenchmarkTaskRequest(KaggleObject): When true, also publishes the backing notebook (`source_kernel_id`) in the same request. When false/unset, the backing notebook is left alone. Unpublishing the notebook is not supported through this endpoint. + publish_run_artifacts (bool) + When true, also makes the task's raw run artifacts (run.json, logs, output + files, trajectories) visible to everyone. When false/unset, only the + task's writers can see them and everyone else sees scores only. Hiding + them again is not supported through this endpoint. """ def __init__(self): self._slug = None self._publish_backing_notebook = False + self._publish_run_artifacts = False self._freeze() @property @@ -1345,6 +2263,25 @@ def publish_backing_notebook(self, publish_backing_notebook: bool): raise TypeError('publish_backing_notebook must be of type bool') self._publish_backing_notebook = publish_backing_notebook + @property + def publish_run_artifacts(self) -> bool: + r""" + When true, also makes the task's raw run artifacts (run.json, logs, output + files, trajectories) visible to everyone. When false/unset, only the + task's writers can see them and everyone else sees scores only. Hiding + them again is not supported through this endpoint. + """ + return self._publish_run_artifacts + + @publish_run_artifacts.setter + def publish_run_artifacts(self, publish_run_artifacts: bool): + if publish_run_artifacts is None: + del self.publish_run_artifacts + return + if not isinstance(publish_run_artifacts, bool): + raise TypeError('publish_run_artifacts must be of type bool') + self._publish_run_artifacts = publish_run_artifacts + def endpoint(self): path = '/api/v1/benchmarks/tasks/{slug.task_slug}/publish' return path.format_map(self.to_field_map(self)) @@ -1359,6 +2296,36 @@ def body_fields(): return '*' +class ApiTaskCreationValidationConfig(KaggleObject): + r""" + The public API input shape of TaskCreationValidationConfig. + + Attributes: + validation_stages (ApiValidationStage) + The validation stages to run against the task version. + """ + + def __init__(self): + self._validation_stages = [] + self._freeze() + + @property + def validation_stages(self) -> Optional[List[Optional['ApiValidationStage']]]: + """The validation stages to run against the task version.""" + return self._validation_stages + + @validation_stages.setter + def validation_stages(self, validation_stages: Optional[List[Optional['ApiValidationStage']]]): + if validation_stages is None: + del self.validation_stages + return + if not isinstance(validation_stages, list): + raise TypeError('validation_stages must be of type list') + if not all([isinstance(t, ApiValidationStage) for t in validation_stages]): + raise TypeError('validation_stages must contain only items of type ApiValidationStage') + self._validation_stages = validation_stages + + class ApiUpdateBenchmarkTaskRequest(KaggleObject): r""" Attributes: @@ -1592,6 +2559,11 @@ class ApiUpdateBenchmarkTaskVersionRequest(KaggleObject): update_mask (FieldMask) Which of the above to write. Only fields named here are read. `tags` is not writable here: tags are task-scoped, so use UpdateBenchmarkTask. + artifact_path (str) + The file the task's preview shows, as a glob relative to a run's output + root. Written together with artifact_type: naming either in update_mask + writes both, and leaving both unset clears them. + artifact_type (BenchmarkArenaArtifactType) """ def __init__(self): @@ -1600,6 +2572,8 @@ def __init__(self): self._provenance_sources = None self._citations = [] self._update_mask = None + self._artifact_path = None + self._artifact_type = None self._freeze() @property @@ -1674,6 +2648,37 @@ def citations(self, citations: Optional[List[Optional['ApiBenchmarkTaskCitation' raise TypeError('citations must contain only items of type ApiBenchmarkTaskCitation') self._citations = citations + @property + def artifact_path(self) -> str: + r""" + The file the task's preview shows, as a glob relative to a run's output + root. Written together with artifact_type: naming either in update_mask + writes both, and leaving both unset clears them. + """ + return self._artifact_path or "" + + @artifact_path.setter + def artifact_path(self, artifact_path: Optional[str]): + if artifact_path is None: + del self.artifact_path + return + if not isinstance(artifact_path, str): + raise TypeError('artifact_path must be of type str') + self._artifact_path = artifact_path + + @property + def artifact_type(self) -> 'BenchmarkArenaArtifactType': + return self._artifact_type or BenchmarkArenaArtifactType.BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED + + @artifact_type.setter + def artifact_type(self, artifact_type: Optional['BenchmarkArenaArtifactType']): + if artifact_type is None: + del self.artifact_type + return + if not isinstance(artifact_type, BenchmarkArenaArtifactType): + raise TypeError('artifact_type must be of type BenchmarkArenaArtifactType') + self._artifact_type = artifact_type + @property def update_mask(self) -> FieldMask: r""" @@ -1705,359 +2710,127 @@ def body_fields(): return '*' -class BenchmarkTaskDefinition(KaggleObject): +class ApiValidationStage(KaggleObject): r""" - Attributes: - notebook (BenchmarkTaskNotebookDefinition) - docker_image (BenchmarkTaskDockerImageDefinition) - harbor_git (BenchmarkTaskGitDefinition) - harbor_kaggle_datasets (BenchmarkTaskKaggleDatasetsDefinition) - """ - - def __init__(self): - self._notebook = None - self._docker_image = None - self._harbor_git = None - self._harbor_kaggle_datasets = None - self._freeze() - - @property - def notebook(self) -> Optional['BenchmarkTaskNotebookDefinition']: - return self._notebook or None - - @notebook.setter - def notebook(self, notebook: Optional['BenchmarkTaskNotebookDefinition']): - if notebook is None: - del self.notebook - return - if not isinstance(notebook, BenchmarkTaskNotebookDefinition): - raise TypeError('notebook must be of type BenchmarkTaskNotebookDefinition') - del self.docker_image - del self.harbor_git - del self.harbor_kaggle_datasets - self._notebook = notebook - - @property - def docker_image(self) -> Optional['BenchmarkTaskDockerImageDefinition']: - return self._docker_image or None - - @docker_image.setter - def docker_image(self, docker_image: Optional['BenchmarkTaskDockerImageDefinition']): - if docker_image is None: - del self.docker_image - return - if not isinstance(docker_image, BenchmarkTaskDockerImageDefinition): - raise TypeError('docker_image must be of type BenchmarkTaskDockerImageDefinition') - del self.notebook - del self.harbor_git - del self.harbor_kaggle_datasets - self._docker_image = docker_image - - @property - def harbor_git(self) -> Optional['BenchmarkTaskGitDefinition']: - return self._harbor_git or None - - @harbor_git.setter - def harbor_git(self, harbor_git: Optional['BenchmarkTaskGitDefinition']): - if harbor_git is None: - del self.harbor_git - return - if not isinstance(harbor_git, BenchmarkTaskGitDefinition): - raise TypeError('harbor_git must be of type BenchmarkTaskGitDefinition') - del self.notebook - del self.docker_image - del self.harbor_kaggle_datasets - self._harbor_git = harbor_git - - @property - def harbor_kaggle_datasets(self) -> Optional['BenchmarkTaskKaggleDatasetsDefinition']: - return self._harbor_kaggle_datasets or None - - @harbor_kaggle_datasets.setter - def harbor_kaggle_datasets(self, harbor_kaggle_datasets: Optional['BenchmarkTaskKaggleDatasetsDefinition']): - if harbor_kaggle_datasets is None: - del self.harbor_kaggle_datasets - return - if not isinstance(harbor_kaggle_datasets, BenchmarkTaskKaggleDatasetsDefinition): - raise TypeError('harbor_kaggle_datasets must be of type BenchmarkTaskKaggleDatasetsDefinition') - del self.notebook - del self.docker_image - del self.harbor_git - self._harbor_kaggle_datasets = harbor_kaggle_datasets - - -class BenchmarkTaskDockerImageDefinition(KaggleObject): - r""" - User-supplied Docker image task definition. + The public API input shape of ValidationStage, which identifies the default + candidate by slug instead of id. Attributes: - docker_image_url (str) - Required. Full image ref (e.g. gcr.io/project/image:tag). - description (str) - Required on new tasks; ignored on version bumps (existing description - carries over). Task description. - title (str) - Optional. Display name for the task. Defaults to the slug when omitted. - Ignored on version bumps. - env_variables (EnvVariable) - Optional. Environment variables set on the container. - secrets (UserSecret) - Optional. Create/update UserSecrets inline before running. - nested_virtualization_required (bool) - Optional. Run the container with nested virtualization enabled — set - when the image needs to launch VMs (e.g. QEMU/KVM guests) inside its - sandbox. Ignored unless the caller has the - BENCHMARKS_ALLOW_NESTED_VIRTUALIZATION feature flag. + type (BenchmarkTaskValidationStage) + Required. Not UNSPECIFIED. + default_model_version_slug (str) + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + On Harbor tasks, requires override_harness_version: without one the task + evaluates agents, so pick the model via default_agent_slug instead. + default_agent_slug (str) + Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high'). + default_model_version_config_slug (str) + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. Harbor tasks only; requires + override_harness_version. """ def __init__(self): - self._docker_image_url = "" - self._description = None - self._title = None - self._env_variables = [] - self._secrets = [] - self._nested_virtualization_required = False + self._type = BenchmarkTaskValidationStage.BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED + self._default_model_version_slug = None + self._default_agent_slug = None + self._default_model_version_config_slug = None self._freeze() @property - def docker_image_url(self) -> str: - """Required. Full image ref (e.g. gcr.io/project/image:tag).""" - return self._docker_image_url + def type(self) -> 'BenchmarkTaskValidationStage': + """Required. Not UNSPECIFIED.""" + return self._type - @docker_image_url.setter - def docker_image_url(self, docker_image_url: str): - if docker_image_url is None: - del self.docker_image_url + @type.setter + def type(self, type: 'BenchmarkTaskValidationStage'): + if type is None: + del self.type return - if not isinstance(docker_image_url, str): - raise TypeError('docker_image_url must be of type str') - self._docker_image_url = docker_image_url + if not isinstance(type, BenchmarkTaskValidationStage): + raise TypeError('type must be of type BenchmarkTaskValidationStage') + self._type = type @property - def description(self) -> str: + def default_model_version_slug(self) -> str: r""" - Required on new tasks; ignored on version bumps (existing description - carries over). Task description. + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + On Harbor tasks, requires override_harness_version: without one the task + evaluates agents, so pick the model via default_agent_slug instead. """ - return self._description or "" - - @description.setter - def description(self, description: Optional[str]): - if description is None: - del self.description - return - if not isinstance(description, str): - raise TypeError('description must be of type str') - self._description = description - - @property - def title(self) -> str: - r""" - Optional. Display name for the task. Defaults to the slug when omitted. - Ignored on version bumps. - """ - return self._title or "" - - @title.setter - def title(self, title: Optional[str]): - if title is None: - del self.title - return - if not isinstance(title, str): - raise TypeError('title must be of type str') - self._title = title - - @property - def env_variables(self) -> Optional[List[Optional['EnvVariable']]]: - """Optional. Environment variables set on the container.""" - return self._env_variables - - @env_variables.setter - def env_variables(self, env_variables: Optional[List[Optional['EnvVariable']]]): - if env_variables is None: - del self.env_variables - return - if not isinstance(env_variables, list): - raise TypeError('env_variables must be of type list') - if not all([isinstance(t, EnvVariable) for t in env_variables]): - raise TypeError('env_variables must contain only items of type EnvVariable') - self._env_variables = env_variables - - @property - def secrets(self) -> Optional[List[Optional['UserSecret']]]: - """Optional. Create/update UserSecrets inline before running.""" - return self._secrets - - @secrets.setter - def secrets(self, secrets: Optional[List[Optional['UserSecret']]]): - if secrets is None: - del self.secrets - return - if not isinstance(secrets, list): - raise TypeError('secrets must be of type list') - if not all([isinstance(t, UserSecret) for t in secrets]): - raise TypeError('secrets must contain only items of type UserSecret') - self._secrets = secrets - - @property - def nested_virtualization_required(self) -> bool: - r""" - Optional. Run the container with nested virtualization enabled — set - when the image needs to launch VMs (e.g. QEMU/KVM guests) inside its - sandbox. Ignored unless the caller has the - BENCHMARKS_ALLOW_NESTED_VIRTUALIZATION feature flag. - """ - return self._nested_virtualization_required - - @nested_virtualization_required.setter - def nested_virtualization_required(self, nested_virtualization_required: bool): - if nested_virtualization_required is None: - del self.nested_virtualization_required - return - if not isinstance(nested_virtualization_required, bool): - raise TypeError('nested_virtualization_required must be of type bool') - self._nested_virtualization_required = nested_virtualization_required - - -class BenchmarkTaskGitDefinition(KaggleObject): - r""" - Harbor task definition in a GitHub repo subpath (at a specific commit SHA). - - Attributes: - url (str) - HTTPS URL of the GitHub repo containing the Harbor task. - commit_hash (str) - Specific commit SHA for the repository to pin against. - subpath (str) - Subpath to the task within the repo. - env_variables (EnvVariable) - Optional. Environment variables to expose to the Harbor session container. - secrets (UserSecret) - Optional. Create/update UserSecrets inline before running. - use_accelerator (bool) - Optional. Route the Harbor session to an L4-GPU exeunit pool. Ignored - unless the caller has the BENCHMARKS_ALLOW_USE_ACCELERATOR feature flag. - """ + return self._default_model_version_slug or "" - def __init__(self): - self._url = "" - self._commit_hash = "" - self._subpath = "" - self._env_variables = [] - self._secrets = [] - self._use_accelerator = False - self._freeze() - - @property - def url(self) -> str: - """HTTPS URL of the GitHub repo containing the Harbor task.""" - return self._url - - @url.setter - def url(self, url: str): - if url is None: - del self.url - return - if not isinstance(url, str): - raise TypeError('url must be of type str') - self._url = url - - @property - def commit_hash(self) -> str: - """Specific commit SHA for the repository to pin against.""" - return self._commit_hash - - @commit_hash.setter - def commit_hash(self, commit_hash: str): - if commit_hash is None: - del self.commit_hash - return - if not isinstance(commit_hash, str): - raise TypeError('commit_hash must be of type str') - self._commit_hash = commit_hash - - @property - def subpath(self) -> str: - """Subpath to the task within the repo.""" - return self._subpath - - @subpath.setter - def subpath(self, subpath: str): - if subpath is None: - del self.subpath - return - if not isinstance(subpath, str): - raise TypeError('subpath must be of type str') - self._subpath = subpath - - @property - def env_variables(self) -> Optional[List[Optional['EnvVariable']]]: - """Optional. Environment variables to expose to the Harbor session container.""" - return self._env_variables - - @env_variables.setter - def env_variables(self, env_variables: Optional[List[Optional['EnvVariable']]]): - if env_variables is None: - del self.env_variables + @default_model_version_slug.setter + def default_model_version_slug(self, default_model_version_slug: str): + if default_model_version_slug is None: + del self.default_model_version_slug return - if not isinstance(env_variables, list): - raise TypeError('env_variables must be of type list') - if not all([isinstance(t, EnvVariable) for t in env_variables]): - raise TypeError('env_variables must contain only items of type EnvVariable') - self._env_variables = env_variables + if not isinstance(default_model_version_slug, str): + raise TypeError('default_model_version_slug must be of type str') + del self.default_agent_slug + del self.default_model_version_config_slug + self._default_model_version_slug = default_model_version_slug @property - def secrets(self) -> Optional[List[Optional['UserSecret']]]: - """Optional. Create/update UserSecrets inline before running.""" - return self._secrets + def default_agent_slug(self) -> str: + """Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high').""" + return self._default_agent_slug or "" - @secrets.setter - def secrets(self, secrets: Optional[List[Optional['UserSecret']]]): - if secrets is None: - del self.secrets + @default_agent_slug.setter + def default_agent_slug(self, default_agent_slug: str): + if default_agent_slug is None: + del self.default_agent_slug return - if not isinstance(secrets, list): - raise TypeError('secrets must be of type list') - if not all([isinstance(t, UserSecret) for t in secrets]): - raise TypeError('secrets must contain only items of type UserSecret') - self._secrets = secrets + if not isinstance(default_agent_slug, str): + raise TypeError('default_agent_slug must be of type str') + del self.default_model_version_slug + del self.default_model_version_config_slug + self._default_agent_slug = default_agent_slug @property - def use_accelerator(self) -> bool: + def default_model_version_config_slug(self) -> str: r""" - Optional. Route the Harbor session to an L4-GPU exeunit pool. Ignored - unless the caller has the BENCHMARKS_ALLOW_USE_ACCELERATOR feature flag. + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. Harbor tasks only; requires + override_harness_version. """ - return self._use_accelerator + return self._default_model_version_config_slug or "" - @use_accelerator.setter - def use_accelerator(self, use_accelerator: bool): - if use_accelerator is None: - del self.use_accelerator + @default_model_version_config_slug.setter + def default_model_version_config_slug(self, default_model_version_config_slug: str): + if default_model_version_config_slug is None: + del self.default_model_version_config_slug return - if not isinstance(use_accelerator, bool): - raise TypeError('use_accelerator must be of type bool') - self._use_accelerator = use_accelerator - + if not isinstance(default_model_version_config_slug, str): + raise TypeError('default_model_version_config_slug must be of type str') + del self.default_model_version_slug + del self.default_agent_slug + self._default_model_version_config_slug = default_model_version_config_slug -class BenchmarkTaskNotebookDefinition(KaggleObject): - r""" - TODO(bml): Get parity with text + options fields in the create request proto - - """ - - pass ApiBatchScheduleBenchmarkTaskRunsRequest._fields = [ FieldMetadata("taskSlugs", "task_slugs", "_task_slugs", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), FieldMetadata("modelVersionSlugs", "model_version_slugs", "_model_version_slugs", str, [], ListSerializer(PredefinedSerializer())), FieldMetadata("agentSlugs", "agent_slugs", "_agent_slugs", str, [], ListSerializer(PredefinedSerializer())), + FieldMetadata("reasoningEffort", "reasoning_effort", "_reasoning_effort", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("enqueue", "enqueue", "_enqueue", bool, None, PredefinedSerializer(), optional=True), + FieldMetadata("skipPolicy", "skip_policy", "_skip_policy", BenchmarkEnqueueSkipPolicy, BenchmarkEnqueueSkipPolicy.BENCHMARK_ENQUEUE_SKIP_POLICY_UNSPECIFIED, EnumSerializer()), + FieldMetadata("staleAfter", "stale_after", "_stale_after", timedelta, None, TimeDeltaSerializer(), optional=True), + FieldMetadata("benchmark", "benchmark", "_benchmark", ApiBenchmarkSlug, None, KaggleObjectSerializer()), ] ApiBatchScheduleBenchmarkTaskRunsResponse._fields = [ FieldMetadata("results", "results", "_results", BatchScheduleBenchmarkModelVersionResult, [], ListSerializer(KaggleObjectSerializer())), ] +ApiBenchmarkSlug._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), +] + ApiBenchmarkTask._fields = [ FieldMetadata("slug", "slug", "_slug", ApiBenchmarkTaskSlug, None, KaggleObjectSerializer()), FieldMetadata("url", "url", "_url", str, "", PredefinedSerializer()), @@ -2074,6 +2847,12 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("tags", "tags", "_tags", str, [], ListSerializer(PredefinedSerializer())), FieldMetadata("provenanceSources", "provenance_sources", "_provenance_sources", str, None, PredefinedSerializer(), optional=True), FieldMetadata("citations", "citations", "_citations", ApiBenchmarkTaskCitation, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("candidateType", "candidate_type", "_candidate_type", BenchmarkCandidateType, BenchmarkCandidateType.BENCHMARK_CANDIDATE_TYPE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("definition", "definition", "_definition", BenchmarkTaskDefinition, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("definitionPartial", "definition_partial", "_definition_partial", bool, False, PredefinedSerializer()), + FieldMetadata("artifactPath", "artifact_path", "_artifact_path", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("artifactType", "artifact_type", "_artifact_type", BenchmarkArenaArtifactType, None, EnumSerializer(), optional=True), + FieldMetadata("areRunArtifactsPublic", "are_run_artifacts_public", "_are_run_artifacts_public", bool, None, PredefinedSerializer(), optional=True), ] ApiBenchmarkTaskCitation._fields = [ @@ -2089,6 +2868,18 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("startTime", "start_time", "_start_time", datetime, None, DateTimeSerializer(), optional=True), FieldMetadata("endTime", "end_time", "_end_time", datetime, None, DateTimeSerializer(), optional=True), FieldMetadata("errorMessage", "error_message", "_error_message", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("agentSlug", "agent_slug", "_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("modelVersionConfigSlug", "model_version_config_slug", "_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("result", "result", "_result", ApiBenchmarkTaskRunResult, None, KaggleObjectSerializer(), optional=True), +] + +ApiBenchmarkTaskRunIds._fields = [ + FieldMetadata("ids", "ids", "_ids", int, [], ListSerializer(PredefinedSerializer())), +] + +ApiBenchmarkTaskRunResult._fields = [ + FieldMetadata("numericResult", "numeric_result", "_numeric_result", float, None, PredefinedSerializer(), optional=True), + FieldMetadata("booleanResult", "boolean_result", "_boolean_result", bool, None, PredefinedSerializer(), optional=True), ] ApiBenchmarkTaskSlug._fields = [ @@ -2097,11 +2888,29 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), ] +ApiBenchmarkTaskSlugs._fields = [ + FieldMetadata("slugs", "slugs", "_slugs", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), +] + +ApiCancelBenchmarkTaskRunsRequest._fields = [ + FieldMetadata("taskSlugs", "task_slugs", "_task_slugs", ApiBenchmarkTaskSlugs, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("modelVersion", "model_version", "_model_version", ApiModelVersionCandidate, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("agentSlug", "agent_slug", "_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("includeActive", "include_active", "_include_active", bool, False, PredefinedSerializer()), + FieldMetadata("runIds", "run_ids", "_run_ids", ApiBenchmarkTaskRunIds, None, KaggleObjectSerializer(), optional=True), +] + +ApiCancelBenchmarkTaskRunsResponse._fields = [ + FieldMetadata("queuedCancelledCount", "queued_cancelled_count", "_queued_cancelled_count", int, 0, PredefinedSerializer()), + FieldMetadata("activeTerminatedCount", "active_terminated_count", "_active_terminated_count", int, 0, PredefinedSerializer()), +] + ApiCreateBenchmarkTaskRequest._fields = [ FieldMetadata("slug", "slug", "_slug", str, "", PredefinedSerializer()), FieldMetadata("text", "text", "_text", str, "", PredefinedSerializer()), FieldMetadata("options", "options", "_options", BenchmarkTaskOptions, None, KaggleObjectSerializer(), optional=True), FieldMetadata("definition", "definition", "_definition", BenchmarkTaskDefinition, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("validationConfig", "validation_config", "_validation_config", ApiTaskCreationValidationConfig, None, KaggleObjectSerializer(), optional=True), ] ApiDownloadBenchmarkTaskRunOutputRequest._fields = [ @@ -2109,6 +2918,8 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("includeSource", "include_source", "_include_source", bool, False, PredefinedSerializer()), ] +ApiGetBenchmarkSchedulerStatusRequest._fields = [] + ApiGetBenchmarkTaskQuotaRequest._fields = [] ApiGetBenchmarkTaskQuotaResponse._fields = [ @@ -2118,6 +2929,7 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): ApiGetBenchmarkTaskRequest._fields = [ FieldMetadata("slug", "slug", "_slug", ApiBenchmarkTaskSlug, None, KaggleObjectSerializer()), + FieldMetadata("includeDefinition", "include_definition", "_include_definition", bool, False, PredefinedSerializer()), ] ApiGetBenchmarkTaskRunLogsRequest._fields = [ @@ -2144,6 +2956,10 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("pageSize", "page_size", "_page_size", int, 0, PredefinedSerializer()), FieldMetadata("pageToken", "page_token", "_page_token", str, "", PredefinedSerializer()), FieldMetadata("skip", "skip", "_skip", int, 0, PredefinedSerializer()), + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("includeDefinition", "include_definition", "_include_definition", bool, False, PredefinedSerializer()), ] ApiListBenchmarkTasksResponse._fields = [ @@ -2152,9 +2968,19 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("nextPageToken", "next_page_token", "_next_page_token", str, "", PredefinedSerializer()), ] +ApiModelVersionCandidate._fields = [ + FieldMetadata("modelVersionSlug", "model_version_slug", "_model_version_slug", str, "", PredefinedSerializer()), + FieldMetadata("reasoningEffort", "reasoning_effort", "_reasoning_effort", str, None, PredefinedSerializer(), optional=True), +] + ApiPublishBenchmarkTaskRequest._fields = [ FieldMetadata("slug", "slug", "_slug", ApiBenchmarkTaskSlug, None, KaggleObjectSerializer()), FieldMetadata("publishBackingNotebook", "publish_backing_notebook", "_publish_backing_notebook", bool, False, PredefinedSerializer()), + FieldMetadata("publishRunArtifacts", "publish_run_artifacts", "_publish_run_artifacts", bool, False, PredefinedSerializer()), +] + +ApiTaskCreationValidationConfig._fields = [ + FieldMetadata("validationStages", "validation_stages", "_validation_stages", ApiValidationStage, [], ListSerializer(KaggleObjectSerializer())), ] ApiUpdateBenchmarkTaskRequest._fields = [ @@ -2177,32 +3003,14 @@ class BenchmarkTaskNotebookDefinition(KaggleObject): FieldMetadata("provenanceSources", "provenance_sources", "_provenance_sources", str, None, PredefinedSerializer(), optional=True), FieldMetadata("citations", "citations", "_citations", ApiBenchmarkTaskCitation, [], ListSerializer(KaggleObjectSerializer())), FieldMetadata("updateMask", "update_mask", "_update_mask", FieldMask, None, FieldMaskSerializer()), + FieldMetadata("artifactPath", "artifact_path", "_artifact_path", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("artifactType", "artifact_type", "_artifact_type", BenchmarkArenaArtifactType, None, EnumSerializer(), optional=True), ] -BenchmarkTaskDefinition._fields = [ - FieldMetadata("notebook", "notebook", "_notebook", BenchmarkTaskNotebookDefinition, None, KaggleObjectSerializer(), optional=True), - FieldMetadata("dockerImage", "docker_image", "_docker_image", BenchmarkTaskDockerImageDefinition, None, KaggleObjectSerializer(), optional=True), - FieldMetadata("harborGit", "harbor_git", "_harbor_git", BenchmarkTaskGitDefinition, None, KaggleObjectSerializer(), optional=True), - FieldMetadata("harborKaggleDatasets", "harbor_kaggle_datasets", "_harbor_kaggle_datasets", BenchmarkTaskKaggleDatasetsDefinition, None, KaggleObjectSerializer(), optional=True), +ApiValidationStage._fields = [ + FieldMetadata("type", "type", "_type", BenchmarkTaskValidationStage, BenchmarkTaskValidationStage.BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("defaultModelVersionSlug", "default_model_version_slug", "_default_model_version_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultAgentSlug", "default_agent_slug", "_default_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultModelVersionConfigSlug", "default_model_version_config_slug", "_default_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), ] -BenchmarkTaskDockerImageDefinition._fields = [ - FieldMetadata("dockerImageUrl", "docker_image_url", "_docker_image_url", str, "", PredefinedSerializer()), - FieldMetadata("description", "description", "_description", str, None, PredefinedSerializer(), optional=True), - FieldMetadata("title", "title", "_title", str, None, PredefinedSerializer(), optional=True), - FieldMetadata("envVariables", "env_variables", "_env_variables", EnvVariable, [], ListSerializer(KaggleObjectSerializer())), - FieldMetadata("secrets", "secrets", "_secrets", UserSecret, [], ListSerializer(KaggleObjectSerializer())), - FieldMetadata("nestedVirtualizationRequired", "nested_virtualization_required", "_nested_virtualization_required", bool, False, PredefinedSerializer()), -] - -BenchmarkTaskGitDefinition._fields = [ - FieldMetadata("url", "url", "_url", str, "", PredefinedSerializer()), - FieldMetadata("commitHash", "commit_hash", "_commit_hash", str, "", PredefinedSerializer()), - FieldMetadata("subpath", "subpath", "_subpath", str, "", PredefinedSerializer()), - FieldMetadata("envVariables", "env_variables", "_env_variables", EnvVariable, [], ListSerializer(KaggleObjectSerializer())), - FieldMetadata("secrets", "secrets", "_secrets", UserSecret, [], ListSerializer(KaggleObjectSerializer())), - FieldMetadata("useAccelerator", "use_accelerator", "_use_accelerator", bool, False, PredefinedSerializer()), -] - -BenchmarkTaskNotebookDefinition._fields = [] - diff --git a/kagglesdk/benchmarks/types/benchmark_types.py b/kagglesdk/benchmarks/types/benchmark_types.py index ac94ed5..16fb1c9 100644 --- a/kagglesdk/benchmarks/types/benchmark_types.py +++ b/kagglesdk/benchmarks/types/benchmark_types.py @@ -1,10 +1,340 @@ from datetime import datetime -from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkModelImportanceLevel, Modality +import enum +from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkArenaArtifactType, BenchmarkGalleryCardSize, BenchmarkModelImportanceLevel, Modality from kagglesdk.kaggle_object import * from kagglesdk.licenses.types.licenses_types import License from kagglesdk.users.types.legacy_organizations_service import OrganizationCard from typing import Optional, List +class BenchmarkTaskValidationStage(enum.Enum): + r""" + A stage run against a new task version before it is marked completed. Saved + to the DB. Do not modify existing values. + """ + BENCHMARK_TASK_VALIDATION_STAGE_UNSPECIFIED = 0 + BENCHMARK_TASK_VALIDATION_STAGE_NOP = 1 + r""" + Harbor's nop agent, which takes no action. Passes when no attempt earns a + reward, catching tasks whose tests pass on an untouched environment. + """ + BENCHMARK_TASK_VALIDATION_STAGE_ORACLE = 2 + r""" + Harbor's oracle agent, which runs the task's solution script. Passes when + every attempt earns a reward of 1. + """ + BENCHMARK_TASK_VALIDATION_STAGE_DEFAULT_CANDIDATE = 3 + """The task version's default candidate. Passes when its run completes.""" + +class HarborMetricType(enum.Enum): + r""" + Canonical metric reducers shipped by Harbor (minus uv-script), used to + resolve a task version's N attempts into aggregate values (e.g. for pass@K + scoring). Saved to the DB. Do not modify existing values. + """ + HARBOR_METRIC_TYPE_UNSPECIFIED = 0 + HARBOR_METRIC_TYPE_SUM = 1 + HARBOR_METRIC_TYPE_MIN = 2 + HARBOR_METRIC_TYPE_MAX = 3 + HARBOR_METRIC_TYPE_MEAN = 4 + +class BenchmarkArenaCombination(KaggleObject): + r""" + A (task version, candidate) combination opted into a benchmark version's + arena. The candidate matches the task version's candidate type. + + Attributes: + task_version_id (int) + candidate (BenchmarkCandidate) + reasoning_effort is not supported; pass the config instead. + """ + + def __init__(self): + self._task_version_id = 0 + self._candidate = None + self._freeze() + + @property + def task_version_id(self) -> int: + return self._task_version_id + + @task_version_id.setter + def task_version_id(self, task_version_id: int): + if task_version_id is None: + del self.task_version_id + return + if not isinstance(task_version_id, int): + raise TypeError('task_version_id must be of type int') + self._task_version_id = task_version_id + + @property + def candidate(self) -> Optional['BenchmarkCandidate']: + """reasoning_effort is not supported; pass the config instead.""" + return self._candidate + + @candidate.setter + def candidate(self, candidate: Optional['BenchmarkCandidate']): + if candidate is None: + del self.candidate + return + if not isinstance(candidate, BenchmarkCandidate): + raise TypeError('candidate must be of type BenchmarkCandidate') + self._candidate = candidate + + +class BenchmarkArenaSettings(KaggleObject): + r""" + Human-vote arena over a benchmark version's task run artifacts. + + Attributes: + artifact_path (str) + Glob, relative to a run's session output root, selecting the artifact, + e.g. '*.mp4'. + artifact_type (BenchmarkArenaArtifactType) + reveal_candidates (bool) + Show candidate names after a vote. Off keeps voting fully blind. + voting_enabled (bool) + Kill switches for the arena's public surfaces. Off when the arena is + created. + gallery_enabled (bool) + arena_task_version_id (int) + The task version the arena's overall Elo is written to, shown as a + leaderboard column. Created once the arena has votes to rank. + gallery_card_size (BenchmarkGalleryCardSize) + expand_artifacts (bool) + Clicking an artifact opens it full size, rather than voting for it in the + arena. HTML artifacts always do; this extends it to every type, for outputs + too detailed to judge at card size. + """ + + def __init__(self): + self._artifact_path = "" + self._artifact_type = BenchmarkArenaArtifactType.BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED + self._reveal_candidates = False + self._voting_enabled = False + self._gallery_enabled = False + self._arena_task_version_id = None + self._gallery_card_size = BenchmarkGalleryCardSize.BENCHMARK_GALLERY_CARD_SIZE_UNSPECIFIED + self._expand_artifacts = False + self._freeze() + + @property + def artifact_path(self) -> str: + r""" + Glob, relative to a run's session output root, selecting the artifact, + e.g. '*.mp4'. + """ + return self._artifact_path + + @artifact_path.setter + def artifact_path(self, artifact_path: str): + if artifact_path is None: + del self.artifact_path + return + if not isinstance(artifact_path, str): + raise TypeError('artifact_path must be of type str') + self._artifact_path = artifact_path + + @property + def artifact_type(self) -> 'BenchmarkArenaArtifactType': + return self._artifact_type + + @artifact_type.setter + def artifact_type(self, artifact_type: 'BenchmarkArenaArtifactType'): + if artifact_type is None: + del self.artifact_type + return + if not isinstance(artifact_type, BenchmarkArenaArtifactType): + raise TypeError('artifact_type must be of type BenchmarkArenaArtifactType') + self._artifact_type = artifact_type + + @property + def reveal_candidates(self) -> bool: + """Show candidate names after a vote. Off keeps voting fully blind.""" + return self._reveal_candidates + + @reveal_candidates.setter + def reveal_candidates(self, reveal_candidates: bool): + if reveal_candidates is None: + del self.reveal_candidates + return + if not isinstance(reveal_candidates, bool): + raise TypeError('reveal_candidates must be of type bool') + self._reveal_candidates = reveal_candidates + + @property + def voting_enabled(self) -> bool: + r""" + Kill switches for the arena's public surfaces. Off when the arena is + created. + """ + return self._voting_enabled + + @voting_enabled.setter + def voting_enabled(self, voting_enabled: bool): + if voting_enabled is None: + del self.voting_enabled + return + if not isinstance(voting_enabled, bool): + raise TypeError('voting_enabled must be of type bool') + self._voting_enabled = voting_enabled + + @property + def gallery_enabled(self) -> bool: + return self._gallery_enabled + + @gallery_enabled.setter + def gallery_enabled(self, gallery_enabled: bool): + if gallery_enabled is None: + del self.gallery_enabled + return + if not isinstance(gallery_enabled, bool): + raise TypeError('gallery_enabled must be of type bool') + self._gallery_enabled = gallery_enabled + + @property + def arena_task_version_id(self) -> int: + r""" + The task version the arena's overall Elo is written to, shown as a + leaderboard column. Created once the arena has votes to rank. + """ + return self._arena_task_version_id or 0 + + @arena_task_version_id.setter + def arena_task_version_id(self, arena_task_version_id: Optional[int]): + if arena_task_version_id is None: + del self.arena_task_version_id + return + if not isinstance(arena_task_version_id, int): + raise TypeError('arena_task_version_id must be of type int') + self._arena_task_version_id = arena_task_version_id + + @property + def gallery_card_size(self) -> 'BenchmarkGalleryCardSize': + return self._gallery_card_size + + @gallery_card_size.setter + def gallery_card_size(self, gallery_card_size: 'BenchmarkGalleryCardSize'): + if gallery_card_size is None: + del self.gallery_card_size + return + if not isinstance(gallery_card_size, BenchmarkGalleryCardSize): + raise TypeError('gallery_card_size must be of type BenchmarkGalleryCardSize') + self._gallery_card_size = gallery_card_size + + @property + def expand_artifacts(self) -> bool: + r""" + Clicking an artifact opens it full size, rather than voting for it in the + arena. HTML artifacts always do; this extends it to every type, for outputs + too detailed to judge at card size. + """ + return self._expand_artifacts + + @expand_artifacts.setter + def expand_artifacts(self, expand_artifacts: bool): + if expand_artifacts is None: + del self.expand_artifacts + return + if not isinstance(expand_artifacts, bool): + raise TypeError('expand_artifacts must be of type bool') + self._expand_artifacts = expand_artifacts + + +class BenchmarkCandidate(KaggleObject): + r""" + One contestant to run a benchmark against. A config can be named either by + its id or as its model version plus reasoning_effort, because the public API + knows an effort string while the admin UI already holds config ids. + + Attributes: + benchmark_model_version_id (int) + The model version to run against. + agent_id (int) + The agent to run against. Agents are backed by a BenchmarkModelVersion. + reasoning_effort (str) + Reasoning effort to evaluate the model version at (e.g. 'high'), resolved + to the model version's config at that effort; no such config is rejected. + Only valid alongside benchmark_model_version_id, since an agent and a + config each already pin their own. + benchmark_model_version_config_id (int) + The config to run against, which pins its own model version and effort. + """ + + def __init__(self): + self._benchmark_model_version_id = None + self._agent_id = None + self._reasoning_effort = None + self._benchmark_model_version_config_id = None + self._freeze() + + @property + def benchmark_model_version_id(self) -> int: + """The model version to run against.""" + return self._benchmark_model_version_id or 0 + + @benchmark_model_version_id.setter + def benchmark_model_version_id(self, benchmark_model_version_id: int): + if benchmark_model_version_id is None: + del self.benchmark_model_version_id + return + if not isinstance(benchmark_model_version_id, int): + raise TypeError('benchmark_model_version_id must be of type int') + del self.agent_id + del self.benchmark_model_version_config_id + self._benchmark_model_version_id = benchmark_model_version_id + + @property + def agent_id(self) -> int: + """The agent to run against. Agents are backed by a BenchmarkModelVersion.""" + return self._agent_id or 0 + + @agent_id.setter + def agent_id(self, agent_id: int): + if agent_id is None: + del self.agent_id + return + if not isinstance(agent_id, int): + raise TypeError('agent_id must be of type int') + del self.benchmark_model_version_id + del self.benchmark_model_version_config_id + self._agent_id = agent_id + + @property + def benchmark_model_version_config_id(self) -> int: + """The config to run against, which pins its own model version and effort.""" + return self._benchmark_model_version_config_id or 0 + + @benchmark_model_version_config_id.setter + def benchmark_model_version_config_id(self, benchmark_model_version_config_id: int): + if benchmark_model_version_config_id is None: + del self.benchmark_model_version_config_id + return + if not isinstance(benchmark_model_version_config_id, int): + raise TypeError('benchmark_model_version_config_id must be of type int') + del self.benchmark_model_version_id + del self.agent_id + self._benchmark_model_version_config_id = benchmark_model_version_config_id + + @property + def reasoning_effort(self) -> str: + r""" + Reasoning effort to evaluate the model version at (e.g. 'high'), resolved + to the model version's config at that effort; no such config is rejected. + Only valid alongside benchmark_model_version_id, since an agent and a + config each already pin their own. + """ + return self._reasoning_effort or "" + + @reasoning_effort.setter + def reasoning_effort(self, reasoning_effort: Optional[str]): + if reasoning_effort is None: + del self.reasoning_effort + return + if not isinstance(reasoning_effort, str): + raise TypeError('reasoning_effort must be of type str') + self._reasoning_effort = reasoning_effort + + class BenchmarkModel(KaggleObject): r""" Attributes: @@ -258,6 +588,16 @@ class BenchmarkModelVersion(KaggleObject): deprecation_time (datetime) Timestamp when this model version was deprecated. A future value indicates a scheduled deprecation. Null/unset means not deprecated. + benchmark_model_slug (str) + The slug of the parent benchmark model. ex: 'gemini-1.5-pro'. Distinct from + `slug` above, which identifies the version. + supports_reasoning_effort (bool) + Whether the model takes a reasoning effort. Its configs must pin one + unless this is false. Unset reads as true. + release_date (datetime) + When the vendor released this model version. Distinct from the row's + CreateTime (Kaggle onboarding), which lags by weeks to months for + third-party and open-weight models. Unset when unknown. """ def __init__(self): @@ -279,6 +619,9 @@ def __init__(self): self._input_modalities = [] self._output_modalities = [] self._deprecation_time = None + self._benchmark_model_slug = None + self._supports_reasoning_effort = None + self._release_date = None self._freeze() @property @@ -551,6 +894,58 @@ def deprecation_time(self, deprecation_time: Optional[datetime]): raise TypeError('deprecation_time must be of type datetime') self._deprecation_time = deprecation_time + @property + def benchmark_model_slug(self) -> str: + r""" + The slug of the parent benchmark model. ex: 'gemini-1.5-pro'. Distinct from + `slug` above, which identifies the version. + """ + return self._benchmark_model_slug or "" + + @benchmark_model_slug.setter + def benchmark_model_slug(self, benchmark_model_slug: Optional[str]): + if benchmark_model_slug is None: + del self.benchmark_model_slug + return + if not isinstance(benchmark_model_slug, str): + raise TypeError('benchmark_model_slug must be of type str') + self._benchmark_model_slug = benchmark_model_slug + + @property + def supports_reasoning_effort(self) -> bool: + r""" + Whether the model takes a reasoning effort. Its configs must pin one + unless this is false. Unset reads as true. + """ + return self._supports_reasoning_effort or False + + @supports_reasoning_effort.setter + def supports_reasoning_effort(self, supports_reasoning_effort: Optional[bool]): + if supports_reasoning_effort is None: + del self.supports_reasoning_effort + return + if not isinstance(supports_reasoning_effort, bool): + raise TypeError('supports_reasoning_effort must be of type bool') + self._supports_reasoning_effort = supports_reasoning_effort + + @property + def release_date(self) -> datetime: + r""" + When the vendor released this model version. Distinct from the row's + CreateTime (Kaggle onboarding), which lags by weeks to months for + third-party and open-weight models. Unset when unknown. + """ + return self._release_date or None + + @release_date.setter + def release_date(self, release_date: Optional[datetime]): + if release_date is None: + del self.release_date + return + if not isinstance(release_date, datetime): + raise TypeError('release_date must be of type datetime') + self._release_date = release_date + class BenchmarkResult(KaggleObject): r""" @@ -683,160 +1078,699 @@ def evaluation_date(self, evaluation_date: Optional[datetime]): self._evaluation_date = evaluation_date -class BenchmarkTaskKaggleDatasetsDefinition(KaggleObject): +class BenchmarkSchedulerSettings(KaggleObject): r""" - Task definition and asset mounts for a Harbor task whose definition and data - live in mounted Kaggle datasets instead of a git clone. Dataset references - are pinned to an immutable version for reproducibility. + Global settings for the benchmark task-run scheduler, stored in a single + row (Id = 1). The concurrency caps count Running runs that went through the + queue; synchronously started runs don't count. + + Attributes: + enabled (bool) + Kill switch. When false, the drain dispatches nothing and runs stay Queued. + max_concurrency (int) + Fleet-wide cap on Running runs. Unset = unbounded. + max_concurrency_per_candidate_model (int) + Cap on Running runs per candidate model version. Unset = unbounded. + max_kickoffs_per_tick (int) + Max runs one drain tick may dispatch. Unset = unbounded, though Model + Proxy's batch limit also applies when reservations are enabled. + dispatch_concurrency (int) + How many dispatches a tick runs concurrently. Unset = the drain's default; + values below 1 are treated as 1. + pool_batch_size (int) + How many Queued runs a tick fetches to consider. Unset = the drain's + default; values below 1 are treated as 1. + stale_queued_ttl_hours (int) + How long a run may sit Queued before the stale-GC cron errors it out, + measured from its enqueue time. Unset = skip the Queued sweep entirely. + stale_running_ttl_hours (int) + How long a run may sit Running before the stale-GC cron cancels its session + and errors it out, measured from its start time. Set this above the longest + legitimate task phase timeout or the sweep will kill healthy long-running + work. Unset = skip the Running sweep entirely. + reservations_enabled (bool) + Whether the drain reserves Model Proxy capacity before starting each run. + max_retries (int) + How many cron-driven re-queues a task version + candidate may accumulate, + not counting the original attempt. It does not bound the drain's in-place + requeue of a transient dispatch failure, which leaves the count alone. + Unset, or 0, disables retries entirely -- this is the retry kill-switch. + retry_cooldown_minutes (int) + How long a failed run must sit before the retry cron will re-queue it, + measured from its end time. Unset = no cooldown, so a failure is eligible + on the next tick. + max_retries_per_tick (int) + Upper bound on how many runs a single retry tick may re-queue. Like + dispatch_concurrency, unset is not 'unbounded' but 'use the cron's + built-in default'. + screening_enabled (bool) + Whether the drain asks Model Proxy which models have capacity before + admitting runs. Turn off during a screening outage, which otherwise defers + every run. Optional because it defaults to on, so a client that omits it + doesn't disable it. + global_min_model_proxy_priority (float) + Minimum priority value sent to Model Proxy for every run. 0.0 is most + urgent, so this caps how urgent a run can be. Unset = no floor. + """ + + def __init__(self): + self._enabled = False + self._max_concurrency = None + self._max_concurrency_per_candidate_model = None + self._max_kickoffs_per_tick = None + self._dispatch_concurrency = None + self._pool_batch_size = None + self._stale_queued_ttl_hours = None + self._stale_running_ttl_hours = None + self._reservations_enabled = False + self._max_retries = None + self._retry_cooldown_minutes = None + self._max_retries_per_tick = None + self._screening_enabled = None + self._global_min_model_proxy_priority = None + self._freeze() + + @property + def enabled(self) -> bool: + """Kill switch. When false, the drain dispatches nothing and runs stay Queued.""" + return self._enabled + + @enabled.setter + def enabled(self, enabled: bool): + if enabled is None: + del self.enabled + return + if not isinstance(enabled, bool): + raise TypeError('enabled must be of type bool') + self._enabled = enabled + + @property + def max_concurrency(self) -> int: + """Fleet-wide cap on Running runs. Unset = unbounded.""" + return self._max_concurrency or 0 + + @max_concurrency.setter + def max_concurrency(self, max_concurrency: Optional[int]): + if max_concurrency is None: + del self.max_concurrency + return + if not isinstance(max_concurrency, int): + raise TypeError('max_concurrency must be of type int') + self._max_concurrency = max_concurrency + + @property + def max_concurrency_per_candidate_model(self) -> int: + """Cap on Running runs per candidate model version. Unset = unbounded.""" + return self._max_concurrency_per_candidate_model or 0 + + @max_concurrency_per_candidate_model.setter + def max_concurrency_per_candidate_model(self, max_concurrency_per_candidate_model: Optional[int]): + if max_concurrency_per_candidate_model is None: + del self.max_concurrency_per_candidate_model + return + if not isinstance(max_concurrency_per_candidate_model, int): + raise TypeError('max_concurrency_per_candidate_model must be of type int') + self._max_concurrency_per_candidate_model = max_concurrency_per_candidate_model + + @property + def max_kickoffs_per_tick(self) -> int: + r""" + Max runs one drain tick may dispatch. Unset = unbounded, though Model + Proxy's batch limit also applies when reservations are enabled. + """ + return self._max_kickoffs_per_tick or 0 + + @max_kickoffs_per_tick.setter + def max_kickoffs_per_tick(self, max_kickoffs_per_tick: Optional[int]): + if max_kickoffs_per_tick is None: + del self.max_kickoffs_per_tick + return + if not isinstance(max_kickoffs_per_tick, int): + raise TypeError('max_kickoffs_per_tick must be of type int') + self._max_kickoffs_per_tick = max_kickoffs_per_tick + + @property + def dispatch_concurrency(self) -> int: + r""" + How many dispatches a tick runs concurrently. Unset = the drain's default; + values below 1 are treated as 1. + """ + return self._dispatch_concurrency or 0 + + @dispatch_concurrency.setter + def dispatch_concurrency(self, dispatch_concurrency: Optional[int]): + if dispatch_concurrency is None: + del self.dispatch_concurrency + return + if not isinstance(dispatch_concurrency, int): + raise TypeError('dispatch_concurrency must be of type int') + self._dispatch_concurrency = dispatch_concurrency + + @property + def pool_batch_size(self) -> int: + r""" + How many Queued runs a tick fetches to consider. Unset = the drain's + default; values below 1 are treated as 1. + """ + return self._pool_batch_size or 0 + + @pool_batch_size.setter + def pool_batch_size(self, pool_batch_size: Optional[int]): + if pool_batch_size is None: + del self.pool_batch_size + return + if not isinstance(pool_batch_size, int): + raise TypeError('pool_batch_size must be of type int') + self._pool_batch_size = pool_batch_size + + @property + def stale_queued_ttl_hours(self) -> int: + r""" + How long a run may sit Queued before the stale-GC cron errors it out, + measured from its enqueue time. Unset = skip the Queued sweep entirely. + """ + return self._stale_queued_ttl_hours or 0 + + @stale_queued_ttl_hours.setter + def stale_queued_ttl_hours(self, stale_queued_ttl_hours: Optional[int]): + if stale_queued_ttl_hours is None: + del self.stale_queued_ttl_hours + return + if not isinstance(stale_queued_ttl_hours, int): + raise TypeError('stale_queued_ttl_hours must be of type int') + self._stale_queued_ttl_hours = stale_queued_ttl_hours + + @property + def stale_running_ttl_hours(self) -> int: + r""" + How long a run may sit Running before the stale-GC cron cancels its session + and errors it out, measured from its start time. Set this above the longest + legitimate task phase timeout or the sweep will kill healthy long-running + work. Unset = skip the Running sweep entirely. + """ + return self._stale_running_ttl_hours or 0 + + @stale_running_ttl_hours.setter + def stale_running_ttl_hours(self, stale_running_ttl_hours: Optional[int]): + if stale_running_ttl_hours is None: + del self.stale_running_ttl_hours + return + if not isinstance(stale_running_ttl_hours, int): + raise TypeError('stale_running_ttl_hours must be of type int') + self._stale_running_ttl_hours = stale_running_ttl_hours + + @property + def reservations_enabled(self) -> bool: + """Whether the drain reserves Model Proxy capacity before starting each run.""" + return self._reservations_enabled + + @reservations_enabled.setter + def reservations_enabled(self, reservations_enabled: bool): + if reservations_enabled is None: + del self.reservations_enabled + return + if not isinstance(reservations_enabled, bool): + raise TypeError('reservations_enabled must be of type bool') + self._reservations_enabled = reservations_enabled + + @property + def max_retries(self) -> int: + r""" + How many cron-driven re-queues a task version + candidate may accumulate, + not counting the original attempt. It does not bound the drain's in-place + requeue of a transient dispatch failure, which leaves the count alone. + Unset, or 0, disables retries entirely -- this is the retry kill-switch. + """ + return self._max_retries or 0 + + @max_retries.setter + def max_retries(self, max_retries: Optional[int]): + if max_retries is None: + del self.max_retries + return + if not isinstance(max_retries, int): + raise TypeError('max_retries must be of type int') + self._max_retries = max_retries + + @property + def retry_cooldown_minutes(self) -> int: + r""" + How long a failed run must sit before the retry cron will re-queue it, + measured from its end time. Unset = no cooldown, so a failure is eligible + on the next tick. + """ + return self._retry_cooldown_minutes or 0 + + @retry_cooldown_minutes.setter + def retry_cooldown_minutes(self, retry_cooldown_minutes: Optional[int]): + if retry_cooldown_minutes is None: + del self.retry_cooldown_minutes + return + if not isinstance(retry_cooldown_minutes, int): + raise TypeError('retry_cooldown_minutes must be of type int') + self._retry_cooldown_minutes = retry_cooldown_minutes + + @property + def max_retries_per_tick(self) -> int: + r""" + Upper bound on how many runs a single retry tick may re-queue. Like + dispatch_concurrency, unset is not 'unbounded' but 'use the cron's + built-in default'. + """ + return self._max_retries_per_tick or 0 + + @max_retries_per_tick.setter + def max_retries_per_tick(self, max_retries_per_tick: Optional[int]): + if max_retries_per_tick is None: + del self.max_retries_per_tick + return + if not isinstance(max_retries_per_tick, int): + raise TypeError('max_retries_per_tick must be of type int') + self._max_retries_per_tick = max_retries_per_tick + + @property + def screening_enabled(self) -> bool: + r""" + Whether the drain asks Model Proxy which models have capacity before + admitting runs. Turn off during a screening outage, which otherwise defers + every run. Optional because it defaults to on, so a client that omits it + doesn't disable it. + """ + return self._screening_enabled or False + + @screening_enabled.setter + def screening_enabled(self, screening_enabled: Optional[bool]): + if screening_enabled is None: + del self.screening_enabled + return + if not isinstance(screening_enabled, bool): + raise TypeError('screening_enabled must be of type bool') + self._screening_enabled = screening_enabled + + @property + def global_min_model_proxy_priority(self) -> float: + r""" + Minimum priority value sent to Model Proxy for every run. 0.0 is most + urgent, so this caps how urgent a run can be. Unset = no floor. + """ + return self._global_min_model_proxy_priority or 0.0 + + @global_min_model_proxy_priority.setter + def global_min_model_proxy_priority(self, global_min_model_proxy_priority: Optional[float]): + if global_min_model_proxy_priority is None: + del self.global_min_model_proxy_priority + return + if not isinstance(global_min_model_proxy_priority, float): + raise TypeError('global_min_model_proxy_priority must be of type float') + self._global_min_model_proxy_priority = global_min_model_proxy_priority + +class BenchmarkTaskDefinition(KaggleObject): + r""" Attributes: - definition_source (BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource) - mounts (BenchmarkTaskKaggleDatasetsDefinition.DatasetMount) + notebook (BenchmarkTaskNotebookDefinition) + docker_image (BenchmarkTaskDockerImageDefinition) + harbor_kaggle_datasets (BenchmarkTaskKaggleDatasetsDefinition) + """ + + def __init__(self): + self._notebook = None + self._docker_image = None + self._harbor_kaggle_datasets = None + self._freeze() + + @property + def notebook(self) -> Optional['BenchmarkTaskNotebookDefinition']: + return self._notebook or None + + @notebook.setter + def notebook(self, notebook: Optional['BenchmarkTaskNotebookDefinition']): + if notebook is None: + del self.notebook + return + if not isinstance(notebook, BenchmarkTaskNotebookDefinition): + raise TypeError('notebook must be of type BenchmarkTaskNotebookDefinition') + del self.docker_image + del self.harbor_kaggle_datasets + self._notebook = notebook + + @property + def docker_image(self) -> Optional['BenchmarkTaskDockerImageDefinition']: + return self._docker_image or None + + @docker_image.setter + def docker_image(self, docker_image: Optional['BenchmarkTaskDockerImageDefinition']): + if docker_image is None: + del self.docker_image + return + if not isinstance(docker_image, BenchmarkTaskDockerImageDefinition): + raise TypeError('docker_image must be of type BenchmarkTaskDockerImageDefinition') + del self.notebook + del self.harbor_kaggle_datasets + self._docker_image = docker_image + + @property + def harbor_kaggle_datasets(self) -> Optional['BenchmarkTaskKaggleDatasetsDefinition']: + return self._harbor_kaggle_datasets or None + + @harbor_kaggle_datasets.setter + def harbor_kaggle_datasets(self, harbor_kaggle_datasets: Optional['BenchmarkTaskKaggleDatasetsDefinition']): + if harbor_kaggle_datasets is None: + del self.harbor_kaggle_datasets + return + if not isinstance(harbor_kaggle_datasets, BenchmarkTaskKaggleDatasetsDefinition): + raise TypeError('harbor_kaggle_datasets must be of type BenchmarkTaskKaggleDatasetsDefinition') + del self.notebook + del self.docker_image + self._harbor_kaggle_datasets = harbor_kaggle_datasets + + +class BenchmarkTaskDockerImageDefinition(KaggleObject): + r""" + User-supplied Docker image task definition. + + Attributes: + docker_image_url (str) + Required. Full image ref (e.g. gcr.io/project/image:tag). + description (str) + Required on new tasks; ignored on version bumps (existing description + carries over). Task description. + title (str) + Optional. Display name for the task. Defaults to the slug when omitted. + Ignored on version bumps. env_variables (EnvVariable) - Optional. Environment variables to expose to the Harbor session container. - Keys the executor injects itself — the task-definition path, the agent - pin, and the model-proxy variables — are rejected with InvalidArgument. + Optional. Environment variables set on the container. secrets (UserSecret) Optional. Create/update UserSecrets inline before running. + nested_virtualization_required (bool) + Optional. Run the container with nested virtualization enabled — set + when the image needs to launch VMs (e.g. QEMU/KVM guests) inside its + sandbox. Ignored unless the caller has the + BENCHMARKS_ALLOW_NESTED_VIRTUALIZATION feature flag. + default_model_version_slug (str) + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + default_agent_slug (str) + Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high'). + auxiliary_model_version_config_slugs (str) + Optional. Canonical BenchmarkModelVersionConfig.slugs the task invokes + beyond the candidate (e.g. LLM judges), recorded so the scheduler can + account for their model-proxy demand. Slugs rather than ids because this + type is the public API input shape; the BenchmarkTasksApi handler resolves + them to the internal auxiliary_model_version_config_ids. + default_model_version_config_slug (str) + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. """ - class DatasetMount(KaggleObject): + def __init__(self): + self._docker_image_url = "" + self._description = None + self._title = None + self._env_variables = [] + self._secrets = [] + self._nested_virtualization_required = False + self._default_model_version_slug = None + self._default_agent_slug = None + self._auxiliary_model_version_config_slugs = [] + self._default_model_version_config_slug = None + self._freeze() + + @property + def docker_image_url(self) -> str: + """Required. Full image ref (e.g. gcr.io/project/image:tag).""" + return self._docker_image_url + + @docker_image_url.setter + def docker_image_url(self, docker_image_url: str): + if docker_image_url is None: + del self.docker_image_url + return + if not isinstance(docker_image_url, str): + raise TypeError('docker_image_url must be of type str') + self._docker_image_url = docker_image_url + + @property + def description(self) -> str: r""" - Attributes: - dataset_version_slug (str) - 'owner/slug/versions/N' — the version segment is REQUIRED for - reproducibility. Requests without an explicit version are rejected. - mount_path (str) - Absolute path inside the container where the dataset is mounted. Callers - pick this so tasks can locate their data anywhere. Unique across the - mount list; must not shadow the task-definition mount. + Required on new tasks; ignored on version bumps (existing description + carries over). Task description. """ + return self._description or "" - def __init__(self): - self._dataset_version_slug = "" - self._mount_path = "" - self._freeze() + @description.setter + def description(self, description: Optional[str]): + if description is None: + del self.description + return + if not isinstance(description, str): + raise TypeError('description must be of type str') + self._description = description - @property - def dataset_version_slug(self) -> str: - r""" - 'owner/slug/versions/N' — the version segment is REQUIRED for - reproducibility. Requests without an explicit version are rejected. - """ - return self._dataset_version_slug - - @dataset_version_slug.setter - def dataset_version_slug(self, dataset_version_slug: str): - if dataset_version_slug is None: - del self.dataset_version_slug - return - if not isinstance(dataset_version_slug, str): - raise TypeError('dataset_version_slug must be of type str') - self._dataset_version_slug = dataset_version_slug - - @property - def mount_path(self) -> str: - r""" - Absolute path inside the container where the dataset is mounted. Callers - pick this so tasks can locate their data anywhere. Unique across the - mount list; must not shadow the task-definition mount. - """ - return self._mount_path + @property + def title(self) -> str: + r""" + Optional. Display name for the task. Defaults to the slug when omitted. + Ignored on version bumps. + """ + return self._title or "" - @mount_path.setter - def mount_path(self, mount_path: str): - if mount_path is None: - del self.mount_path - return - if not isinstance(mount_path, str): - raise TypeError('mount_path must be of type str') - self._mount_path = mount_path + @title.setter + def title(self, title: Optional[str]): + if title is None: + del self.title + return + if not isinstance(title, str): + raise TypeError('title must be of type str') + self._title = title + @property + def env_variables(self) -> Optional[List[Optional['EnvVariable']]]: + """Optional. Environment variables set on the container.""" + return self._env_variables - class DefinitionSource(KaggleObject): + @env_variables.setter + def env_variables(self, env_variables: Optional[List[Optional['EnvVariable']]]): + if env_variables is None: + del self.env_variables + return + if not isinstance(env_variables, list): + raise TypeError('env_variables must be of type list') + if not all([isinstance(t, EnvVariable) for t in env_variables]): + raise TypeError('env_variables must contain only items of type EnvVariable') + self._env_variables = env_variables + + @property + def secrets(self) -> Optional[List[Optional['UserSecret']]]: + """Optional. Create/update UserSecrets inline before running.""" + return self._secrets + + @secrets.setter + def secrets(self, secrets: Optional[List[Optional['UserSecret']]]): + if secrets is None: + del self.secrets + return + if not isinstance(secrets, list): + raise TypeError('secrets must be of type list') + if not all([isinstance(t, UserSecret) for t in secrets]): + raise TypeError('secrets must contain only items of type UserSecret') + self._secrets = secrets + + @property + def nested_virtualization_required(self) -> bool: r""" - Attributes: - dataset_version_slug (str) - 'owner/slug/versions/N' — the version segment is REQUIRED for - reproducibility. Requests without an explicit version are rejected. - sub_path (str) - Sub-directory within the dataset containing task.toml. Empty means the - dataset root. + Optional. Run the container with nested virtualization enabled — set + when the image needs to launch VMs (e.g. QEMU/KVM guests) inside its + sandbox. Ignored unless the caller has the + BENCHMARKS_ALLOW_NESTED_VIRTUALIZATION feature flag. """ + return self._nested_virtualization_required - def __init__(self): - self._dataset_version_slug = "" - self._sub_path = "" - self._freeze() + @nested_virtualization_required.setter + def nested_virtualization_required(self, nested_virtualization_required: bool): + if nested_virtualization_required is None: + del self.nested_virtualization_required + return + if not isinstance(nested_virtualization_required, bool): + raise TypeError('nested_virtualization_required must be of type bool') + self._nested_virtualization_required = nested_virtualization_required - @property - def dataset_version_slug(self) -> str: - r""" - 'owner/slug/versions/N' — the version segment is REQUIRED for - reproducibility. Requests without an explicit version are rejected. - """ - return self._dataset_version_slug - - @dataset_version_slug.setter - def dataset_version_slug(self, dataset_version_slug: str): - if dataset_version_slug is None: - del self.dataset_version_slug - return - if not isinstance(dataset_version_slug, str): - raise TypeError('dataset_version_slug must be of type str') - self._dataset_version_slug = dataset_version_slug - - @property - def sub_path(self) -> str: - r""" - Sub-directory within the dataset containing task.toml. Empty means the - dataset root. - """ - return self._sub_path + @property + def default_model_version_slug(self) -> str: + """Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default').""" + return self._default_model_version_slug or "" + + @default_model_version_slug.setter + def default_model_version_slug(self, default_model_version_slug: str): + if default_model_version_slug is None: + del self.default_model_version_slug + return + if not isinstance(default_model_version_slug, str): + raise TypeError('default_model_version_slug must be of type str') + del self.default_agent_slug + del self.default_model_version_config_slug + self._default_model_version_slug = default_model_version_slug + + @property + def default_agent_slug(self) -> str: + """Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high').""" + return self._default_agent_slug or "" + + @default_agent_slug.setter + def default_agent_slug(self, default_agent_slug: str): + if default_agent_slug is None: + del self.default_agent_slug + return + if not isinstance(default_agent_slug, str): + raise TypeError('default_agent_slug must be of type str') + del self.default_model_version_slug + del self.default_model_version_config_slug + self._default_agent_slug = default_agent_slug + + @property + def default_model_version_config_slug(self) -> str: + r""" + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. + """ + return self._default_model_version_config_slug or "" + + @default_model_version_config_slug.setter + def default_model_version_config_slug(self, default_model_version_config_slug: str): + if default_model_version_config_slug is None: + del self.default_model_version_config_slug + return + if not isinstance(default_model_version_config_slug, str): + raise TypeError('default_model_version_config_slug must be of type str') + del self.default_model_version_slug + del self.default_agent_slug + self._default_model_version_config_slug = default_model_version_config_slug + + @property + def auxiliary_model_version_config_slugs(self) -> Optional[List[str]]: + r""" + Optional. Canonical BenchmarkModelVersionConfig.slugs the task invokes + beyond the candidate (e.g. LLM judges), recorded so the scheduler can + account for their model-proxy demand. Slugs rather than ids because this + type is the public API input shape; the BenchmarkTasksApi handler resolves + them to the internal auxiliary_model_version_config_ids. + """ + return self._auxiliary_model_version_config_slugs - @sub_path.setter - def sub_path(self, sub_path: str): - if sub_path is None: - del self.sub_path - return - if not isinstance(sub_path, str): - raise TypeError('sub_path must be of type str') - self._sub_path = sub_path + @auxiliary_model_version_config_slugs.setter + def auxiliary_model_version_config_slugs(self, auxiliary_model_version_config_slugs: Optional[List[str]]): + if auxiliary_model_version_config_slugs is None: + del self.auxiliary_model_version_config_slugs + return + if not isinstance(auxiliary_model_version_config_slugs, list): + raise TypeError('auxiliary_model_version_config_slugs must be of type list') + if not all([isinstance(t, str) for t in auxiliary_model_version_config_slugs]): + raise TypeError('auxiliary_model_version_config_slugs must contain only items of type str') + self._auxiliary_model_version_config_slugs = auxiliary_model_version_config_slugs +class BenchmarkTaskKaggleDatasetsDefinition(KaggleObject): + r""" + Task definition and asset mounts for a Harbor task whose definition and data + live in mounted Kaggle datasets instead of a git clone. Dataset references + are pinned to an immutable version for reproducibility. + + Attributes: + definition_source (DefinitionSource) + mounts (DatasetMount) + env_variables (EnvVariable) + Optional. Environment variables to expose to the Harbor session container. + Keys the executor injects itself — the task-definition path, the agent + pin, and the model-proxy variables — are rejected with InvalidArgument. + secrets (UserSecret) + Optional. Create/update UserSecrets inline before running. + override_harness_version_slug (str) + HarnessVersion.slug (e.g. 'claude-code-2.1.216'). Slug rather than id + because this type is the public API input shape; the BenchmarkTasksApi + handler resolves it to the internal override_harness_version_id. + default_model_version_slug (str) + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + Requires override_harness_version: without one the task evaluates + agents, so pick the model via default_agent_slug instead. + default_agent_slug (str) + Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high'). + override_harness_version_custom_slug (str) + Passed directly to the Harbor run command's --agent flag. + auxiliary_model_version_config_slugs (str) + Optional. Canonical BenchmarkModelVersionConfig.slugs the task invokes + beyond the candidate (e.g. LLM judges), recorded so the scheduler can + account for their model-proxy demand. Slugs rather than ids because this + type is the public API input shape; the BenchmarkTasksApi handler resolves + them to the internal auxiliary_model_version_config_ids. + nested_virtualization_required (bool) + Optional. Run the Harbor executor container with nested virtualization + enabled — set when the task needs to launch VMs (e.g. QEMU/KVM guests) + inside its sandbox. Ignored unless the caller has the + BENCHMARKS_ALLOW_NESTED_VIRTUALIZATION feature flag. + n_attempts (int) + Optional. How many times the task is run per invocation, mapped to the + Harbor run's n_attempts (used to compute metrics like pass@K). Defaults to + 1 when omitted. + metrics (HarborMetricType) + Optional. Canonical Harbor metrics the task version declares to reduce its + attempts into aggregate values, supplied to Harbor's JobConfig.Metrics when + the task is invoked. Duplicates are collapsed. When omitted, Harbor + defaults to mean. + session_concurrency (int) + Optional. Maximum number of the task version's attempts run concurrently + across VMs when n_attempts > 1. Defaults to 1 (serial) when omitted. + Rejected with PERMISSION_DENIED unless the caller has the + BENCHMARKS_ATTEMPTS_ACROSS_VMS feature flag. + default_model_version_config_slug (str) + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. Requires override_harness_version. + """ + def __init__(self): self._definition_source = None self._mounts = [] self._env_variables = [] self._secrets = [] + self._override_harness_version_slug = None + self._default_model_version_slug = None + self._default_agent_slug = None + self._override_harness_version_custom_slug = None + self._auxiliary_model_version_config_slugs = [] + self._nested_virtualization_required = False + self._n_attempts = None + self._metrics = [] + self._session_concurrency = None + self._default_model_version_config_slug = None self._freeze() @property - def definition_source(self) -> Optional['BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource']: + def definition_source(self) -> Optional['DefinitionSource']: return self._definition_source @definition_source.setter - def definition_source(self, definition_source: Optional['BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource']): + def definition_source(self, definition_source: Optional['DefinitionSource']): if definition_source is None: del self.definition_source return - if not isinstance(definition_source, BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource): - raise TypeError('definition_source must be of type BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource') + if not isinstance(definition_source, DefinitionSource): + raise TypeError('definition_source must be of type DefinitionSource') self._definition_source = definition_source @property - def mounts(self) -> Optional[List[Optional['BenchmarkTaskKaggleDatasetsDefinition.DatasetMount']]]: + def mounts(self) -> Optional[List[Optional['DatasetMount']]]: return self._mounts @mounts.setter - def mounts(self, mounts: Optional[List[Optional['BenchmarkTaskKaggleDatasetsDefinition.DatasetMount']]]): + def mounts(self, mounts: Optional[List[Optional['DatasetMount']]]): if mounts is None: del self.mounts return if not isinstance(mounts, list): raise TypeError('mounts must be of type list') - if not all([isinstance(t, BenchmarkTaskKaggleDatasetsDefinition.DatasetMount) for t in mounts]): - raise TypeError('mounts must contain only items of type BenchmarkTaskKaggleDatasetsDefinition.DatasetMount') + if not all([isinstance(t, DatasetMount) for t in mounts]): + raise TypeError('mounts must contain only items of type DatasetMount') self._mounts = mounts @property @@ -875,6 +1809,203 @@ def secrets(self, secrets: Optional[List[Optional['UserSecret']]]): raise TypeError('secrets must contain only items of type UserSecret') self._secrets = secrets + @property + def override_harness_version_slug(self) -> str: + r""" + HarnessVersion.slug (e.g. 'claude-code-2.1.216'). Slug rather than id + because this type is the public API input shape; the BenchmarkTasksApi + handler resolves it to the internal override_harness_version_id. + """ + return self._override_harness_version_slug or "" + + @override_harness_version_slug.setter + def override_harness_version_slug(self, override_harness_version_slug: str): + if override_harness_version_slug is None: + del self.override_harness_version_slug + return + if not isinstance(override_harness_version_slug, str): + raise TypeError('override_harness_version_slug must be of type str') + del self.override_harness_version_custom_slug + self._override_harness_version_slug = override_harness_version_slug + + @property + def override_harness_version_custom_slug(self) -> str: + """Passed directly to the Harbor run command's --agent flag.""" + return self._override_harness_version_custom_slug or "" + + @override_harness_version_custom_slug.setter + def override_harness_version_custom_slug(self, override_harness_version_custom_slug: str): + if override_harness_version_custom_slug is None: + del self.override_harness_version_custom_slug + return + if not isinstance(override_harness_version_custom_slug, str): + raise TypeError('override_harness_version_custom_slug must be of type str') + del self.override_harness_version_slug + self._override_harness_version_custom_slug = override_harness_version_custom_slug + + @property + def default_model_version_slug(self) -> str: + r""" + Canonical BenchmarkModelVersion.slug (e.g. 'claude-sonnet-4-6-default'). + Requires override_harness_version: without one the task evaluates + agents, so pick the model via default_agent_slug instead. + """ + return self._default_model_version_slug or "" + + @default_model_version_slug.setter + def default_model_version_slug(self, default_model_version_slug: str): + if default_model_version_slug is None: + del self.default_model_version_slug + return + if not isinstance(default_model_version_slug, str): + raise TypeError('default_model_version_slug must be of type str') + del self.default_agent_slug + del self.default_model_version_config_slug + self._default_model_version_slug = default_model_version_slug + + @property + def default_agent_slug(self) -> str: + """Canonical Agent.slug (e.g. 'claude-code-2.1.216-opus-4.8-high').""" + return self._default_agent_slug or "" + + @default_agent_slug.setter + def default_agent_slug(self, default_agent_slug: str): + if default_agent_slug is None: + del self.default_agent_slug + return + if not isinstance(default_agent_slug, str): + raise TypeError('default_agent_slug must be of type str') + del self.default_model_version_slug + del self.default_model_version_config_slug + self._default_agent_slug = default_agent_slug + + @property + def default_model_version_config_slug(self) -> str: + r""" + Canonical BenchmarkModelVersionConfig.slug (e.g. + 'claude-sonnet-4-6-high'). Like default_model_version_slug, but also pins + the reasoning effort. Requires override_harness_version. + """ + return self._default_model_version_config_slug or "" + + @default_model_version_config_slug.setter + def default_model_version_config_slug(self, default_model_version_config_slug: str): + if default_model_version_config_slug is None: + del self.default_model_version_config_slug + return + if not isinstance(default_model_version_config_slug, str): + raise TypeError('default_model_version_config_slug must be of type str') + del self.default_model_version_slug + del self.default_agent_slug + self._default_model_version_config_slug = default_model_version_config_slug + + @property + def auxiliary_model_version_config_slugs(self) -> Optional[List[str]]: + r""" + Optional. Canonical BenchmarkModelVersionConfig.slugs the task invokes + beyond the candidate (e.g. LLM judges), recorded so the scheduler can + account for their model-proxy demand. Slugs rather than ids because this + type is the public API input shape; the BenchmarkTasksApi handler resolves + them to the internal auxiliary_model_version_config_ids. + """ + return self._auxiliary_model_version_config_slugs + + @auxiliary_model_version_config_slugs.setter + def auxiliary_model_version_config_slugs(self, auxiliary_model_version_config_slugs: Optional[List[str]]): + if auxiliary_model_version_config_slugs is None: + del self.auxiliary_model_version_config_slugs + return + if not isinstance(auxiliary_model_version_config_slugs, list): + raise TypeError('auxiliary_model_version_config_slugs must be of type list') + if not all([isinstance(t, str) for t in auxiliary_model_version_config_slugs]): + raise TypeError('auxiliary_model_version_config_slugs must contain only items of type str') + self._auxiliary_model_version_config_slugs = auxiliary_model_version_config_slugs + + @property + def nested_virtualization_required(self) -> bool: + r""" + Optional. Run the Harbor executor container with nested virtualization + enabled — set when the task needs to launch VMs (e.g. QEMU/KVM guests) + inside its sandbox. Ignored unless the caller has the + BENCHMARKS_ALLOW_NESTED_VIRTUALIZATION feature flag. + """ + return self._nested_virtualization_required + + @nested_virtualization_required.setter + def nested_virtualization_required(self, nested_virtualization_required: bool): + if nested_virtualization_required is None: + del self.nested_virtualization_required + return + if not isinstance(nested_virtualization_required, bool): + raise TypeError('nested_virtualization_required must be of type bool') + self._nested_virtualization_required = nested_virtualization_required + + @property + def n_attempts(self) -> int: + r""" + Optional. How many times the task is run per invocation, mapped to the + Harbor run's n_attempts (used to compute metrics like pass@K). Defaults to + 1 when omitted. + """ + return self._n_attempts or 0 + + @n_attempts.setter + def n_attempts(self, n_attempts: Optional[int]): + if n_attempts is None: + del self.n_attempts + return + if not isinstance(n_attempts, int): + raise TypeError('n_attempts must be of type int') + self._n_attempts = n_attempts + + @property + def metrics(self) -> Optional[List['HarborMetricType']]: + r""" + Optional. Canonical Harbor metrics the task version declares to reduce its + attempts into aggregate values, supplied to Harbor's JobConfig.Metrics when + the task is invoked. Duplicates are collapsed. When omitted, Harbor + defaults to mean. + """ + return self._metrics + + @metrics.setter + def metrics(self, metrics: Optional[List['HarborMetricType']]): + if metrics is None: + del self.metrics + return + if not isinstance(metrics, list): + raise TypeError('metrics must be of type list') + if not all([isinstance(t, HarborMetricType) for t in metrics]): + raise TypeError('metrics must contain only items of type HarborMetricType') + self._metrics = metrics + + @property + def session_concurrency(self) -> int: + r""" + Optional. Maximum number of the task version's attempts run concurrently + across VMs when n_attempts > 1. Defaults to 1 (serial) when omitted. + Rejected with PERMISSION_DENIED unless the caller has the + BENCHMARKS_ATTEMPTS_ACROSS_VMS feature flag. + """ + return self._session_concurrency or 0 + + @session_concurrency.setter + def session_concurrency(self, session_concurrency: Optional[int]): + if session_concurrency is None: + del self.session_concurrency + return + if not isinstance(session_concurrency, int): + raise TypeError('session_concurrency must be of type int') + self._session_concurrency = session_concurrency + + +class BenchmarkTaskNotebookDefinition(KaggleObject): + r""" + TODO(bml): Get parity with text + options fields in the create request proto + + """ + + pass class BenchmarkTaskOptions(KaggleObject): r""" @@ -957,15 +2088,126 @@ def value(self, value: str): self._value = value +class DatasetMount(KaggleObject): + r""" + A Kaggle dataset mounted as an asset for a Harbor task, pinned to an + immutable version for reproducibility. + + Attributes: + dataset_version_slug (str) + 'owner/slug/versions/N' — the version segment is REQUIRED for + reproducibility. Requests without an explicit version are rejected. + mount_path (str) + Absolute path inside the container where the dataset is mounted. Callers + pick this so tasks can locate their data anywhere. Unique across the + mount list; must not shadow the task-definition mount. + """ + + def __init__(self): + self._dataset_version_slug = "" + self._mount_path = "" + self._freeze() + + @property + def dataset_version_slug(self) -> str: + r""" + 'owner/slug/versions/N' — the version segment is REQUIRED for + reproducibility. Requests without an explicit version are rejected. + """ + return self._dataset_version_slug + + @dataset_version_slug.setter + def dataset_version_slug(self, dataset_version_slug: str): + if dataset_version_slug is None: + del self.dataset_version_slug + return + if not isinstance(dataset_version_slug, str): + raise TypeError('dataset_version_slug must be of type str') + self._dataset_version_slug = dataset_version_slug + + @property + def mount_path(self) -> str: + r""" + Absolute path inside the container where the dataset is mounted. Callers + pick this so tasks can locate their data anywhere. Unique across the + mount list; must not shadow the task-definition mount. + """ + return self._mount_path + + @mount_path.setter + def mount_path(self, mount_path: str): + if mount_path is None: + del self.mount_path + return + if not isinstance(mount_path, str): + raise TypeError('mount_path must be of type str') + self._mount_path = mount_path + + +class DefinitionSource(KaggleObject): + r""" + Task-definition dataset for a Harbor task whose definition lives in a mounted + Kaggle dataset. Pinned to an immutable version for reproducibility. + + Attributes: + dataset_version_slug (str) + 'owner/slug/versions/N' — the version segment is REQUIRED for + reproducibility. Requests without an explicit version are rejected. + sub_path (str) + Sub-directory within the dataset containing task.toml. Empty means the + dataset root. + """ + + def __init__(self): + self._dataset_version_slug = "" + self._sub_path = "" + self._freeze() + + @property + def dataset_version_slug(self) -> str: + r""" + 'owner/slug/versions/N' — the version segment is REQUIRED for + reproducibility. Requests without an explicit version are rejected. + """ + return self._dataset_version_slug + + @dataset_version_slug.setter + def dataset_version_slug(self, dataset_version_slug: str): + if dataset_version_slug is None: + del self.dataset_version_slug + return + if not isinstance(dataset_version_slug, str): + raise TypeError('dataset_version_slug must be of type str') + self._dataset_version_slug = dataset_version_slug + + @property + def sub_path(self) -> str: + r""" + Sub-directory within the dataset containing task.toml. Empty means the + dataset root. + """ + return self._sub_path + + @sub_path.setter + def sub_path(self, sub_path: str): + if sub_path is None: + del self.sub_path + return + if not isinstance(sub_path, str): + raise TypeError('sub_path must be of type str') + self._sub_path = sub_path + + class EnvVariable(KaggleObject): r""" Environment variable on a container-backed BenchmarkTask. - `key`: variable name (e.g. `API_BASE_URL`). - - `value`: literal value, OR (when `is_secret_name` is true) the name - of a UserSecret on the caller's account. - - `is_secret_name`: when true, the system resolves `value` against the - caller's UserSecrets at runtime and injects the resolved secret. + - `value`: literal value, OR (when `is_secret_name` is true) a secret + name. + - `is_secret_name`: when true, the system resolves `value` at runtime + against the running user's UserSecrets and injects the resolved + secret. Attributes: key (str) @@ -1023,6 +2265,8 @@ class NumericResult(KaggleObject): r""" Attributes: value (float) + The largest double marks an Elo with no finite value, for a model that has + never lost or tied (see BenchmarksConstants.UndefeatedScore). confidence_interval (float) Note, while we call this the 'confidence interval' - the value we store here is actually the 'confidence radius', it should always be displayed @@ -1030,16 +2274,27 @@ class NumericResult(KaggleObject): uneven_confidence_interval (UnevenConfidenceInterval) For asymmetric confidence intervals in which the +/- values differ If set, prioritized over confidence_interval (if both are set) + standard_deviation (float) + Sample standard deviation across the attempts this result pooled. Kept + alongside confidence_interval rather than derived from it: the interval is + a radius already scaled by a critical value, so recovering the spread would + mean knowing which multiplier produced it. Aggregations that combine + children into a single interval need the spread itself. """ def __init__(self): self._value = 0.0 self._confidence_interval = None self._uneven_confidence_interval = None + self._standard_deviation = None self._freeze() @property def value(self) -> float: + r""" + The largest double marks an Elo with no finite value, for a model that has + never lost or tied (see BenchmarksConstants.UndefeatedScore). + """ return self._value @value.setter @@ -1086,6 +2341,26 @@ def uneven_confidence_interval(self, uneven_confidence_interval: Optional[Option raise TypeError('uneven_confidence_interval must be of type UnevenConfidenceInterval') self._uneven_confidence_interval = uneven_confidence_interval + @property + def standard_deviation(self) -> float: + r""" + Sample standard deviation across the attempts this result pooled. Kept + alongside confidence_interval rather than derived from it: the interval is + a radius already scaled by a critical value, so recovering the spread would + mean knowing which multiplier produced it. Aggregations that combine + children into a single interval need the spread itself. + """ + return self._standard_deviation or 0.0 + + @standard_deviation.setter + def standard_deviation(self, standard_deviation: Optional[float]): + if standard_deviation is None: + del self.standard_deviation + return + if not isinstance(standard_deviation, float): + raise TypeError('standard_deviation must be of type float') + self._standard_deviation = standard_deviation + class UnevenConfidenceInterval(KaggleObject): r""" @@ -1188,6 +2463,29 @@ def override_existing(self, override_existing: bool): self._override_existing = override_existing +BenchmarkArenaCombination._fields = [ + FieldMetadata("taskVersionId", "task_version_id", "_task_version_id", int, 0, PredefinedSerializer()), + FieldMetadata("candidate", "candidate", "_candidate", BenchmarkCandidate, None, KaggleObjectSerializer()), +] + +BenchmarkArenaSettings._fields = [ + FieldMetadata("artifactPath", "artifact_path", "_artifact_path", str, "", PredefinedSerializer()), + FieldMetadata("artifactType", "artifact_type", "_artifact_type", BenchmarkArenaArtifactType, BenchmarkArenaArtifactType.BENCHMARK_ARENA_ARTIFACT_TYPE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("revealCandidates", "reveal_candidates", "_reveal_candidates", bool, False, PredefinedSerializer()), + FieldMetadata("votingEnabled", "voting_enabled", "_voting_enabled", bool, False, PredefinedSerializer()), + FieldMetadata("galleryEnabled", "gallery_enabled", "_gallery_enabled", bool, False, PredefinedSerializer()), + FieldMetadata("arenaTaskVersionId", "arena_task_version_id", "_arena_task_version_id", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("galleryCardSize", "gallery_card_size", "_gallery_card_size", BenchmarkGalleryCardSize, BenchmarkGalleryCardSize.BENCHMARK_GALLERY_CARD_SIZE_UNSPECIFIED, EnumSerializer()), + FieldMetadata("expandArtifacts", "expand_artifacts", "_expand_artifacts", bool, False, PredefinedSerializer()), +] + +BenchmarkCandidate._fields = [ + FieldMetadata("benchmarkModelVersionId", "benchmark_model_version_id", "_benchmark_model_version_id", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("agentId", "agent_id", "_agent_id", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("reasoningEffort", "reasoning_effort", "_reasoning_effort", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("benchmarkModelVersionConfigId", "benchmark_model_version_config_id", "_benchmark_model_version_config_id", int, None, PredefinedSerializer(), optional=True), +] + BenchmarkModel._fields = [ FieldMetadata("id", "id", "_id", int, 0, PredefinedSerializer()), FieldMetadata("displayName", "display_name", "_display_name", str, "", PredefinedSerializer()), @@ -1221,6 +2519,9 @@ def override_existing(self, override_existing: bool): FieldMetadata("inputModalities", "input_modalities", "_input_modalities", Modality, [], ListSerializer(EnumSerializer())), FieldMetadata("outputModalities", "output_modalities", "_output_modalities", Modality, [], ListSerializer(EnumSerializer())), FieldMetadata("deprecationTime", "deprecation_time", "_deprecation_time", datetime, None, DateTimeSerializer(), optional=True), + FieldMetadata("benchmarkModelSlug", "benchmark_model_slug", "_benchmark_model_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("supportsReasoningEffort", "supports_reasoning_effort", "_supports_reasoning_effort", bool, None, PredefinedSerializer(), optional=True), + FieldMetadata("releaseDate", "release_date", "_release_date", datetime, None, DateTimeSerializer(), optional=True), ] BenchmarkResult._fields = [ @@ -1233,23 +2534,61 @@ def override_existing(self, override_existing: bool): FieldMetadata("taskVersionId", "task_version_id", "_task_version_id", int, None, PredefinedSerializer(), optional=True), ] -BenchmarkTaskKaggleDatasetsDefinition.DatasetMount._fields = [ - FieldMetadata("datasetVersionSlug", "dataset_version_slug", "_dataset_version_slug", str, "", PredefinedSerializer()), - FieldMetadata("mountPath", "mount_path", "_mount_path", str, "", PredefinedSerializer()), +BenchmarkSchedulerSettings._fields = [ + FieldMetadata("enabled", "enabled", "_enabled", bool, False, PredefinedSerializer()), + FieldMetadata("maxConcurrency", "max_concurrency", "_max_concurrency", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("maxConcurrencyPerCandidateModel", "max_concurrency_per_candidate_model", "_max_concurrency_per_candidate_model", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("maxKickoffsPerTick", "max_kickoffs_per_tick", "_max_kickoffs_per_tick", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("dispatchConcurrency", "dispatch_concurrency", "_dispatch_concurrency", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("poolBatchSize", "pool_batch_size", "_pool_batch_size", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("staleQueuedTtlHours", "stale_queued_ttl_hours", "_stale_queued_ttl_hours", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("staleRunningTtlHours", "stale_running_ttl_hours", "_stale_running_ttl_hours", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("reservationsEnabled", "reservations_enabled", "_reservations_enabled", bool, False, PredefinedSerializer()), + FieldMetadata("maxRetries", "max_retries", "_max_retries", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("retryCooldownMinutes", "retry_cooldown_minutes", "_retry_cooldown_minutes", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("maxRetriesPerTick", "max_retries_per_tick", "_max_retries_per_tick", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("screeningEnabled", "screening_enabled", "_screening_enabled", bool, None, PredefinedSerializer(), optional=True), + FieldMetadata("globalMinModelProxyPriority", "global_min_model_proxy_priority", "_global_min_model_proxy_priority", float, None, PredefinedSerializer(), optional=True), ] -BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource._fields = [ - FieldMetadata("datasetVersionSlug", "dataset_version_slug", "_dataset_version_slug", str, "", PredefinedSerializer()), - FieldMetadata("subPath", "sub_path", "_sub_path", str, "", PredefinedSerializer()), +BenchmarkTaskDefinition._fields = [ + FieldMetadata("notebook", "notebook", "_notebook", BenchmarkTaskNotebookDefinition, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("dockerImage", "docker_image", "_docker_image", BenchmarkTaskDockerImageDefinition, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("harborKaggleDatasets", "harbor_kaggle_datasets", "_harbor_kaggle_datasets", BenchmarkTaskKaggleDatasetsDefinition, None, KaggleObjectSerializer(), optional=True), +] + +BenchmarkTaskDockerImageDefinition._fields = [ + FieldMetadata("dockerImageUrl", "docker_image_url", "_docker_image_url", str, "", PredefinedSerializer()), + FieldMetadata("description", "description", "_description", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("title", "title", "_title", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("envVariables", "env_variables", "_env_variables", EnvVariable, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("secrets", "secrets", "_secrets", UserSecret, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("nestedVirtualizationRequired", "nested_virtualization_required", "_nested_virtualization_required", bool, False, PredefinedSerializer()), + FieldMetadata("defaultModelVersionSlug", "default_model_version_slug", "_default_model_version_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultAgentSlug", "default_agent_slug", "_default_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("auxiliaryModelVersionConfigSlugs", "auxiliary_model_version_config_slugs", "_auxiliary_model_version_config_slugs", str, [], ListSerializer(PredefinedSerializer())), + FieldMetadata("defaultModelVersionConfigSlug", "default_model_version_config_slug", "_default_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), ] BenchmarkTaskKaggleDatasetsDefinition._fields = [ - FieldMetadata("definitionSource", "definition_source", "_definition_source", BenchmarkTaskKaggleDatasetsDefinition.DefinitionSource, None, KaggleObjectSerializer()), - FieldMetadata("mounts", "mounts", "_mounts", BenchmarkTaskKaggleDatasetsDefinition.DatasetMount, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("definitionSource", "definition_source", "_definition_source", DefinitionSource, None, KaggleObjectSerializer()), + FieldMetadata("mounts", "mounts", "_mounts", DatasetMount, [], ListSerializer(KaggleObjectSerializer())), FieldMetadata("envVariables", "env_variables", "_env_variables", EnvVariable, [], ListSerializer(KaggleObjectSerializer())), FieldMetadata("secrets", "secrets", "_secrets", UserSecret, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("overrideHarnessVersionSlug", "override_harness_version_slug", "_override_harness_version_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultModelVersionSlug", "default_model_version_slug", "_default_model_version_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultAgentSlug", "default_agent_slug", "_default_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("overrideHarnessVersionCustomSlug", "override_harness_version_custom_slug", "_override_harness_version_custom_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("auxiliaryModelVersionConfigSlugs", "auxiliary_model_version_config_slugs", "_auxiliary_model_version_config_slugs", str, [], ListSerializer(PredefinedSerializer())), + FieldMetadata("nestedVirtualizationRequired", "nested_virtualization_required", "_nested_virtualization_required", bool, False, PredefinedSerializer()), + FieldMetadata("nAttempts", "n_attempts", "_n_attempts", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("metrics", "metrics", "_metrics", HarborMetricType, [], ListSerializer(EnumSerializer())), + FieldMetadata("sessionConcurrency", "session_concurrency", "_session_concurrency", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("defaultModelVersionConfigSlug", "default_model_version_config_slug", "_default_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), ] +BenchmarkTaskNotebookDefinition._fields = [] + BenchmarkTaskOptions._fields = [ FieldMetadata("datasetDataSources", "dataset_data_sources", "_dataset_data_sources", str, [], ListSerializer(PredefinedSerializer())), ] @@ -1259,6 +2598,16 @@ def override_existing(self, override_existing: bool): FieldMetadata("value", "value", "_value", str, "", PredefinedSerializer()), ] +DatasetMount._fields = [ + FieldMetadata("datasetVersionSlug", "dataset_version_slug", "_dataset_version_slug", str, "", PredefinedSerializer()), + FieldMetadata("mountPath", "mount_path", "_mount_path", str, "", PredefinedSerializer()), +] + +DefinitionSource._fields = [ + FieldMetadata("datasetVersionSlug", "dataset_version_slug", "_dataset_version_slug", str, "", PredefinedSerializer()), + FieldMetadata("subPath", "sub_path", "_sub_path", str, "", PredefinedSerializer()), +] + EnvVariable._fields = [ FieldMetadata("key", "key", "_key", str, "", PredefinedSerializer()), FieldMetadata("value", "value", "_value", str, "", PredefinedSerializer()), @@ -1269,6 +2618,7 @@ def override_existing(self, override_existing: bool): FieldMetadata("value", "value", "_value", float, 0.0, PredefinedSerializer()), FieldMetadata("confidenceInterval", "confidence_interval", "_confidence_interval", float, None, PredefinedSerializer(), optional=True), FieldMetadata("unevenConfidenceInterval", "uneven_confidence_interval", "_uneven_confidence_interval", UnevenConfidenceInterval, None, KaggleObjectSerializer(), optional=True), + FieldMetadata("standardDeviation", "standard_deviation", "_standard_deviation", float, None, PredefinedSerializer(), optional=True), ] UnevenConfidenceInterval._fields = [ diff --git a/kagglesdk/benchmarks/types/benchmarks_api_service.py b/kagglesdk/benchmarks/types/benchmarks_api_service.py index aac9f11..aceb649 100644 --- a/kagglesdk/benchmarks/types/benchmarks_api_service.py +++ b/kagglesdk/benchmarks/types/benchmarks_api_service.py @@ -1,8 +1,7 @@ from google.protobuf.field_mask_pb2 import FieldMask -from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkVersionAgentMappingType -from kagglesdk.benchmarks.types.benchmark_service import ListBenchmarkVersionAgentMappingsFilter +from kagglesdk.benchmarks.types.benchmark_enums import BenchmarkCandidateClass, BenchmarkLeaderboardDisplayType, BenchmarkVersionAgentMappingType from kagglesdk.benchmarks.types.benchmark_tasks_api_service import ApiBenchmarkTaskSlug -from kagglesdk.benchmarks.types.benchmark_types import BenchmarkModel, BenchmarkResult +from kagglesdk.benchmarks.types.benchmark_types import BenchmarkArenaCombination, BenchmarkArenaSettings, BenchmarkModel, BenchmarkResult from kagglesdk.kaggle_object import * from typing import Optional, List @@ -163,20 +162,45 @@ class ApiBenchmarkLeaderboard(KaggleObject): class LeaderboardRow(KaggleObject): r""" + One contestant and its scores. Which facets are populated follows from + `candidate_class`. + Attributes: model_version_name (str) + Always set, but does not identify the row on its own: sibling configs, + and the agents running them, share one model version. model_version_slug (str) task_results (ApiBenchmarkLeaderboard.TaskResult) + candidate_class (BenchmarkCandidateClass) + model_version_config_name (str) + Set for MODEL_VERSION_CONFIG and AGENT rows; an agent pins a config too. + model_version_config_slug (str) + agent_name (str) + Set for AGENT rows. + agent_slug (str) + harness_name (str) + harness_slug (str) """ def __init__(self): self._model_version_name = "" self._model_version_slug = "" self._task_results = [] + self._candidate_class = BenchmarkCandidateClass.BENCHMARK_CANDIDATE_CLASS_UNSPECIFIED + self._model_version_config_name = None + self._model_version_config_slug = None + self._agent_name = None + self._agent_slug = None + self._harness_name = None + self._harness_slug = None self._freeze() @property def model_version_name(self) -> str: + r""" + Always set, but does not identify the row on its own: sibling configs, + and the agents running them, share one model version. + """ return self._model_version_name @model_version_name.setter @@ -216,6 +240,99 @@ def task_results(self, task_results: Optional[List[Optional['ApiBenchmarkLeaderb raise TypeError('task_results must contain only items of type ApiBenchmarkLeaderboard.TaskResult') self._task_results = task_results + @property + def candidate_class(self) -> 'BenchmarkCandidateClass': + return self._candidate_class + + @candidate_class.setter + def candidate_class(self, candidate_class: 'BenchmarkCandidateClass'): + if candidate_class is None: + del self.candidate_class + return + if not isinstance(candidate_class, BenchmarkCandidateClass): + raise TypeError('candidate_class must be of type BenchmarkCandidateClass') + self._candidate_class = candidate_class + + @property + def model_version_config_name(self) -> str: + """Set for MODEL_VERSION_CONFIG and AGENT rows; an agent pins a config too.""" + return self._model_version_config_name or "" + + @model_version_config_name.setter + def model_version_config_name(self, model_version_config_name: Optional[str]): + if model_version_config_name is None: + del self.model_version_config_name + return + if not isinstance(model_version_config_name, str): + raise TypeError('model_version_config_name must be of type str') + self._model_version_config_name = model_version_config_name + + @property + def model_version_config_slug(self) -> str: + return self._model_version_config_slug or "" + + @model_version_config_slug.setter + def model_version_config_slug(self, model_version_config_slug: Optional[str]): + if model_version_config_slug is None: + del self.model_version_config_slug + return + if not isinstance(model_version_config_slug, str): + raise TypeError('model_version_config_slug must be of type str') + self._model_version_config_slug = model_version_config_slug + + @property + def agent_name(self) -> str: + """Set for AGENT rows.""" + return self._agent_name or "" + + @agent_name.setter + def agent_name(self, agent_name: Optional[str]): + if agent_name is None: + del self.agent_name + return + if not isinstance(agent_name, str): + raise TypeError('agent_name must be of type str') + self._agent_name = agent_name + + @property + def agent_slug(self) -> str: + return self._agent_slug or "" + + @agent_slug.setter + def agent_slug(self, agent_slug: Optional[str]): + if agent_slug is None: + del self.agent_slug + return + if not isinstance(agent_slug, str): + raise TypeError('agent_slug must be of type str') + self._agent_slug = agent_slug + + @property + def harness_name(self) -> str: + return self._harness_name or "" + + @harness_name.setter + def harness_name(self, harness_name: Optional[str]): + if harness_name is None: + del self.harness_name + return + if not isinstance(harness_name, str): + raise TypeError('harness_name must be of type str') + self._harness_name = harness_name + + @property + def harness_slug(self) -> str: + return self._harness_slug or "" + + @harness_slug.setter + def harness_slug(self, harness_slug: Optional[str]): + if harness_slug is None: + del self.harness_slug + return + if not isinstance(harness_slug, str): + raise TypeError('harness_slug must be of type str') + self._harness_slug = harness_slug + class TaskResult(KaggleObject): r""" @@ -224,6 +341,10 @@ class TaskResult(KaggleObject): benchmark_task_slug (str) task_version (int) result (BenchmarkResult) + display_type (BenchmarkLeaderboardDisplayType) + The scale `result`'s value is on: PERCENTAGES stores 0.85 for 85% while + RAW_PERCENTAGES stores 85. Columns of one board can differ, so read this + before comparing numbers across them. """ def __init__(self): @@ -231,6 +352,7 @@ def __init__(self): self._benchmark_task_slug = "" self._task_version = 0 self._result = None + self._display_type = BenchmarkLeaderboardDisplayType.BENCHMARK_LEADERBOARD_DISPLAY_TYPE_UNSPECIFIED self._freeze() @property @@ -285,6 +407,24 @@ def result(self, result: Optional['BenchmarkResult']): raise TypeError('result must be of type BenchmarkResult') self._result = result + @property + def display_type(self) -> 'BenchmarkLeaderboardDisplayType': + r""" + The scale `result`'s value is on: PERCENTAGES stores 0.85 for 85% while + RAW_PERCENTAGES stores 85. Columns of one board can differ, so read this + before comparing numbers across them. + """ + return self._display_type + + @display_type.setter + def display_type(self, display_type: 'BenchmarkLeaderboardDisplayType'): + if display_type is None: + del self.display_type + return + if not isinstance(display_type, BenchmarkLeaderboardDisplayType): + raise TypeError('display_type must be of type BenchmarkLeaderboardDisplayType') + self._display_type = display_type + def __init__(self): self._rows = [] @@ -322,6 +462,8 @@ class ApiBenchmarkModelVersionConfig(KaggleObject): slug (str) ex: 'claude-opus-4.8-high-reasoning-effort'. reasoning_effort (str) + ex: 'high'. Unset when the config pins no effort, so the model runs at its + provider default. """ def __init__(self): @@ -329,7 +471,7 @@ def __init__(self): self._benchmark_model_version_id = 0 self._display_name = "" self._slug = "" - self._reasoning_effort = "" + self._reasoning_effort = None self._freeze() @property @@ -390,10 +532,14 @@ def slug(self, slug: str): @property def reasoning_effort(self) -> str: - return self._reasoning_effort + r""" + ex: 'high'. Unset when the config pins no effort, so the model runs at its + provider default. + """ + return self._reasoning_effort or "" @reasoning_effort.setter - def reasoning_effort(self, reasoning_effort: str): + def reasoning_effort(self, reasoning_effort: Optional[str]): if reasoning_effort is None: del self.reasoning_effort return @@ -481,6 +627,97 @@ def type(self, type: 'BenchmarkVersionAgentMappingType'): self._type = type +class ApiCreateBenchmarkArenaSettingsRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + settings (BenchmarkArenaSettings) + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._settings = None + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def settings(self) -> Optional['BenchmarkArenaSettings']: + return self._settings + + @settings.setter + def settings(self, settings: Optional['BenchmarkArenaSettings']): + if settings is None: + del self.settings + return + if not isinstance(settings, BenchmarkArenaSettings): + raise TypeError('settings must be of type BenchmarkArenaSettings') + self._settings = settings + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/arena/settings/create' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/settings/create' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + class ApiCreateBenchmarkModelVersionConfigRequest(KaggleObject): r""" Attributes: @@ -664,6 +901,77 @@ def deletedMappingsCount(self): return self.deleted_mappings_count +class ApiGetBenchmarkArenaSettingsRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/arena/settings' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/settings' + return path.format_map(self.to_field_map(self)) + + @staticmethod + def endpoint_path(): + return '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/settings' + + class ApiGetBenchmarkLeaderboardRequest(KaggleObject): r""" Attributes: @@ -763,65 +1071,168 @@ def endpoint_path(): return '/api/v1/benchmarks/models/versions/configs/{id}' -class ApiListBenchmarkModelsRequest(KaggleObject): +class ApiListBenchmarkArenaCombinationsRequest(KaggleObject): r""" Attributes: - page_size (int) - page_token (str) - read_mask (FieldMask) + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. """ def __init__(self): - self._page_size = 0 - self._page_token = "" - self._read_mask = None + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None self._freeze() @property - def page_size(self) -> int: - return self._page_size + def owner_slug(self) -> str: + return self._owner_slug - @page_size.setter - def page_size(self, page_size: int): - if page_size is None: - del self.page_size + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug return - if not isinstance(page_size, int): - raise TypeError('page_size must be of type int') - self._page_size = page_size + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug @property - def page_token(self) -> str: - return self._page_token + def benchmark_slug(self) -> str: + return self._benchmark_slug - @page_token.setter - def page_token(self, page_token: str): - if page_token is None: - del self.page_token + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug return - if not isinstance(page_token, str): - raise TypeError('page_token must be of type str') - self._page_token = page_token + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug @property - def read_mask(self) -> FieldMask: - return self._read_mask + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 - @read_mask.setter - def read_mask(self, read_mask: FieldMask): - if read_mask is None: - del self.read_mask + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number return - if not isinstance(read_mask, FieldMask): - raise TypeError('read_mask must be of type FieldMask') - self._read_mask = read_mask + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number def endpoint(self): - path = '/api/v1/benchmarks/models' + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/arena/combinations' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/combinations' return path.format_map(self.to_field_map(self)) + @staticmethod + def endpoint_path(): + return '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/combinations' -class ApiListBenchmarkModelsResponse(KaggleObject): + +class ApiListBenchmarkArenaCombinationsResponse(KaggleObject): + r""" + Attributes: + combinations (BenchmarkArenaCombination) + Only those whose task version and candidate are still in the benchmark + version and readable by the caller. + """ + + def __init__(self): + self._combinations = [] + self._freeze() + + @property + def combinations(self) -> Optional[List[Optional['BenchmarkArenaCombination']]]: + r""" + Only those whose task version and candidate are still in the benchmark + version and readable by the caller. + """ + return self._combinations + + @combinations.setter + def combinations(self, combinations: Optional[List[Optional['BenchmarkArenaCombination']]]): + if combinations is None: + del self.combinations + return + if not isinstance(combinations, list): + raise TypeError('combinations must be of type list') + if not all([isinstance(t, BenchmarkArenaCombination) for t in combinations]): + raise TypeError('combinations must contain only items of type BenchmarkArenaCombination') + self._combinations = combinations + + +class ApiListBenchmarkModelsRequest(KaggleObject): + r""" + Attributes: + page_size (int) + page_token (str) + read_mask (FieldMask) + """ + + def __init__(self): + self._page_size = 0 + self._page_token = "" + self._read_mask = None + self._freeze() + + @property + def page_size(self) -> int: + return self._page_size + + @page_size.setter + def page_size(self, page_size: int): + if page_size is None: + del self.page_size + return + if not isinstance(page_size, int): + raise TypeError('page_size must be of type int') + self._page_size = page_size + + @property + def page_token(self) -> str: + return self._page_token + + @page_token.setter + def page_token(self, page_token: str): + if page_token is None: + del self.page_token + return + if not isinstance(page_token, str): + raise TypeError('page_token must be of type str') + self._page_token = page_token + + @property + def read_mask(self) -> FieldMask: + return self._read_mask + + @read_mask.setter + def read_mask(self, read_mask: FieldMask): + if read_mask is None: + del self.read_mask + return + if not isinstance(read_mask, FieldMask): + raise TypeError('read_mask must be of type FieldMask') + self._read_mask = read_mask + + def endpoint(self): + path = '/api/v1/benchmarks/models' + return path.format_map(self.to_field_map(self)) + + +class ApiListBenchmarkModelsResponse(KaggleObject): r""" Attributes: benchmark_models (BenchmarkModel) @@ -987,10 +1398,61 @@ def nextPageToken(self): return self.next_page_token +class ApiListBenchmarkVersionAgentMappingsFilter(KaggleObject): + r""" + Attributes: + parent_benchmark_version_ids (int) + If set, only return mappings whose parent BenchmarkVersionId is in this + list. + child_agent_ids (int) + If set, only return mappings whose child AgentId is in this list. + """ + + def __init__(self): + self._parent_benchmark_version_ids = [] + self._child_agent_ids = [] + self._freeze() + + @property + def parent_benchmark_version_ids(self) -> Optional[List[int]]: + r""" + If set, only return mappings whose parent BenchmarkVersionId is in this + list. + """ + return self._parent_benchmark_version_ids + + @parent_benchmark_version_ids.setter + def parent_benchmark_version_ids(self, parent_benchmark_version_ids: Optional[List[int]]): + if parent_benchmark_version_ids is None: + del self.parent_benchmark_version_ids + return + if not isinstance(parent_benchmark_version_ids, list): + raise TypeError('parent_benchmark_version_ids must be of type list') + if not all([isinstance(t, int) for t in parent_benchmark_version_ids]): + raise TypeError('parent_benchmark_version_ids must contain only items of type int') + self._parent_benchmark_version_ids = parent_benchmark_version_ids + + @property + def child_agent_ids(self) -> Optional[List[int]]: + """If set, only return mappings whose child AgentId is in this list.""" + return self._child_agent_ids + + @child_agent_ids.setter + def child_agent_ids(self, child_agent_ids: Optional[List[int]]): + if child_agent_ids is None: + del self.child_agent_ids + return + if not isinstance(child_agent_ids, list): + raise TypeError('child_agent_ids must be of type list') + if not all([isinstance(t, int) for t in child_agent_ids]): + raise TypeError('child_agent_ids must contain only items of type int') + self._child_agent_ids = child_agent_ids + + class ApiListBenchmarkVersionAgentMappingsRequest(KaggleObject): r""" Attributes: - filter (ListBenchmarkVersionAgentMappingsFilter) + filter (ApiListBenchmarkVersionAgentMappingsFilter) page_size (int) page_token (str) """ @@ -1002,16 +1464,16 @@ def __init__(self): self._freeze() @property - def filter(self) -> Optional['ListBenchmarkVersionAgentMappingsFilter']: + def filter(self) -> Optional['ApiListBenchmarkVersionAgentMappingsFilter']: return self._filter @filter.setter - def filter(self, filter: Optional['ListBenchmarkVersionAgentMappingsFilter']): + def filter(self, filter: Optional['ApiListBenchmarkVersionAgentMappingsFilter']): if filter is None: del self.filter return - if not isinstance(filter, ListBenchmarkVersionAgentMappingsFilter): - raise TypeError('filter must be of type ListBenchmarkVersionAgentMappingsFilter') + if not isinstance(filter, ApiListBenchmarkVersionAgentMappingsFilter): + raise TypeError('filter must be of type ApiListBenchmarkVersionAgentMappingsFilter') self._filter = filter @property @@ -1090,6 +1552,549 @@ def nextPageToken(self): return self.next_page_token +class ApiRegisterAuxiliaryModelVersionConfigsRequest(KaggleObject): + r""" + Attributes: + task_slug (str) + Slug of the task to attach configs to. Always resolved under the current + user; tasks owned by anyone else are not addressable here. + version_number (int) + Targets the task's latest version if omitted. + config_slugs (str) + Slugs of the auxiliary BenchmarkModelVersionConfigs to attach. Duplicates + are collapsed; configs already registered on the version are skipped. + """ + + def __init__(self): + self._task_slug = "" + self._version_number = None + self._config_slugs = [] + self._freeze() + + @property + def task_slug(self) -> str: + r""" + Slug of the task to attach configs to. Always resolved under the current + user; tasks owned by anyone else are not addressable here. + """ + return self._task_slug + + @task_slug.setter + def task_slug(self, task_slug: str): + if task_slug is None: + del self.task_slug + return + if not isinstance(task_slug, str): + raise TypeError('task_slug must be of type str') + self._task_slug = task_slug + + @property + def version_number(self) -> int: + """Targets the task's latest version if omitted.""" + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def config_slugs(self) -> Optional[List[str]]: + r""" + Slugs of the auxiliary BenchmarkModelVersionConfigs to attach. Duplicates + are collapsed; configs already registered on the version are skipped. + """ + return self._config_slugs + + @config_slugs.setter + def config_slugs(self, config_slugs: Optional[List[str]]): + if config_slugs is None: + del self.config_slugs + return + if not isinstance(config_slugs, list): + raise TypeError('config_slugs must be of type list') + if not all([isinstance(t, str) for t in config_slugs]): + raise TypeError('config_slugs must contain only items of type str') + self._config_slugs = config_slugs + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/tasks/{task_slug}/versions/{version_number}/auxiliary-configs/register' + else: + path = '/api/v1/benchmarks/tasks/{task_slug}/auxiliary-configs/register' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiRemoveTasksFromBenchmarkRequest(KaggleObject): + r""" + Attributes: + benchmark_slug (str) + Slug of the benchmark to remove from. Always resolved under the current + user; benchmarks owned by anyone else are not addressable here. + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + task_slugs (ApiBenchmarkTaskSlug) + Tasks to unmap; must be non-empty. Duplicates are collapsed. + """ + + def __init__(self): + self._benchmark_slug = "" + self._version_number = None + self._task_slugs = [] + self._freeze() + + @property + def benchmark_slug(self) -> str: + r""" + Slug of the benchmark to remove from. Always resolved under the current + user; benchmarks owned by anyone else are not addressable here. + """ + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def task_slugs(self) -> Optional[List[Optional['ApiBenchmarkTaskSlug']]]: + """Tasks to unmap; must be non-empty. Duplicates are collapsed.""" + return self._task_slugs + + @task_slugs.setter + def task_slugs(self, task_slugs: Optional[List[Optional['ApiBenchmarkTaskSlug']]]): + if task_slugs is None: + del self.task_slugs + return + if not isinstance(task_slugs, list): + raise TypeError('task_slugs must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskSlug) for t in task_slugs]): + raise TypeError('task_slugs must contain only items of type ApiBenchmarkTaskSlug') + self._task_slugs = task_slugs + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{benchmark_slug}/versions/{version_number}/tasks/remove' + else: + path = '/api/v1/benchmarks/{benchmark_slug}/tasks/remove' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiRemoveTasksFromBenchmarkResponse(KaggleObject): + r""" + Attributes: + removed_tasks (ApiBenchmarkTaskSlug) + Tasks newly unmapped by this call, with the resolved owner and version + number filled in. + skipped_tasks (ApiBenchmarkTaskSlug) + Tasks that were not mapped to this benchmark and were left alone. + """ + + def __init__(self): + self._removed_tasks = [] + self._skipped_tasks = [] + self._freeze() + + @property + def removed_tasks(self) -> Optional[List[Optional['ApiBenchmarkTaskSlug']]]: + r""" + Tasks newly unmapped by this call, with the resolved owner and version + number filled in. + """ + return self._removed_tasks + + @removed_tasks.setter + def removed_tasks(self, removed_tasks: Optional[List[Optional['ApiBenchmarkTaskSlug']]]): + if removed_tasks is None: + del self.removed_tasks + return + if not isinstance(removed_tasks, list): + raise TypeError('removed_tasks must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskSlug) for t in removed_tasks]): + raise TypeError('removed_tasks must contain only items of type ApiBenchmarkTaskSlug') + self._removed_tasks = removed_tasks + + @property + def skipped_tasks(self) -> Optional[List[Optional['ApiBenchmarkTaskSlug']]]: + """Tasks that were not mapped to this benchmark and were left alone.""" + return self._skipped_tasks + + @skipped_tasks.setter + def skipped_tasks(self, skipped_tasks: Optional[List[Optional['ApiBenchmarkTaskSlug']]]): + if skipped_tasks is None: + del self.skipped_tasks + return + if not isinstance(skipped_tasks, list): + raise TypeError('skipped_tasks must be of type list') + if not all([isinstance(t, ApiBenchmarkTaskSlug) for t in skipped_tasks]): + raise TypeError('skipped_tasks must contain only items of type ApiBenchmarkTaskSlug') + self._skipped_tasks = skipped_tasks + + @property + def removedTasks(self): + return self.removed_tasks + + @property + def skippedTasks(self): + return self.skipped_tasks + + +class ApiUnregisterAuxiliaryModelVersionConfigsRequest(KaggleObject): + r""" + Attributes: + task_slug (str) + Slug of the task to detach configs from. Always resolved under the current + user; tasks owned by anyone else are not addressable here. + version_number (int) + Targets the task's latest version if omitted. + config_slugs (str) + Slugs of the auxiliary BenchmarkModelVersionConfigs to remove. Configs not + currently registered are ignored. + """ + + def __init__(self): + self._task_slug = "" + self._version_number = None + self._config_slugs = [] + self._freeze() + + @property + def task_slug(self) -> str: + r""" + Slug of the task to detach configs from. Always resolved under the current + user; tasks owned by anyone else are not addressable here. + """ + return self._task_slug + + @task_slug.setter + def task_slug(self, task_slug: str): + if task_slug is None: + del self.task_slug + return + if not isinstance(task_slug, str): + raise TypeError('task_slug must be of type str') + self._task_slug = task_slug + + @property + def version_number(self) -> int: + """Targets the task's latest version if omitted.""" + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def config_slugs(self) -> Optional[List[str]]: + r""" + Slugs of the auxiliary BenchmarkModelVersionConfigs to remove. Configs not + currently registered are ignored. + """ + return self._config_slugs + + @config_slugs.setter + def config_slugs(self, config_slugs: Optional[List[str]]): + if config_slugs is None: + del self.config_slugs + return + if not isinstance(config_slugs, list): + raise TypeError('config_slugs must be of type list') + if not all([isinstance(t, str) for t in config_slugs]): + raise TypeError('config_slugs must contain only items of type str') + self._config_slugs = config_slugs + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/tasks/{task_slug}/versions/{version_number}/auxiliary-configs/unregister' + else: + path = '/api/v1/benchmarks/tasks/{task_slug}/auxiliary-configs/unregister' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiUpdateBenchmarkArenaCombinationsRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + add (BenchmarkArenaCombination) + Each must be a task version holding the benchmark version's runs and a + mapped candidate of that task version's candidate type. Already opted-in + combinations are ignored. + remove (BenchmarkArenaCombination) + Combinations not opted in are ignored. + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._add = [] + self._remove = [] + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def add(self) -> Optional[List[Optional['BenchmarkArenaCombination']]]: + r""" + Each must be a task version holding the benchmark version's runs and a + mapped candidate of that task version's candidate type. Already opted-in + combinations are ignored. + """ + return self._add + + @add.setter + def add(self, add: Optional[List[Optional['BenchmarkArenaCombination']]]): + if add is None: + del self.add + return + if not isinstance(add, list): + raise TypeError('add must be of type list') + if not all([isinstance(t, BenchmarkArenaCombination) for t in add]): + raise TypeError('add must contain only items of type BenchmarkArenaCombination') + self._add = add + + @property + def remove(self) -> Optional[List[Optional['BenchmarkArenaCombination']]]: + """Combinations not opted in are ignored.""" + return self._remove + + @remove.setter + def remove(self, remove: Optional[List[Optional['BenchmarkArenaCombination']]]): + if remove is None: + del self.remove + return + if not isinstance(remove, list): + raise TypeError('remove must be of type list') + if not all([isinstance(t, BenchmarkArenaCombination) for t in remove]): + raise TypeError('remove must contain only items of type BenchmarkArenaCombination') + self._remove = remove + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/arena/combinations/update' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/combinations/update' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + +class ApiUpdateBenchmarkArenaSettingsRequest(KaggleObject): + r""" + Attributes: + owner_slug (str) + benchmark_slug (str) + version_number (int) + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + settings (BenchmarkArenaSettings) + update_mask (FieldMask) + """ + + def __init__(self): + self._owner_slug = "" + self._benchmark_slug = "" + self._version_number = None + self._settings = None + self._update_mask = None + self._freeze() + + @property + def owner_slug(self) -> str: + return self._owner_slug + + @owner_slug.setter + def owner_slug(self, owner_slug: str): + if owner_slug is None: + del self.owner_slug + return + if not isinstance(owner_slug, str): + raise TypeError('owner_slug must be of type str') + self._owner_slug = owner_slug + + @property + def benchmark_slug(self) -> str: + return self._benchmark_slug + + @benchmark_slug.setter + def benchmark_slug(self, benchmark_slug: str): + if benchmark_slug is None: + del self.benchmark_slug + return + if not isinstance(benchmark_slug, str): + raise TypeError('benchmark_slug must be of type str') + self._benchmark_slug = benchmark_slug + + @property + def version_number(self) -> int: + r""" + Targets the benchmark's current published version, or its draft when + there is no published version, if omitted. + """ + return self._version_number or 0 + + @version_number.setter + def version_number(self, version_number: Optional[int]): + if version_number is None: + del self.version_number + return + if not isinstance(version_number, int): + raise TypeError('version_number must be of type int') + self._version_number = version_number + + @property + def settings(self) -> Optional['BenchmarkArenaSettings']: + return self._settings + + @settings.setter + def settings(self, settings: Optional['BenchmarkArenaSettings']): + if settings is None: + del self.settings + return + if not isinstance(settings, BenchmarkArenaSettings): + raise TypeError('settings must be of type BenchmarkArenaSettings') + self._settings = settings + + @property + def update_mask(self) -> FieldMask: + return self._update_mask + + @update_mask.setter + def update_mask(self, update_mask: FieldMask): + if update_mask is None: + del self.update_mask + return + if not isinstance(update_mask, FieldMask): + raise TypeError('update_mask must be of type FieldMask') + self._update_mask = update_mask + + def endpoint(self): + if self.version_number: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/versions/{version_number}/arena/settings/update' + else: + path = '/api/v1/benchmarks/{owner_slug}/{benchmark_slug}/arena/settings/update' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + ApiAddTasksToBenchmarkRequest._fields = [ FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), @@ -1105,6 +2110,13 @@ def nextPageToken(self): FieldMetadata("modelVersionName", "model_version_name", "_model_version_name", str, "", PredefinedSerializer()), FieldMetadata("modelVersionSlug", "model_version_slug", "_model_version_slug", str, "", PredefinedSerializer()), FieldMetadata("taskResults", "task_results", "_task_results", ApiBenchmarkLeaderboard.TaskResult, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("candidateClass", "candidate_class", "_candidate_class", BenchmarkCandidateClass, BenchmarkCandidateClass.BENCHMARK_CANDIDATE_CLASS_UNSPECIFIED, EnumSerializer()), + FieldMetadata("modelVersionConfigName", "model_version_config_name", "_model_version_config_name", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("modelVersionConfigSlug", "model_version_config_slug", "_model_version_config_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("agentName", "agent_name", "_agent_name", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("agentSlug", "agent_slug", "_agent_slug", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("harnessName", "harness_name", "_harness_name", str, None, PredefinedSerializer(), optional=True), + FieldMetadata("harnessSlug", "harness_slug", "_harness_slug", str, None, PredefinedSerializer(), optional=True), ] ApiBenchmarkLeaderboard.TaskResult._fields = [ @@ -1112,6 +2124,7 @@ def nextPageToken(self): FieldMetadata("benchmarkTaskSlug", "benchmark_task_slug", "_benchmark_task_slug", str, "", PredefinedSerializer()), FieldMetadata("taskVersion", "task_version", "_task_version", int, 0, PredefinedSerializer()), FieldMetadata("result", "result", "_result", BenchmarkResult, None, KaggleObjectSerializer()), + FieldMetadata("displayType", "display_type", "_display_type", BenchmarkLeaderboardDisplayType, BenchmarkLeaderboardDisplayType.BENCHMARK_LEADERBOARD_DISPLAY_TYPE_UNSPECIFIED, EnumSerializer()), ] ApiBenchmarkLeaderboard._fields = [ @@ -1123,7 +2136,7 @@ def nextPageToken(self): FieldMetadata("benchmarkModelVersionId", "benchmark_model_version_id", "_benchmark_model_version_id", int, 0, PredefinedSerializer()), FieldMetadata("displayName", "display_name", "_display_name", str, "", PredefinedSerializer()), FieldMetadata("slug", "slug", "_slug", str, "", PredefinedSerializer()), - FieldMetadata("reasoningEffort", "reasoning_effort", "_reasoning_effort", str, "", PredefinedSerializer()), + FieldMetadata("reasoningEffort", "reasoning_effort", "_reasoning_effort", str, None, PredefinedSerializer(), optional=True), ] ApiBenchmarkVersionAgentMapping._fields = [ @@ -1133,6 +2146,13 @@ def nextPageToken(self): FieldMetadata("type", "type", "_type", BenchmarkVersionAgentMappingType, BenchmarkVersionAgentMappingType.BENCHMARK_VERSION_AGENT_MAPPING_TYPE_UNSPECIFIED, EnumSerializer()), ] +ApiCreateBenchmarkArenaSettingsRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("settings", "settings", "_settings", BenchmarkArenaSettings, None, KaggleObjectSerializer()), +] + ApiCreateBenchmarkModelVersionConfigRequest._fields = [ FieldMetadata("config", "config", "_config", ApiBenchmarkModelVersionConfig, None, KaggleObjectSerializer()), ] @@ -1153,6 +2173,12 @@ def nextPageToken(self): FieldMetadata("deletedMappingsCount", "deleted_mappings_count", "_deleted_mappings_count", int, 0, PredefinedSerializer()), ] +ApiGetBenchmarkArenaSettingsRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), +] + ApiGetBenchmarkLeaderboardRequest._fields = [ FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), @@ -1163,6 +2189,16 @@ def nextPageToken(self): FieldMetadata("id", "id", "_id", int, 0, PredefinedSerializer()), ] +ApiListBenchmarkArenaCombinationsRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), +] + +ApiListBenchmarkArenaCombinationsResponse._fields = [ + FieldMetadata("combinations", "combinations", "_combinations", BenchmarkArenaCombination, [], ListSerializer(KaggleObjectSerializer())), +] + ApiListBenchmarkModelsRequest._fields = [ FieldMetadata("pageSize", "page_size", "_page_size", int, 0, PredefinedSerializer()), FieldMetadata("pageToken", "page_token", "_page_token", str, "", PredefinedSerializer()), @@ -1185,8 +2221,13 @@ def nextPageToken(self): FieldMetadata("nextPageToken", "next_page_token", "_next_page_token", str, None, PredefinedSerializer(), optional=True), ] +ApiListBenchmarkVersionAgentMappingsFilter._fields = [ + FieldMetadata("parentBenchmarkVersionIds", "parent_benchmark_version_ids", "_parent_benchmark_version_ids", int, [], ListSerializer(PredefinedSerializer())), + FieldMetadata("childAgentIds", "child_agent_ids", "_child_agent_ids", int, [], ListSerializer(PredefinedSerializer())), +] + ApiListBenchmarkVersionAgentMappingsRequest._fields = [ - FieldMetadata("filter", "filter", "_filter", ListBenchmarkVersionAgentMappingsFilter, None, KaggleObjectSerializer()), + FieldMetadata("filter", "filter", "_filter", ApiListBenchmarkVersionAgentMappingsFilter, None, KaggleObjectSerializer()), FieldMetadata("pageSize", "page_size", "_page_size", int, 0, PredefinedSerializer()), FieldMetadata("pageToken", "page_token", "_page_token", str, "", PredefinedSerializer()), ] @@ -1196,3 +2237,42 @@ def nextPageToken(self): FieldMetadata("nextPageToken", "next_page_token", "_next_page_token", str, None, PredefinedSerializer(), optional=True), ] +ApiRegisterAuxiliaryModelVersionConfigsRequest._fields = [ + FieldMetadata("taskSlug", "task_slug", "_task_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("configSlugs", "config_slugs", "_config_slugs", str, [], ListSerializer(PredefinedSerializer())), +] + +ApiRemoveTasksFromBenchmarkRequest._fields = [ + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("taskSlugs", "task_slugs", "_task_slugs", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), +] + +ApiRemoveTasksFromBenchmarkResponse._fields = [ + FieldMetadata("removedTasks", "removed_tasks", "_removed_tasks", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("skippedTasks", "skipped_tasks", "_skipped_tasks", ApiBenchmarkTaskSlug, [], ListSerializer(KaggleObjectSerializer())), +] + +ApiUnregisterAuxiliaryModelVersionConfigsRequest._fields = [ + FieldMetadata("taskSlug", "task_slug", "_task_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("configSlugs", "config_slugs", "_config_slugs", str, [], ListSerializer(PredefinedSerializer())), +] + +ApiUpdateBenchmarkArenaCombinationsRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("add", "add", "_add", BenchmarkArenaCombination, [], ListSerializer(KaggleObjectSerializer())), + FieldMetadata("remove", "remove", "_remove", BenchmarkArenaCombination, [], ListSerializer(KaggleObjectSerializer())), +] + +ApiUpdateBenchmarkArenaSettingsRequest._fields = [ + FieldMetadata("ownerSlug", "owner_slug", "_owner_slug", str, "", PredefinedSerializer()), + FieldMetadata("benchmarkSlug", "benchmark_slug", "_benchmark_slug", str, "", PredefinedSerializer()), + FieldMetadata("versionNumber", "version_number", "_version_number", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("settings", "settings", "_settings", BenchmarkArenaSettings, None, KaggleObjectSerializer()), + FieldMetadata("updateMask", "update_mask", "_update_mask", FieldMask, None, FieldMaskSerializer()), +] + diff --git a/kagglesdk/competitions/services/competition_api_service.py b/kagglesdk/competitions/services/competition_api_service.py index 6f254a6..0ff9241 100644 --- a/kagglesdk/competitions/services/competition_api_service.py +++ b/kagglesdk/competitions/services/competition_api_service.py @@ -1,6 +1,6 @@ from kagglesdk.common.types.file_download import FileDownload from kagglesdk.common.types.http_redirect import HttpRedirect -from kagglesdk.competitions.types.competition_api_service import ApiAddCompetitionHostRequest, ApiAddCompetitionJudgeRequest, ApiCompetition, ApiCompetitionPage, ApiCompetitionSolutionStatus, ApiCreateCodeSubmissionRequest, ApiCreateCodeSubmissionResponse, ApiCreateCompetitionDataRequest, ApiCreateCompetitionDataResponse, ApiCreateCompetitionPageRequest, ApiCreateCompetitionRequest, ApiCreateCompetitionResponse, ApiCreateCompetitionSampleSubmissionRequest, ApiCreateCompetitionSolutionRequest, ApiCreateSubmissionRequest, ApiCreateSubmissionResponse, ApiDeleteCompetitionPageRequest, ApiDownloadDataFileRequest, ApiDownloadDataFilesRequest, ApiDownloadLeaderboardRequest, ApiDownloadSubmissionRequest, ApiGetCompetitionDataFilesSummaryRequest, ApiGetCompetitionRequest, ApiGetCompetitionSettingsRequest, ApiGetCompetitionSolutionStatusRequest, ApiGetEpisodeAgentLogsRequest, ApiGetEpisodeReplayRequest, ApiGetLeaderboardRequest, ApiGetLeaderboardResponse, ApiGetSubmissionLimitsRequest, ApiGetSubmissionRequest, ApiLaunchCompetitionRequest, ApiListCompetitionHostsRequest, ApiListCompetitionHostsResponse, ApiListCompetitionJudgesRequest, ApiListCompetitionJudgesResponse, ApiListCompetitionPagesRequest, ApiListCompetitionPagesResponse, ApiListCompetitionsRequest, ApiListCompetitionsResponse, ApiListCompetitionTopicsRequest, ApiListCompetitionTopicsResponse, ApiListDataFilesRequest, ApiListDataFilesResponse, ApiListDataTreeFilesRequest, ApiListSubmissionEpisodesRequest, ApiListSubmissionEpisodesResponse, ApiListSubmissionsRequest, ApiListSubmissionsResponse, ApiListTeamPublicSubmissionsRequest, ApiListTeamPublicSubmissionsResponse, ApiListTopicMessagesRequest, ApiListTopicMessagesResponse, ApiStartSubmissionUploadRequest, ApiStartSubmissionUploadResponse, ApiSubmission, ApiSubmissionLimits, ApiUpdateCompetitionPageRequest, ApiUpdateCompetitionSettingsRequest +from kagglesdk.competitions.types.competition_api_service import ApiAddCompetitionHostRequest, ApiAddCompetitionJudgeRequest, ApiCompetition, ApiCompetitionPage, ApiCompetitionSolutionStatus, ApiCreateCodeSubmissionRequest, ApiCreateCodeSubmissionResponse, ApiCreateCompetitionDataRequest, ApiCreateCompetitionDataResponse, ApiCreateCompetitionPageRequest, ApiCreateCompetitionRequest, ApiCreateCompetitionResponse, ApiCreateCompetitionSampleSubmissionRequest, ApiCreateCompetitionSolutionRequest, ApiCreateSubmissionRequest, ApiCreateSubmissionResponse, ApiDeleteCompetitionPageRequest, ApiDownloadDataFileRequest, ApiDownloadDataFilesRequest, ApiDownloadLeaderboardRequest, ApiDownloadSubmissionRequest, ApiGetCompetitionDataFilesSummaryRequest, ApiGetCompetitionRequest, ApiGetCompetitionSettingsRequest, ApiGetCompetitionSolutionStatusRequest, ApiGetEpisodeAgentLogsRequest, ApiGetEpisodeReplayRequest, ApiGetLeaderboardRequest, ApiGetLeaderboardResponse, ApiGetSubmissionLimitsRequest, ApiGetSubmissionRequest, ApiLaunchCompetitionRequest, ApiListCompetitionHostsRequest, ApiListCompetitionHostsResponse, ApiListCompetitionJudgesRequest, ApiListCompetitionJudgesResponse, ApiListCompetitionPagesRequest, ApiListCompetitionPagesResponse, ApiListCompetitionsRequest, ApiListCompetitionsResponse, ApiListCompetitionTopicsRequest, ApiListCompetitionTopicsResponse, ApiListDataFilesRequest, ApiListDataFilesResponse, ApiListDataTreeFilesRequest, ApiListSubmissionEpisodesRequest, ApiListSubmissionEpisodesResponse, ApiListSubmissionsRequest, ApiListSubmissionsResponse, ApiListTeamPublicSubmissionsRequest, ApiListTeamPublicSubmissionsResponse, ApiListTopicMessagesRequest, ApiListTopicMessagesResponse, ApiStartSubmissionUploadRequest, ApiStartSubmissionUploadResponse, ApiSubmission, ApiSubmissionLimits, ApiUpdateCompetitionPageRequest, ApiUpdateCompetitionSettingsRequest, ApiUpdateSubmissionBenchmarkRequest from kagglesdk.competitions.types.host_service import CompetitionSettings from kagglesdk.datasets.databundles.types.databundle_api_types import ApiDirectoryContent, ApiFilesSummary from kagglesdk.kaggle_http_client import KaggleHttpClient @@ -154,6 +154,18 @@ def download_submission(self, request: ApiDownloadSubmissionRequest = None) -> H return self._client.call("competitions.CompetitionApiService", "DownloadSubmission", request, HttpRedirect) + def update_submission_benchmark(self, request: ApiUpdateSubmissionBenchmarkRequest = None): + r""" + Args: + request (ApiUpdateSubmissionBenchmarkRequest): + The request object; initialized to empty instance if not specified. + """ + + if request is None: + request = ApiUpdateSubmissionBenchmarkRequest() + + self._client.call("competitions.CompetitionApiService", "UpdateSubmissionBenchmark", request, None) + def get_submission_limits(self, request: ApiGetSubmissionLimitsRequest = None) -> ApiSubmissionLimits: r""" Args: diff --git a/kagglesdk/competitions/types/competition_api_service.py b/kagglesdk/competitions/types/competition_api_service.py index 0bb50ae..ff6f978 100644 --- a/kagglesdk/competitions/types/competition_api_service.py +++ b/kagglesdk/competitions/types/competition_api_service.py @@ -282,6 +282,8 @@ class ApiCompetition(KaggleObject): thumbnail_image_url (str) host_name (str) date_created (datetime) + license_name (str) + Display name of the data license, e.g. 'CC0: Public Domain'. """ def __init__(self): @@ -312,6 +314,7 @@ def __init__(self): self._thumbnail_image_url = None self._host_name = "" self._date_created = None + self._license_name = None self._freeze() @property @@ -667,6 +670,20 @@ def date_created(self, date_created: datetime): raise TypeError('date_created must be of type datetime') self._date_created = date_created + @property + def license_name(self) -> str: + """Display name of the data license, e.g. 'CC0: Public Domain'.""" + return self._license_name or "" + + @license_name.setter + def license_name(self, license_name: Optional[str]): + if license_name is None: + del self.license_name + return + if not isinstance(license_name, str): + raise TypeError('license_name must be of type str') + self._license_name = license_name + class ApiCompetitionDataFile(KaggleObject): r""" @@ -1596,6 +1613,8 @@ class ApiCreateCompetitionRequest(KaggleObject): emails/domains. license_id (int) Optional. License ID for the competition data. See ListCompetitionLicenses. + Must not be set when clone_competition_id is set, since a clone reuses the + source competition's dataset and inherits its license. organization_id (int) Optional. Tie this competition to an organization (grants read-only access to all org members). @@ -1610,6 +1629,10 @@ class ApiCreateCompetitionRequest(KaggleObject): Optional. Create as a hackathon competition. num_prizes (int) Optional. Number of leaderboard prize positions. + task_creation (bool) + Optional. Create as a task creation competition, with empty task creation + settings. Can't be combined with hackathon or clone_competition_id: + cloning a task creation competition copies its settings. """ def __init__(self): @@ -1627,6 +1650,7 @@ def __init__(self): self._reward = None self._hackathon = None self._num_prizes = None + self._task_creation = None self._freeze() @property @@ -1738,7 +1762,11 @@ def restrict_link_to_email_list(self, restrict_link_to_email_list: Optional[bool @property def license_id(self) -> int: - """Optional. License ID for the competition data. See ListCompetitionLicenses.""" + r""" + Optional. License ID for the competition data. See ListCompetitionLicenses. + Must not be set when clone_competition_id is set, since a clone reuses the + source competition's dataset and inherits its license. + """ return self._license_id or 0 @license_id.setter @@ -1842,6 +1870,24 @@ def num_prizes(self, num_prizes: Optional[int]): raise TypeError('num_prizes must be of type int') self._num_prizes = num_prizes + @property + def task_creation(self) -> bool: + r""" + Optional. Create as a task creation competition, with empty task creation + settings. Can't be combined with hackathon or clone_competition_id: + cloning a task creation competition copies its settings. + """ + return self._task_creation or False + + @task_creation.setter + def task_creation(self, task_creation: Optional[bool]): + if task_creation is None: + del self.task_creation + return + if not isinstance(task_creation, bool): + raise TypeError('task_creation must be of type bool') + self._task_creation = task_creation + def endpoint(self): path = '/api/v1/competitions' return path.format_map(self.to_field_map(self)) @@ -5251,6 +5297,64 @@ def body_fields(): return '*' +class ApiUpdateSubmissionBenchmarkRequest(KaggleObject): + r""" + Attributes: + submission_id (int) + benchmark (bool) + True publishes the sandbox submission as a benchmark; false moves it back + to the sandbox. Required. + """ + + def __init__(self): + self._submission_id = 0 + self._benchmark = None + self._freeze() + + @property + def submission_id(self) -> int: + return self._submission_id + + @submission_id.setter + def submission_id(self, submission_id: int): + if submission_id is None: + del self.submission_id + return + if not isinstance(submission_id, int): + raise TypeError('submission_id must be of type int') + self._submission_id = submission_id + + @property + def benchmark(self) -> bool: + r""" + True publishes the sandbox submission as a benchmark; false moves it back + to the sandbox. Required. + """ + return self._benchmark or False + + @benchmark.setter + def benchmark(self, benchmark: Optional[bool]): + if benchmark is None: + del self.benchmark + return + if not isinstance(benchmark, bool): + raise TypeError('benchmark must be of type bool') + self._benchmark = benchmark + + def endpoint(self): + path = '/api/v1/competitions/submissions/{submission_id}/benchmark' + return path.format_map(self.to_field_map(self)) + + + @staticmethod + def method(): + return 'POST' + + @staticmethod + def body_fields(): + return '*' + + ApiAddCompetitionHostRequest._fields = [ FieldMetadata("competitionName", "competition_name", "_competition_name", str, "", PredefinedSerializer()), FieldMetadata("userName", "user_name", "_user_name", str, "", PredefinedSerializer()), @@ -5300,6 +5404,7 @@ def body_fields(): FieldMetadata("thumbnailImageUrl", "thumbnail_image_url", "_thumbnail_image_url", str, None, PredefinedSerializer(), optional=True), FieldMetadata("hostName", "host_name", "_host_name", str, "", PredefinedSerializer()), FieldMetadata("dateCreated", "date_created", "_date_created", datetime, None, DateTimeSerializer()), + FieldMetadata("licenseName", "license_name", "_license_name", str, None, PredefinedSerializer(), optional=True), ] ApiCompetitionDataFile._fields = [ @@ -5391,6 +5496,7 @@ def body_fields(): FieldMetadata("reward", "reward", "_reward", Reward, None, KaggleObjectSerializer(), optional=True), FieldMetadata("hackathon", "hackathon", "_hackathon", bool, None, PredefinedSerializer(), optional=True), FieldMetadata("numPrizes", "num_prizes", "_num_prizes", int, None, PredefinedSerializer(), optional=True), + FieldMetadata("taskCreation", "task_creation", "_task_creation", bool, None, PredefinedSerializer(), optional=True), ] ApiCreateCompetitionResponse._fields = [ @@ -5719,3 +5825,8 @@ def body_fields(): FieldMetadata("updateMask", "update_mask", "_update_mask", FieldMask, None, FieldMaskSerializer()), ] +ApiUpdateSubmissionBenchmarkRequest._fields = [ + FieldMetadata("submissionId", "submission_id", "_submission_id", int, 0, PredefinedSerializer()), + FieldMetadata("benchmark", "benchmark", "_benchmark", bool, None, PredefinedSerializer(), optional=True), +] + diff --git a/kagglesdk/competitions/types/host_service.py b/kagglesdk/competitions/types/host_service.py index 6d800a3..9f7448d 100644 --- a/kagglesdk/competitions/types/host_service.py +++ b/kagglesdk/competitions/types/host_service.py @@ -80,6 +80,8 @@ class CompetitionSettings(KaggleObject): 'Final Leaderboard Verified' in the UI. final_leaderboard_disclaimer_message (str) 'Private Leaderboard Disclaimer' in the UI. + max_pending_submissions_per_team (int) + 'Max Pending Submissions Per Team' in the UI. Unset or 0 means no cap. only_allow_kernel_submissions (bool) CODE COMPETITION SETTINGS GROUP (field numbers 100-119) 'Code Competition' in the UI. Whether competitors must submit a Notebook. @@ -90,9 +92,11 @@ class CompetitionSettings(KaggleObject): max_gpu_runtime_minutes (int) 'Max GPU Runtime Minutes' in the UI. rerun_override_kernel_id (int) - Host-authored template Notebook the frontend wraps the user's artifact - (e.g. skill.md) into before submitting. Parallels - main_py_override_kernel_id. + Optional host-authored Notebook that is rerun in place of the user's, with + the user's Submission file mounted as a datasource in + /kaggle/input/competition_submission/. Intended for competitions where + users submit an artifact (like skill.md) rather than runnable code. + Requires uses_synchronous_reruns to be set. gateway_kernel_id (int) 'Gateway Kernel Id' in the UI. Identifies the notebook whose latest version is used as the source for the parent (gateway) container in @@ -101,6 +105,11 @@ class CompetitionSettings(KaggleObject): 'Rerun Max Stagger Minutes' in the UI. Random delay added before a scoring run kicks off (sampled from 0..this value, clamped to 60). Used for sync-rerun / Hearth competitions to limit private dataset probing. + rerun_expected_output_filename (str) + Optional override for the rerun notebook output file which we pass to the + submission scoring metric. If unset and using rerun_override_kernel_id we + default to 'submission.parquet', otherwise default to the filename from the + user's original submission. directly_responsible_user_id (int) HOSTS SETTINGS GROUP: ADMIN ONLY (field numbers 120-139) 'Directly Responsible User' in the UI. Kaggle admin responsible for the @@ -131,6 +140,7 @@ def __init__(self): self._withold_final_leaderboard_until_it_has_been_verified = False self._final_leaderboard_has_been_verified = False self._final_leaderboard_disclaimer_message = "" + self._max_pending_submissions_per_team = None self._only_allow_kernel_submissions = False self._uses_synchronous_reruns = False self._max_cpu_runtime_minutes = None @@ -138,6 +148,7 @@ def __init__(self): self._rerun_override_kernel_id = None self._gateway_kernel_id = None self._rerun_max_stagger_minutes = 0.0 + self._rerun_expected_output_filename = None self._directly_responsible_user_id = None self._freeze() @@ -502,6 +513,20 @@ def final_leaderboard_disclaimer_message(self, final_leaderboard_disclaimer_mess raise TypeError('final_leaderboard_disclaimer_message must be of type str') self._final_leaderboard_disclaimer_message = final_leaderboard_disclaimer_message + @property + def max_pending_submissions_per_team(self) -> int: + """'Max Pending Submissions Per Team' in the UI. Unset or 0 means no cap.""" + return self._max_pending_submissions_per_team or 0 + + @max_pending_submissions_per_team.setter + def max_pending_submissions_per_team(self, max_pending_submissions_per_team: Optional[int]): + if max_pending_submissions_per_team is None: + del self.max_pending_submissions_per_team + return + if not isinstance(max_pending_submissions_per_team, int): + raise TypeError('max_pending_submissions_per_team must be of type int') + self._max_pending_submissions_per_team = max_pending_submissions_per_team + @property def only_allow_kernel_submissions(self) -> bool: r""" @@ -564,9 +589,11 @@ def max_gpu_runtime_minutes(self, max_gpu_runtime_minutes: Optional[int]): @property def rerun_override_kernel_id(self) -> int: r""" - Host-authored template Notebook the frontend wraps the user's artifact - (e.g. skill.md) into before submitting. Parallels - main_py_override_kernel_id. + Optional host-authored Notebook that is rerun in place of the user's, with + the user's Submission file mounted as a datasource in + /kaggle/input/competition_submission/. Intended for competitions where + users submit an artifact (like skill.md) rather than runnable code. + Requires uses_synchronous_reruns to be set. """ return self._rerun_override_kernel_id or 0 @@ -615,6 +642,25 @@ def rerun_max_stagger_minutes(self, rerun_max_stagger_minutes: float): raise TypeError('rerun_max_stagger_minutes must be of type float') self._rerun_max_stagger_minutes = rerun_max_stagger_minutes + @property + def rerun_expected_output_filename(self) -> str: + r""" + Optional override for the rerun notebook output file which we pass to the + submission scoring metric. If unset and using rerun_override_kernel_id we + default to 'submission.parquet', otherwise default to the filename from the + user's original submission. + """ + return self._rerun_expected_output_filename or "" + + @rerun_expected_output_filename.setter + def rerun_expected_output_filename(self, rerun_expected_output_filename: Optional[str]): + if rerun_expected_output_filename is None: + del self.rerun_expected_output_filename + return + if not isinstance(rerun_expected_output_filename, str): + raise TypeError('rerun_expected_output_filename must be of type str') + self._rerun_expected_output_filename = rerun_expected_output_filename + @property def directly_responsible_user_id(self) -> int: r""" @@ -658,6 +704,7 @@ def directly_responsible_user_id(self, directly_responsible_user_id: Optional[in FieldMetadata("witholdFinalLeaderboardUntilItHasBeenVerified", "withold_final_leaderboard_until_it_has_been_verified", "_withold_final_leaderboard_until_it_has_been_verified", bool, False, PredefinedSerializer()), FieldMetadata("finalLeaderboardHasBeenVerified", "final_leaderboard_has_been_verified", "_final_leaderboard_has_been_verified", bool, False, PredefinedSerializer()), FieldMetadata("finalLeaderboardDisclaimerMessage", "final_leaderboard_disclaimer_message", "_final_leaderboard_disclaimer_message", str, "", PredefinedSerializer()), + FieldMetadata("maxPendingSubmissionsPerTeam", "max_pending_submissions_per_team", "_max_pending_submissions_per_team", int, None, PredefinedSerializer(), optional=True), FieldMetadata("onlyAllowKernelSubmissions", "only_allow_kernel_submissions", "_only_allow_kernel_submissions", bool, False, PredefinedSerializer()), FieldMetadata("usesSynchronousReruns", "uses_synchronous_reruns", "_uses_synchronous_reruns", bool, False, PredefinedSerializer()), FieldMetadata("maxCpuRuntimeMinutes", "max_cpu_runtime_minutes", "_max_cpu_runtime_minutes", int, None, PredefinedSerializer(), optional=True), @@ -665,6 +712,7 @@ def directly_responsible_user_id(self, directly_responsible_user_id: Optional[in FieldMetadata("rerunOverrideKernelId", "rerun_override_kernel_id", "_rerun_override_kernel_id", int, None, PredefinedSerializer(), optional=True), FieldMetadata("gatewayKernelId", "gateway_kernel_id", "_gateway_kernel_id", int, None, PredefinedSerializer(), optional=True), FieldMetadata("rerunMaxStaggerMinutes", "rerun_max_stagger_minutes", "_rerun_max_stagger_minutes", float, 0.0, PredefinedSerializer()), + FieldMetadata("rerunExpectedOutputFilename", "rerun_expected_output_filename", "_rerun_expected_output_filename", str, None, PredefinedSerializer(), optional=True), FieldMetadata("directlyResponsibleUserId", "directly_responsible_user_id", "_directly_responsible_user_id", int, None, PredefinedSerializer(), optional=True), ] diff --git a/kagglesdk/search/types/search_enums.py b/kagglesdk/search/types/search_enums.py index e3187dd..ce5c1ff 100644 --- a/kagglesdk/search/types/search_enums.py +++ b/kagglesdk/search/types/search_enums.py @@ -24,6 +24,7 @@ class DocumentType(enum.Enum): RESOURCEREFERENCE = 13 BENCHMARKTASK = 14 BENCHMARK = 15 + BENCHMARKMODELCARD = 16 class OwnerType(enum.Enum): OWNER_TYPE_UNSPECIFIED = 0 diff --git a/kagglesdk/security/types/security_types.py b/kagglesdk/security/types/security_types.py index b7b05ef..b7c4040 100644 --- a/kagglesdk/security/types/security_types.py +++ b/kagglesdk/security/types/security_types.py @@ -105,6 +105,7 @@ class KaggleResourceType(enum.Enum): KAGGLE_RESOURCE_TYPE_HARNESSES = 720 KAGGLE_RESOURCE_TYPE_HARNESS_VERSIONS = 722 KAGGLE_RESOURCE_TYPE_AGENTS = 724 + KAGGLE_RESOURCE_TYPE_BENCHMARK_CANDIDATES = 726 KAGGLE_RESOURCE_TYPE_COURSE_TRACKS = 800 """---------- Courses ----------""" KAGGLE_RESOURCE_TYPE_COURSE_LESSONS = 802 diff --git a/kagglesdk/users/types/account_service.py b/kagglesdk/users/types/account_service.py index 950b1f9..28851c3 100644 --- a/kagglesdk/users/types/account_service.py +++ b/kagglesdk/users/types/account_service.py @@ -121,8 +121,9 @@ class GenerateAccessTokenRequest(KaggleObject): Attributes: refresh_token (str) Refresh token used to generate a short-lived restricted access token. - If not specified current user credentials will be used to generate an - unrestricted access token. + If not specified, the caller must be authenticated with a KGAT or legacy + API key; the token is tied to that key and can't outlive it or exceed its + scopes. Cookies and short-lived access tokens can't mint without one. api_version (ApiVersion) Version of the Kaggle API for which this token will be used. expiration_duration (timedelta) @@ -161,8 +162,9 @@ def __init__(self): def refresh_token(self) -> str: r""" Refresh token used to generate a short-lived restricted access token. - If not specified current user credentials will be used to generate an - unrestricted access token. + If not specified, the caller must be authenticated with a KGAT or legacy + API key; the token is tied to that key and can't outlive it or exceed its + scopes. Cookies and short-lived access tokens can't mint without one. """ return self._refresh_token or ""