fix(deployer): terminate workloads gracefully

gpustack_runtime/cmds/deployer.py

@@ -362,6 +362,7 @@ class DeleteWorkloadSubCommand(SubCommand):
 
     namespace: str
     name: str
+    grace_period_seconds: int | None
 
     @staticmethod
     def register(parser: _SubParsersAction):
@@ -376,6 +377,13 @@ def register(parser: _SubParsersAction):
             help="Namespace of the workload",
         )
 
+        delete_parser.add_argument(
+            "--grace-period-seconds",
+            type=int,
+            help="Duration in seconds the workload needs to terminate gracefully, "
+            "which overrides the one declared by the workload plan",
+        )
+
         delete_parser.add_argument(
             "name",
             type=str,
@@ -387,6 +395,7 @@ def register(parser: _SubParsersAction):
     def __init__(self, args: Namespace):
         self.namespace = args.namespace
         self.name = args.name
+        self.grace_period_seconds = args.grace_period_seconds
 
         if not self.name:
             msg = "The name argument is required."
@@ -396,6 +405,7 @@ def run(self):
         st = delete_workload(
             name=self.name,
             namespace=self.namespace,
+            grace_period_seconds=self.grace_period_seconds,
         )
         if st:
             print(f"Deleted workload '{self.name}'.")
@@ -428,11 +438,21 @@ def register(parser: _SubParsersAction):
             help="Filter workloads by labels (key=value pairs separated by commas)",
         )
 
+        delete_parser.add_argument(
+            "--grace-period-seconds",
+            type=int,
+            help="Duration in seconds the workloads need to terminate gracefully, "
+            "which overrides the one declared by the workload plans. "
+            "The workloads are deleted one after another, "
+            "so the whole deletion may take this duration per workload",
+        )
+
         delete_parser.set_defaults(func=DeleteWorkloadsSubCommand)
 
     def __init__(self, args: Namespace):
         self.namespace = args.namespace
         self.labels = args.labels
+        self.grace_period_seconds = args.grace_period_seconds
 
     def run(self):
         sts: list[WorkloadStatus] = list_workloads(
@@ -443,6 +463,7 @@ def run(self):
             delete_workload(
                 name=st.name,
                 namespace=st.namespace,
+                grace_period_seconds=self.grace_period_seconds,
             )
             print(f"Deleted workload '{st.name}'.")
         if not sts:

gpustack_runtime/deployer/__init__.py

@@ -150,6 +150,7 @@ def get_workload(
 def delete_workload(
     name: WorkloadName,
     namespace: WorkloadNamespace | None = None,
+    grace_period_seconds: int | None = None,
 ) -> WorkloadStatus | None:
     """
     Delete the given workload.
@@ -159,11 +160,16 @@ def delete_workload(
             The name of the workload to delete.
         namespace:
             The namespace of the workload.
+        grace_period_seconds:
+            Duration in seconds the workload needs to terminate gracefully,
+            which overrides the one declared by the workload plan.
 
     Return:
         The status if found, None otherwise.
 
     Raises:
+        ValueError:
+            If the grace period is negative.
         UnsupportedError:
             If no deployer supports the given workload.
         OperationError:
@@ -174,7 +180,11 @@ def delete_workload(
         if not dep.is_supported():
             continue
 
-        return dep.delete(name=name, namespace=namespace)
+        return dep.delete(
+            name=name,
+            namespace=namespace,
+            grace_period_seconds=grace_period_seconds,
+        )
 
     raise UnsupportedError(_NO_AVAILABLE_DEPLOYER_MSG)
 

gpustack_runtime/deployer/__types__.py

@@ -832,6 +832,12 @@ class WorkloadSecurity:
 Name for a workload.
 """
 
+DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS = 30
+"""
+Default duration in seconds a workload needs to terminate gracefully,
+which aligns with the Kubernetes API server default.
+"""
+
 
 @dataclass_json
 @dataclass
@@ -862,6 +868,8 @@ class WorkloadPlan(WorkloadSecurity):
             The group ID to own the filesystem of the workload.
         sysctls (dict[str, str] | None):
             Sysctls to set for the workload.
+        termination_grace_period_seconds (int):
+            Duration in seconds the containers of the workload need to terminate gracefully.
         containers (list[Container] | None):
             Containers in the workload.
             It must contain at least one "RUN" profile container.
@@ -909,6 +917,12 @@ class WorkloadPlan(WorkloadSecurity):
     Containers in the workload.
     It must contain at least one "RUN" profile container.
     """
+    termination_grace_period_seconds: int = DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS
+    """
+    Duration in seconds the containers of the workload need to terminate gracefully.
+    Containers are signaled to stop, and killed once the duration elapses.
+    Zero means killing immediately.
+    """
 
     ##
     ## The below are internal use only fields.
@@ -957,6 +971,15 @@ def validate_and_default(self):
             msg = 'Workload must contain at least one "RUN" profile container.'
             raise ValueError(msg)
 
+        # Default and validate termination grace period.
+        if self.termination_grace_period_seconds is None:
+            self.termination_grace_period_seconds = (
+                DEFAULT_TERMINATION_GRACE_PERIOD_SECONDS
+            )
+        elif self.termination_grace_period_seconds < 0:
+            msg = "Workload termination grace period must not be negative."
+            raise ValueError(msg)
+
         # Validate workload labels, including label names and values.
         for ln, lv in self.labels.items():
             for s in ln.split("/"):
@@ -2109,6 +2132,8 @@ def delete(
         self,
         name: WorkloadName,
         namespace: WorkloadNamespace | None = None,
+        *,
+        grace_period_seconds: int | None = None,
         async_mode: bool | None = None,
     ) -> WorkloadStatus | None:
         """
@@ -2119,6 +2144,9 @@ def delete(
                 The name of the workload.
             namespace:
                 The namespace of the workload.
+            grace_period_seconds:
+                Duration in seconds the workload needs to terminate gracefully,
+                which overrides the one declared by the workload plan.
             async_mode:
                 Whether to execute in a separate thread.
 
@@ -2128,16 +2156,23 @@ def delete(
         Raises:
             UnsupportedError:
                 If the deployer is not supported in the current environment.
+            ValueError:
+                If the grace period is negative.
             OperationError:
                 If the workload fails to delete.
 
         """
+        if grace_period_seconds is not None and grace_period_seconds < 0:
+            msg = "Workload termination grace period must not be negative."
+            raise ValueError(msg)
+
         if async_mode:
             try:
                 future = self.pool.submit(
                     self._delete,
                     name,
                     namespace,
+                    grace_period_seconds,
                 )
                 return future.result()
             except OperationError:
@@ -2146,13 +2181,14 @@ def delete(
                 msg = "Asynchronous workload delete failed."
                 raise OperationError(msg) from e
         else:
-            return self._delete(name, namespace)
+            return self._delete(name, namespace, grace_period_seconds)
 
     @abstractmethod
     def _delete(
         self,
         name: WorkloadName,
         namespace: WorkloadNamespace | None = None,
+        grace_period_seconds: int | None = None,
     ) -> WorkloadStatus | None:
         """
         Delete a workload.
@@ -2162,6 +2198,9 @@ def _delete(
                 The name of the workload.
             namespace:
                 The namespace of the workload.
+            grace_period_seconds:
+                Duration in seconds the workload needs to terminate gracefully,
+                which overrides the one declared by the workload plan.
 
         Return:
             The status if found, None otherwise.