From 68e5ff803ed108dca7a19507f3465bd0fa33c064 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Mon, 17 Aug 2026 06:35:28 +0000 Subject: [PATCH 1/4] feat: Resumable Media Upload functionality implementation Public API `Gapic::Rest::ResumableUpload::Session` is the entry point. `Session#run` for initial upload, and `Session#resume` for resuming after error. Also public: `Progress`, `ResumeHandle`, `HasResumeHandle` module, and a typed error hierarchy. Changes to existing files: * `StubLogger#warn` * request-payload abridging in `ClientStub` so that binary upload bodies don't land in logs * `REST_ERROR_PREFIX` constant extracted in Rest::Error Integration testing against Showcase, for manual confirmation (disabled in CI). CI integration tests are planned to land in the generator. --- gapic-common/.toys/test-integration.rb | 139 +++ .../resumable_upload/implementation-guide.md | 687 +++++++++++ .../resumable_upload/integration-test-plan.md | 286 +++++ gapic-common/integration/README.md | 51 + .../integration/integration_helper.rb | 204 ++++ .../chunk_granularity_test.rb | 48 + .../resumable_upload/error_on_start_test.rb | 159 +++ .../resumable_upload/error_recovery_test.rb | 144 +++ .../resumable_upload/golden_path_test.rb | 111 ++ .../resumable_upload/resume_test.rb | 229 ++++ gapic-common/lib/gapic/logging_concerns.rb | 4 + gapic-common/lib/gapic/rest.rb | 1 + gapic-common/lib/gapic/rest/client_stub.rb | 16 +- gapic-common/lib/gapic/rest/error.rb | 5 +- .../lib/gapic/rest/resumable_upload.rb | 73 ++ .../lib/gapic/rest/resumable_upload/core.rb | 77 ++ .../gapic/rest/resumable_upload/data_types.rb | 415 +++++++ .../lib/gapic/rest/resumable_upload/driver.rb | 780 +++++++++++++ .../rest/resumable_upload/driver/abridge.rb | 168 +++ .../resumable_upload/driver/upload_log.rb | 326 ++++++ .../lib/gapic/rest/resumable_upload/errors.rb | 565 +++++++++ .../lib/gapic/rest/resumable_upload/events.rb | 128 +++ .../rest/resumable_upload/instructions.rb | 265 +++++ .../rest/resumable_upload/retry_policies.rb | 165 +++ .../lib/gapic/rest/resumable_upload/rules.rb | 1016 +++++++++++++++++ .../gapic/rest/resumable_upload/session.rb | 546 +++++++++ .../test/gapic/rest/client_stub_test.rb | 41 + gapic-common/test/gapic/rest/error_test.rb | 13 + .../gapic/rest/resumable_upload/core_test.rb | 85 ++ .../rest/resumable_upload/data_types_test.rb | 231 ++++ .../resumable_upload/driver/abridge_test.rb | 101 ++ .../driver/upload_log_test.rb | 216 ++++ .../resumable_upload/driver_buffer_test.rb | 378 ++++++ .../resumable_upload/driver_config_test.rb | 295 +++++ .../driver_error_mapping_test.rb | 209 ++++ .../resumable_upload/driver_logging_test.rb | 574 ++++++++++ .../resumable_upload/driver_progress_test.rb | 178 +++ .../driver_retry_policy_test.rb | 143 +++ .../resumable_upload/driver_retry_test.rb | 187 +++ .../rest/resumable_upload/driver_test.rb | 430 +++++++ .../resumable_upload/retry_policies_test.rb | 222 ++++ .../rules_classification_test.rb | 351 ++++++ .../resumable_upload/rules_decide_test.rb | 334 ++++++ .../rest/resumable_upload/rules_error_test.rb | 474 ++++++++ .../resumable_upload/rules_recovery_test.rb | 134 +++ .../gapic/rest/resumable_upload/rules_test.rb | 279 +++++ .../rest/resumable_upload/session_test.rb | 704 ++++++++++++ gapic-common/test/test_helper.rb | 25 + 48 files changed, 12208 insertions(+), 4 deletions(-) create mode 100644 gapic-common/.toys/test-integration.rb create mode 100644 gapic-common/design/resumable_upload/implementation-guide.md create mode 100644 gapic-common/design/resumable_upload/integration-test-plan.md create mode 100644 gapic-common/integration/README.md create mode 100644 gapic-common/integration/integration_helper.rb create mode 100644 gapic-common/integration/resumable_upload/chunk_granularity_test.rb create mode 100644 gapic-common/integration/resumable_upload/error_on_start_test.rb create mode 100644 gapic-common/integration/resumable_upload/error_recovery_test.rb create mode 100644 gapic-common/integration/resumable_upload/golden_path_test.rb create mode 100644 gapic-common/integration/resumable_upload/resume_test.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/core.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/data_types.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/errors.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/events.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/instructions.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/rules.rb create mode 100644 gapic-common/lib/gapic/rest/resumable_upload/session.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/core_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/driver_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/rules_test.rb create mode 100644 gapic-common/test/gapic/rest/resumable_upload/session_test.rb diff --git a/gapic-common/.toys/test-integration.rb b/gapic-common/.toys/test-integration.rb new file mode 100644 index 0000000..7929523 --- /dev/null +++ b/gapic-common/.toys/test-integration.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "socket" +require "tmpdir" + +expand :minitest, name: "" do |t| + t.libs = ["lib", "integration"] + t.files = ["integration/**/*_test.rb"] + t.bundler = true +end + +alias_method :run_minitest, :run + +def run + if !ENV["SHOWCASE_ENDPOINT"].to_s.empty? + run_minitest + return + end + + bin = resolve_showcase_bin + if bin.nil? + if !ENV["CI"].to_s.empty? + logger.error "No SHOWCASE_ENDPOINT or gapic-showcase binary found in CI environment." + exit 1 + else + logger.warn "Skipping integration tests: no SHOWCASE_ENDPOINT or gapic-showcase binary found." + return + end + end + + verify_showcase_version! bin + + port = allocate_port + fallback_port = allocate_port + log_path = File.join Dir.tmpdir, "gapic-showcase-#{Process.pid}-#{Time.now.to_i}.log" + + pid = Process.spawn( + bin, "run", + "--port", ":#{port}", + "--fallback-port", ":#{fallback_port}", + out: log_path, + err: log_path, + pgroup: true + ) + + begin + wait_for_showcase! pid, port, log_path + ENV["SHOWCASE_ENDPOINT"] = "http://localhost:#{port}" + run_minitest + ensure + if pid + begin + Process.kill "-TERM", pid + Process.waitpid pid + rescue Errno::ESRCH, Errno::ECHILD + # Process already terminated + end + end + end +end + +def resolve_showcase_bin + env_bin = ENV["SHOWCASE_BIN"].to_s + return env_bin unless env_bin.empty? + + ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).each do |dir| + candidate = File.join dir, "gapic-showcase" + return candidate if File.executable?(candidate) && !File.directory?(candidate) + end + nil +end + +def verify_showcase_version!(bin) + output = begin + IO.popen([bin, "--version"], err: [:child, :out], &:read).strip + rescue StandardError => e + logger.error "Failed to execute '#{bin} --version': #{e.message}" + exit 1 + end + + version_match = output[/\d+\.\d+(?:\.\d+)*/] + if version_match.nil? + logger.error "Could not parse version from '#{bin} --version' output: #{output.inspect}" + exit 1 + end + + if Gem::Version.new(version_match) < Gem::Version.new("0.43") + logger.error "gapic-showcase version #{version_match} is too old (minimum required is 0.43)." + exit 1 + end +end + +def allocate_port + server = TCPServer.open "127.0.0.1", 0 + port = server.addr[1] + server.close + port +end + +def wait_for_showcase!(pid, port, log_path) + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 10.0 + + loop do + exited_pid, status = Process.waitpid2 pid, Process::WNOHANG + if exited_pid + logger.error "gapic-showcase exited prematurely (status: #{status.exitstatus}). Log file: #{log_path}" + exit 1 + end + + begin + sock = TCPSocket.new "127.0.0.1", port + sock.close + return + rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH + # Server not ready yet + end + + if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + logger.error "Timed out waiting 10s for gapic-showcase to listen on port #{port}. Log file: #{log_path}" + exit 1 + end + + sleep 0.1 + end +end diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md new file mode 100644 index 0000000..3a561a9 --- /dev/null +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -0,0 +1,687 @@ +# Resumable Upload Protocol (RUP) Implementation Guide + +## 1. System Architecture + +The Resumable Upload Protocol (RUP) implementation in `gapic-common` is structured across three distinct tiers to separate network execution, protocol state progression, and state transition decision logic: + +```mermaid +graph TD + Client[Client Code] -->|StartUploadConfig| Driver + subgraph Gapic::Rest::ResumableUpload + Driver[Driver
Synchronous I/O Adapter] -->|Events| Core[Core
State Container] + Core -->|Instructions| Driver + Core -->|state, event| Rules[Rules
Pure Decision Function] + Rules -->|next_state, instructions| Core + end + Driver -->|RetryPolicy / Faraday| Server[Upload Backend / GCS] + Driver -->|IO#read| Stream[Local Stream] +``` + +### 1.0 Domain Vocabulary +* **Upload**: Server-side entity created by a successful session initiation (`start`), identified by `upload_url`. +* **Resume Handle (`ResumeHandle`)**: An immutable snapshot (`upload_url`, `chunk_size`) identifying an upload for resumption. +* **Session (`Session`)**: Client-side transfer coordinator; performs exactly one run (`start` or `resume`), never both, never twice. It owns the input stream and configuration options. +* **Run**: One invocation of `Driver#run` (either a start or resume execution). +* **Bound**: The property that a session has executed a run or is bound to an upload (`session.bound?`). A session becomes bound when `start` or `resume` begins execution. A bound session never runs again. + +### 1.1 Driver (Synchronous I/O Adapter) +The `Driver` executes all operations with side-effects. It interacts with HTTP transport via `Gapic::Rest::ClientStub`, reads binary data from local input streams, tracks monotonic execution deadlines, and dispatches progress callbacks. + +Crucially, the Driver delegates all **Category 1 (Transient)** transport retries directly to `Gapic::Common::RetryPolicy`. Transient retries occur entirely within the Driver's network execution wrapper. The `Core` state machine is never exposed to transient noise, receiving only verified successful HTTP responses or terminal transport exceptions. + +The Driver exposes: +* `Driver#resume_handle`: Returns a `ResumeHandle` (or `nil` if initiation has not established an upload URL, or if the session is `:rejected`, `:cancelled`, or `:success`; completed uploads are not resumable). Reading this property mid-run provides a best-effort snapshot of current session parameters. +* `Driver#upload_url`: Returns the raw protocol state upload URL under any status (`:active`, `:success`, `:rejected`, `:cancelled`). + +### 1.2 Core (State Container) +The `Core` maintains the immutable `State` snapshot. When `Core#dispatch(event)` is invoked by the Driver, Core forwards `@state`, the event, and static configuration to `Rules.decide`. Core mutates `@state` to `decision.next_state`, records the decision in `@last_decision`, and returns `decision.instructions` back to the Driver. Core contains zero protocol branching logic and zero side effects. + +### 1.3 Rules (Pure Decision Function) +The `Rules` module encapsulates the Resumable Upload Protocol state transitions as a pure functional module. Given a state snapshot, an input event, and configuration, `Rules.decide` evaluates the transition router and returns a `Decision` snapshot containing `from_status`, `shape`, `recipe`, `next_state`, and `instructions`. + +### 1.4 Stream Buffering +Because arbitrary Ruby `IO` objects (network sockets, pipes, `STDIN`) do not support seeking (`#seek`), the Driver buffers the current in-flight chunk in memory (bounded by chunk size, default: 8MB). When `RetryPolicy` executes transport retries, or when `Core` triggers Category 2 recovery realignments within the buffered range, the Driver retransmits directly from memory. The buffer is discarded only after receiving a `200 OK` durably confirming receipt of the chunk. + +### 1.5 Session (Transfer Coordinator) +The `Session` (`Gapic::Rest::ResumableUpload::Session`) provides a client-facing coordinator that encapsulates shared transfer configuration, owns the input stream, and manages upload execution across a strict single-run lifecycle. + +#### Single-Run Contract & Two-State Model +A session adheres to a two-state model with a strict single-run contract: a session performs exactly one run (`start` or `resume`), never both, never twice. + +1. **Unbound (`!session.bound?`)**: + * Initial state upon construction (`Session.new`). The session has not yet executed a run. + * Permitted operations: `start(...)` or `resume(...)`. +2. **Bound (`session.bound?`)**: + * Transitions to bound as soon as `start` or `resume` begins execution. + * The session has executed its run and cannot be reused. + * Both `start` and `resume` raise `SessionStateError` ("Session has already executed a run"). + +#### Constructor & Initiation Signatures +Configuration is split between transfer-wide options passed to `Session.new` (`COMMON_MEMBERS` plus `client_stub` and `logger`) and initiation-only arguments passed to `Session#start`: +* **Constructor (`Session#initialize`)**: + `Session.new(client_stub:, stream:, upload_size: nil, content_type: nil, timeout: nil, control_plane_retry_policy: nil, data_plane_retry_policy: nil, on_progress: nil, logger: nil)` +* **Initiation (`Session#start`)**: + `session.start(initial_url:, initial_body: nil, initial_headers: {}, chunk_size: nil, start_retry_policy: nil)` + * `initial_url` is required (`ArgumentError` if missing or blank). + * `initial_headers` accepts caller-supplied HTTP headers for the initiation request, merged over the driver's headers. Any key in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`, in any casing) raises an `ArgumentError`; callers influence `X-Goog-Upload-Header-Content-Type` and `X-Goog-Upload-Header-Content-Length` through `content_type:` and `upload_size:` on the constructor. Pass-through headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. + +#### Resumability (`session.resumable?`) +* Reports whether a *new* session can resume the transfer (`!session.resume_handle.nil?`). +* Completed uploads are finalized: once a transfer succeeds, `resume_handle` returns `nil` and `resumable?` returns `false`, so there is no handle to resume from. Calling `#resume` on the session that completed the run raises `SessionStateError`. Resuming a *fresh* session against a finalized `upload_url` is undefined behavior: it queries the server and might return the response body or raise an error, depending on the server response. +* When a run fails with a recoverable error, `resume_handle` captures the upload parameters (`upload_url`, `chunk_size`) and `resumable?` returns `true`. + +#### Precondition on Stream Position for Resume +* Before executing `resume`, the caller must ensure the input stream is positioned at byte 0. +* If `stream.respond_to?(:pos) && !stream.pos.zero?`, `Session#resume` raises `ArgumentError` ("Input stream must be at byte 0 to resume; rewind the stream before resuming"). +* For unseekable streams without `:pos` (or streams at `pos == 0`), `Session` trusts the stream is at byte 0 and delegates to `Driver`, which fast-forwards to the server-confirmed offset by seeking or reading and discarding bytes. + +#### Resume Invocations & Forms +The `Session#resume` method accepts strictly keyword-only arguments: `upload_url: nil, chunk_size: nil, resume_handle: nil`. +Resumption always requires an unbound session with one of two mutually exclusive parameter forms: +1. **Explicit URL & Chunk Size**: `session.resume(upload_url: url, chunk_size: size)` +2. **Resume Handle**: `session.resume(resume_handle: handle)` + +Calling `resume` without arguments (bare resume), calling `resume` with `upload_url` but omitting `chunk_size`, or mixing `resume_handle` with other parameters raises `ArgumentError`. + +#### Cross-Session Resumption Flow +Because a session performs only a single run, resuming an interrupted upload requires instantiating a fresh session: +1. Session 1 encounters a recoverable error. +2. Caller extracts `resume_handle = session1.resume_handle` (or from the error's `#resume_handle`). +3. Caller rewinds the stream to byte 0 (if seekable, or provides an equivalent stream starting at byte 0). +4. Caller instantiates Session 2 and invokes `session2.resume(resume_handle: resume_handle)`. + +#### Concurrency & Execution Model +* At most one run (`Driver#run`) may execute at any time. +* `@running` is checked and toggled exclusively inside a `Mutex`. +* Network execution (`driver.run`) occurs outside the mutex to prevent blocking reader threads. +* Invoking `start` or `resume` while `@running` is `true` raises `SessionStateError`. +* Errors propagate unchanged. The failed `Driver` remains referenced so `upload_url`, `resume_handle`, and `bound?` remain inspectable after an exception. + +--- + +## 2. Component Interfaces & Data Models + +### 2.1 Initiation Configuration (`StartUploadConfig`) +```ruby +module Gapic + module Rest + module ResumableUpload + COMMON_MEMBERS = [ + :stream, # [IO] Binary input stream to upload + :upload_size, # [Integer, nil] Total upload bytes if known upfront + :content_type, # [String, nil] MIME type of uploaded media + :timeout, # [Numeric, nil] Total upload timeout in seconds (zero/negative treated as nil) + :control_plane_retry_policy, # [Gapic::Common::RetryPolicy, Hash, nil] Policy or hash override for query/cancel commands + :data_plane_retry_policy, # [Gapic::Common::RetryPolicy, Hash, nil] Policy or hash override for upload/finalize + :on_progress # [Proc, nil] Callback: ->(progress) with a Progress instance + ].freeze + + RESERVED_INITIAL_HEADERS = [ + "x-goog-upload-protocol", + "x-goog-upload-command", + "x-goog-upload-offset", + "x-goog-upload-header-content-type", + "x-goog-upload-header-content-length" + ].freeze + + StartUploadConfig = Data.define( + *COMMON_MEMBERS, + :initial_url, # [String] Initial endpoint URI for session initiation + :initial_body, # [String, nil] Request payload for session initiation + :initial_headers, # [Hash] Additional headers for initiation (RESERVED_INITIAL_HEADERS rejected) + :chunk_size, # [Integer, nil] Explicit chunk size in bytes + :start_retry_policy # [Gapic::Common::RetryPolicy, Hash, nil] Policy or hash override for start command + ) + + Progress = Data.define( + :phase, # [Symbol] Upload lifecycle phase, one of Progress::PHASES + :bytes_uploaded, # [Integer] Cumulative bytes acknowledged by the server (may decrease on recovery rewind) + :total_bytes # [Integer, nil] Total upload size in bytes if known + ) do + # Important to define it via `self.`, since this block is not a class body + self::PHASES = %i[initiating uploading recovering finalizing cancelling completed].freeze + end + end + end +end +``` + +**Reserved Initial Headers Rule (`RESERVED_INITIAL_HEADERS`):** +* The five headers in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`) are protocol machinery owned by the driver. +* Any key in `initial_headers` matching those five names (case-insensitively) is rejected at configuration construction with an `ArgumentError`. Callers shape media descriptors exclusively through `content_type` and `upload_size`. Pass-through headers under the prefix such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. + +**Progress Notification Contract (`on_progress`):** +* `on_progress` fires whenever upload status or server-confirmed byte offset changes. Sequential callbacks may report the same `bytes_uploaded`. +* `bytes_uploaded` represents the server-confirmed offset and is **not guaranteed to be monotonic** — a server rewind during recovery can decrease this value. +* Terminal failures and completed cancellations do not emit `Progress` notifications; however, entering the `:cancelling` phase does. +* Public phases (`Progress::PHASES`): `:initiating`, `:uploading`, `:recovering`, `:finalizing`, `:cancelling`, `:completed`. + +### 2.2 Resume Configuration (`ResumeUploadConfig`) +```ruby +module Gapic + module Rest + module ResumableUpload + ResumeUploadConfig = Data.define( + *COMMON_MEMBERS, + :upload_url, # [String] Upload session URL returned by the upload backend + :chunk_size # [Integer] Chunk size in bytes (> 0) + ) + end + end +end +``` +`ResumeUploadConfig` allows resuming an existing session directly using the session URL (typically obtained from `ResumeHandle#upload_url` or an error's `#resume_handle`). Because a resumed run skips session initiation, `start_retry_policy` and initiation headers/URL are absent. + +### 2.3 Protocol State (`State`) & Decisions (`Decision`) +```ruby +module Gapic + module Rest + module ResumableUpload + State = Data.define( + :status, # [Symbol] :initializing, :starting, :transmission_reading, :transmission_sending, + # :finalizing_sending_upload, :finalizing_sending_finalize, + # :recovery, :cancelling, :cancelled, :success, :error, :rejected + :upload_url, # [String, nil] Session upload URL returned by the upload backend + :offset, # [Integer] Contiguous bytes confirmed by server (protocol_state_offset) + :chunk_size, # [Integer] Resolved effective chunk size + :chunk_granularity, # [Integer, nil] Alignment modulus returned by server + :in_flight_length, # [Integer] Byte length of in-flight chunk currently being transmitted + :last_error # [StandardError, nil] Terminal exception + ) do + end + + Decision = Data.define( + :from_status, # [Symbol] Status before transition + :shape, # [Symbol] Classified canonical event shape + :recipe, # [Symbol] Selected transition recipe method name + :next_state, # [State] Resulting protocol state snapshot + :instructions # [Array] Emitted instructions for the Driver + ) + end + end +end +``` + +### 2.4 Resume Handle (`ResumeHandle`) +```ruby +module Gapic + module Rest + module ResumableUpload + ResumeHandle = Data.define( + :upload_url, # [String] Upload session URL provided by the server + :chunk_size # [Integer] Effective chunk size in bytes + ) + end + end +end +``` +`ResumeHandle` captures server-provided parameters that can be persisted to resume the upload session at a later time. + +### 2.5 Events Vocabulary (Driver -> Core) +* `Event::StartUpload`: Start a new upload session. +* `Event::ResumeUpload.new(upload_url:, chunk_size:, upload_size:)`: Resume an existing upload session with a known upload URL. +* `Event::ChunkRead.new(bytes_buffered:, eof:)`: Binary data buffered in Driver memory; reports total bytes ready in buffer and whether the stream hit EOF. +* `Event::HttpResponse.new(status:, headers:, body:, error: nil)`: Dispatched for any completed HTTP exchange over the wire (including 2xx, 4xx, 5xx, or responses with missing/unexpected headers). Carries optional parsed `error` (`Gapic::Rest::Error`) when rescued from transport errors. `Core` inspects status and headers to determine protocol progression or recovery. +* `Event::RequestFailed.new(kind:, message:, source_error:)`: Dispatched when an HTTP request fails to produce a usable HTTP response (e.g., request timeout, transport connection errors, or `RetryPolicy` exhaustion). + * `kind`: Normalized Symbol enum (`:timeout`, `:connection_failed`, `:retries_exhausted`). `Core` branches on `kind` and treats other fields as opaque. + * `message`: Human-readable summary string. + * `source_error`: Original underlying exception, preserved for terminal error propagation and logging. +* `Event::Cancel`: Caller requested session cancellation. +* `Event::GlobalDeadlineExceeded`: Absolute monotonic clock exceeded the session deadline (`@deadline`) computed at the start of `Driver#run`. + +### 2.6 Instructions Vocabulary (Core -> Driver) +* `Instruction::SendStart.new(url:, headers:, body:)`: Execute initiation request to establish upload session. +* `Instruction::SendChunk.new(url:, offset:, length:, finalize:)`: Transmit buffered chunk of specified `length` starting at `offset`. If `finalize` is true, sends command `upload, finalize`. +* `Instruction::SendFinalize.new(url:)`: Send standalone `finalize` command when all data bytes were already acknowledged. +* `Instruction::SendQuery.new(url:)`: Query backend for current acknowledged offset (`query` command). +* `Instruction::SendCancel.new(url:)`: Cancel upload session on server (`cancel` command). +* `Instruction::RealignBuffer.new(server_offset:)`: Realign Driver in-memory buffer and stream position to match `server_offset`. +* `Instruction::FillBuffer.new(target_bytesize:)`: Read from stream until in-memory buffer reaches `target_bytesize` bytes or stream encounters EOF. +* `Instruction::NotifyProgress.new(progress:)`: Invoke `on_progress` callback with a `Progress` instance. +* `Instruction::TerminateSuccess.new(response:)`: Upload finalized cleanly; Driver returns `response.body`. +* `Instruction::TerminateFailure.new(error:)`: Raise terminal exception. + +### 2.5 Driver Buffer Invariants & Stream Position Model + +The Driver coordinates stream reading and in-memory buffering using four explicit offset markers: +* `server_offset`: Contiguous byte count acknowledged by the server (extracted from `X-Goog-Upload-Size-Received`). +* `protocol_state_offset`: Byte offset maintained in `State.offset`. +* `buffer_start_offset`: Absolute stream offset corresponding to the first byte in the Driver's `@buffer`. +* `buffer_end_offset`: `buffer_start_offset + @buffer.bytesize`. + +```text +Stream Offset: 0 -----------------> buffer_start_offset -------------------> buffer_end_offset ----> (Stream EOF) + |----------------- @buffer -------------| + ^ + server_offset +``` + +#### Buffer Alignment Strategy (`Instruction::RealignBuffer`) +When `Core` resolves a recovery query or offset realignment, the Driver executes one of three alignment paths based on `server_offset`: + +1. **Case 1: Within Buffer Range (`buffer_start_offset <= server_offset <= buffer_end_offset`)** + * The required offset is already buffered in memory. + * Driver trims already-persisted bytes: `@buffer = @buffer.byteslice((server_offset - buffer_start_offset)..-1)`. + * Driver updates `buffer_start_offset = server_offset`. + * When subsequently executing `Instruction::FillBuffer(target_bytesize)`, Driver calculates `needed = target_bytesize - @buffer.bytesize` and reads only the missing difference from `stream` to complete the chunk to full `chunk_size` (unless stream reaches EOF). +2. **Case 2: Server Offset Behind Buffer (`server_offset < buffer_start_offset`)** + * Occurs if the server rolls back beyond the retained buffer window. + * If `stream.respond_to?(:seek)`: Driver seeks the stream back to `server_offset`, resets `@buffer = "".b`, and sets `buffer_start_offset = server_offset`. + * If `stream` is unseekable (e.g. Socket, Pipe, STDIN): Driver raises a terminal `UnseekableStreamError` (Category 3 failure), attaching `resume_handle`. +3. **Case 3: Server Offset Ahead of Buffer (`server_offset > buffer_end_offset`)** + * Occurs when resuming an existing session or when the server processed a previously timed-out request ahead of local state. + * If total `upload_size` is known and `server_offset > upload_size`, Driver raises a terminal `StreamMismatchError` with `resume_handle`. + * If `upload_size` is `nil` and `stream.respond_to?(:size)` and `server_offset > stream.size`, Driver raises a terminal `StreamMismatchError` with `resume_handle` (preventing seek past EOF from silently succeeding on seekable streams). + * Driver resets `@buffer = "".b`. + * Driver advances the stream to `server_offset`: + * If seekable: `stream.seek(server_offset)`. + * If unseekable: Driver reads and discards bytes from `stream` until reaching `server_offset` (reading `server_offset - buffer_end` bytes). If the stream encounters an unexpected EOF before reaching `server_offset`, Driver raises a terminal `StreamMismatchError` with `resume_handle`. + * Driver sets `buffer_start_offset = server_offset`. + +--- + +## 3. Component Architecture + +The authoritative implementation is the source itself, under `lib/gapic/rest/resumable_upload/`. This section describes the contract each component honours; the code is normative where the two disagree. + +### 3.1 Rules Module (`Gapic::Rest::ResumableUpload::Rules`) +The `Rules` module is a pure functional transition engine with zero state awareness and zero side effects. It provides two primary entry points: +* `Rules.shape_of(event)`: Classifies raw input events (`Event::StartUpload`, `Event::ChunkRead`, `Event::HttpResponse`, `Event::RequestFailed`, `Event::Cancel`, `Event::GlobalDeadlineExceeded`) into canonical symbols. +* `Rules.decide(state, event, config)`: Evaluates `case [state.status, shape]` pattern matching to select a transition recipe symbol, dispatches via `public_send(recipe, state, event, config)`, and returns a `Decision` snapshot (`from_status`, `shape`, `recipe`, `next_state`, `instructions`). +* `Rules.step(state, event, config)`: Convenience tuple wrapper around `Rules.decide` returning `[decision.next_state, decision.instructions]`. + +Source: `lib/gapic/rest/resumable_upload/rules.rb` + +### 3.2 Core Class (`Gapic::Rest::ResumableUpload::Core`) +The `Core` class is the state container holding the immutable `State` snapshot. It exposes: +* `#state`: Reader for the current `State` snapshot. +* `#last_decision`: Reader for the `Decision` recorded during the most recent `#dispatch` (or `nil`). +* `#dispatch(event)`: Invokes `Rules.decide(@state, event, @config)`, updates `@state = decision.next_state` and `@last_decision = decision`, and returns `decision.instructions` to the Driver. + +Source: `lib/gapic/rest/resumable_upload/core.rb` + +### 3.3 Driver Class (`Gapic::Rest::ResumableUpload::Driver`) +The `Driver` is the synchronous execution engine for the pure protocol state machine. When `Core#dispatch(event)` is invoked, it returns an ordered list (`Array`) of commands that the Driver executes in sequence. + +#### Instruction Processing Semantics +The Driver categorizes instructions into three execution types: +1. **Synchronous Side-Effects** (`NotifyProgress`, `RealignBuffer`): + * Executed immediately in-process. + * Do not yield a new `Event` and do not break the batch loop. Exceptions raised within user callbacks (e.g. `on_progress`) are not swallowed and immediately propagate to the caller. +2. **I/O & Network Operations** (`FillBuffer`, `SendStart`, `SendChunk`, `SendFinalize`, `SendQuery`, `SendCancel`): + * Execute physical stream reads or HTTP requests (wrapped in `Gapic::Common::RetryPolicy` for Category 1 transient errors). + * Yield a single resulting `Event` (`ChunkRead`, `HttpResponse`, or `RequestFailed`) that becomes the input for the next cycle. +3. **Terminal Handlers** (`TerminateSuccess`, `TerminateFailure`): + * Break the event loop and return the final response body string (`response.body`) or raise the terminal exception. + +Source: `lib/gapic/rest/resumable_upload/driver.rb` + +--- + +## 4. State Machine Protocol Rules + +### 4.1 Upstream Protocol Contract +1. **Logical Header Prefixing**: In the `start` request, logical headers describing the uploaded object must be prefixed with `X-Goog-Upload-Header-`. Specifically: + * `X-Goog-Upload-Header-Content-Type: config.content_type` + * `X-Goog-Upload-Header-Content-Length: config.upload_size` (if known upfront). + * Callers cannot supply either of these two headers via `initial_headers`; see the reserved-headers rule in Section 2 (doing so raises an `ArgumentError`). Other `X-Goog-Upload-Header-*` pass-through headers are permitted. +2. **Offset Extraction**: On `query` responses, the acknowledged byte count is extracted from `X-Goog-Upload-Size-Received` as an integer (`server_offset`). +3. **Request Modification on 4xx**: Retrying Category 2 errors requires querying the backend for `server_offset` first. +4. **Standard Retry Configuration & Distinct Policies**: The Driver manages distinct retry policy configurations for Category 1 transient errors: + * **Start Policy (`start_retry_policy`)**: Applies specifically to session initiation (`start`). Configured with standard retry codes (`["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"]`) and network errors (`[Faraday::ConnectionFailed, Faraday::TimeoutError, SocketError]`). A missing or empty `X-Goog-Upload-Status` header is treated as **retriable** (predicate returns `true`) across **any response code, including 200 OK**. + * **Control Plane Policy (`control_plane_retry_policy`)**: Applies to session control requests (`query`, `cancel`). Configured with standard retry codes and network errors. It does **not** retry on a missing `X-Goog-Upload-Status` header, allowing `Core` to evaluate responses immediately. + * **Data Plane Policy (`data_plane_retry_policy`)**: Applies to data transmission requests (`upload`, `upload,finalize`, and standalone `finalize`). Shares the standard retry codes and network errors, but treats a missing `X-Goog-Upload-Status` header as **unretriable** (predicate returns `false`). This prevents blind chunk re-transmission and returns `Event::HttpResponse` immediately to `Core` so it can initiate Category 2 `Recovery`. + * **Retry Policy Override Contract**: Each retry policy configuration field accepts a `Gapic::Common::RetryPolicy` instance, a `Hash`, or `nil`. Passing a `RetryPolicy` instance replaces the default policy entirely. Passing a `Hash` constructs a new `RetryPolicy` and applies the category's defaults (`RetryPolicy.new(**hash).apply_defaults(defaults)`), overriding the specified fields while preserving unspecified defaults such as `retry_codes` and `retry_predicate`. Passing `nil` constructs the default policy directly from the category defaults. + +### 4.2 State Transition & Data Mutation Specification + +**State Classification:** +* **Non-Terminal States**: `Initializing`, `Starting`, `Transmission | Reading from stream`, `Transmission | Sending`, `Finalizing | Sending with upload`, `Finalizing | Sending finalize`, `Recovery`, `Cancelling`. +* **Terminal States**: `Success`, `Cancelled`, `Error`, `Rejected`. + +| From State | Event Shape | Event & Input Payload | State Mutations | To State | Emitted Instructions & Parameters | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **`Initializing`** | `:start_upload` | `Event::StartUpload` | `status = :starting` | `Starting` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::SendStart.new(url: config.initial_url, headers: config.initial_headers, body: config.initial_body)` | +| **`Initializing`** | `:resume_upload` | `Event::ResumeUpload` | `upload_url = event.upload_url`
`chunk_size = event.chunk_size`
`offset = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: event.upload_size))`
`Instruction::SendQuery.new(url: event.upload_url)` | +| **`Starting`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `upload_url = headers['X-Goog-Upload-URL']`
`chunk_granularity = headers['...-Granularity']&.to_i`
`chunk_size = resolve(config, chunk_granularity)`
`offset = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: config.upload_size))`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Starting`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Starting`** | `:response_cat2` / `:response_fatal_bad_response` | `Event::HttpResponse` (Non-200; see Section 6.1) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Starting`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind:, message:, source_error:)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Transmission \| Reading from stream`** | `:chunk_read_full` | `Event::ChunkRead(bytes_buffered, eof: false)` | `in_flight_length = event.bytes_buffered`
`status = :transmission_sending` | `Transmission \| Sending` | `Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: false)` | +| **`Transmission \| Reading from stream`** | `:chunk_read_eof_with_data` | `Event::ChunkRead(bytes_buffered, eof: true)` where `bytes_buffered > 0` | `in_flight_length = event.bytes_buffered`
`status = :finalizing_sending_upload` | `Finalizing \| Sending with upload` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendChunk.new(url: state.upload_url, offset: state.offset, length: event.bytes_buffered, finalize: true)` | +| **`Transmission \| Reading from stream`** | `:chunk_read_eof_empty` | `Event::ChunkRead(bytes_buffered: 0, eof: true)` | `in_flight_length = 0`
`status = :finalizing_sending_finalize` | `Finalizing \| Sending finalize` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :finalizing, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendFinalize.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Transmission \| Sending`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Transmission \| Sending`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Transmission \| Sending`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Transmission \| Sending`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Finalizing \| Sending with upload`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `offset = state.offset + state.in_flight_length`
`in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | +| **`Finalizing \| Sending with upload`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending with upload`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `in_flight_length = 0`
`status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending with upload`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `in_flight_length = 0`
`last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Finalizing \| Sending with upload`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Finalizing \| Sending with upload`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `in_flight_length = 0`
`last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Finalizing \| Sending finalize`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | +| **`Finalizing \| Sending finalize`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending finalize`** | `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind: :connection_failed \| :timeout)` | `status = :recovery` | `Recovery` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :recovering, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendQuery.new(url: state.upload_url)` | +| **`Finalizing \| Sending finalize`** | `:request_retries_exhausted` | `Event::RequestFailed(kind: :retries_exhausted)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Finalizing \| Sending finalize`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Finalizing \| Sending finalize`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Recovery`** | `:response_active` | `Event::HttpResponse(200, headers, _)` with `Status: active` | `offset = headers['X-Goog-Upload-Size-Received'].to_i`
`in_flight_length = 0`
`status = :transmission_reading` | `Transmission \| Reading from stream` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::RealignBuffer.new(server_offset: state.offset)`
`Instruction::FillBuffer.new(target_bytesize: state.chunk_size)` | +| **`Recovery`** | `:response_final` | `Event::HttpResponse(200, headers, body)` with `Status: final` | `in_flight_length = 0`
`status = :success` | `Success` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :completed, bytes_uploaded: state.offset, total_bytes: state.offset))`
`Instruction::TerminateSuccess.new(response: event)` | +| **`Recovery`** | `:response_cat2` | `Event::HttpResponse` (Category 2; see Section 6.1.2) | `status = :recovery` | `Recovery` | `Instruction::SendQuery.new(url: state.upload_url)` | +| **`Recovery`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` | `Event::RequestFailed(kind:, ...)` | `last_error = event.source_error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: event.source_error)` | +| **`Recovery`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, body)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Recovery`** | `:response_fatal_bad_response` | `Event::HttpResponse` (Fatal status; see Section 6.1.3) | `last_error = Gapic::Rest::ResumableUpload::BadResponseError.from(event)`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **`Transmission \| Reading from stream` / `Transmission \| Sending chunk` / `Finalizing \| Sending with upload` / `Finalizing \| Sending finalize` / `Recovery`** | `:user_cancel` | `Event::Cancel` | `status = :cancelling` | `Cancelling` | `Instruction::NotifyProgress.new(progress: Progress.new(phase: :cancelling, bytes_uploaded: state.offset, total_bytes: config.upload_size))`
`Instruction::SendCancel.new(url: state.upload_url)` | +| **`Cancelling`** | `:response_cancelled` | `Event::HttpResponse(200, headers, _)` with `Status: cancelled` | `status = :cancelled` | `Cancelled` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadCancelledError.from(event))` | +| **`Cancelling`** | `:response_rejected` | `Event::HttpResponse(non-200, headers, _)` with `Status: final` | `status = :rejected` | `Rejected` | `Instruction::TerminateFailure.new(error: Gapic::Rest::ResumableUpload::UploadRejectedError.from(event))` | +| **`Cancelling`** | `:request_retries_exhausted` / `:request_connection_failed` / `:request_timeout` / `:response_fatal_bad_response` | `Event::RequestFailed` or HTTP failure | `last_error = error`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **Any Non-Terminal** | `:global_deadline_exceeded` | `Event::GlobalDeadlineExceeded` | `last_error = Gapic::Rest::ResumableUpload::DeadlineExceededError.new`
`status = :error` | `Error` | `Instruction::TerminateFailure.new(error: state.last_error)` | +| **Any State** | *Unmatched* | Any event not matched above | — | — | `fail_with_unmatched_transition(state, event)`: raises `InvalidTransitionError` stating in human terms what the protocol was doing (e.g. sending a chunk of data), what happened including HTTP status and `X-Goog-Upload-Status` header, and attaches the response. | + +### 4.3 State Transition Graph + +```mermaid +stateDiagram-v2 + [*] --> Initializing + Initializing --> Starting : Event::StartUpload + Initializing --> Recovery : Event::ResumeUpload + Starting --> Transmission_Reading : Event::HttpResponse(200, active) + + state Transmission { + Transmission_Reading --> Transmission_Sending : Event::ChunkRead(eof: false) + Transmission_Sending --> Transmission_Reading : Event::HttpResponse(200, active) + } + + Transmission_Reading --> Finalizing_Sending_Upload : Event::ChunkRead(eof: true, buffered > 0) + Transmission_Reading --> Finalizing_Sending_Finalize : Event::ChunkRead(eof: true, buffered == 0) + + state Finalizing { + Finalizing_Sending_Upload --> Success : Event::HttpResponse(200, final) + Finalizing_Sending_Finalize --> Success : Event::HttpResponse(200, final) + } + + Transmission_Sending --> Recovery : Event::HttpResponse(recoverable) / Event::RequestFailed + Finalizing_Sending_Upload --> Recovery : Event::HttpResponse(recoverable) / Event::RequestFailed + Finalizing_Sending_Finalize --> Recovery : Event::HttpResponse(recoverable) / Event::RequestFailed + + Recovery --> Transmission_Reading : Event::HttpResponse(200, active, server_offset) + Recovery --> Success : Event::HttpResponse(200, final) + + Starting --> Rejected : Event::HttpResponse(non-200, final) + Transmission_Sending --> Rejected : Event::HttpResponse(non-200, final) + Finalizing_Sending_Upload --> Rejected : Event::HttpResponse(non-200, final) + Finalizing_Sending_Finalize --> Rejected : Event::HttpResponse(non-200, final) + Recovery --> Rejected : Event::HttpResponse(non-200, final) + + Starting --> Error : Event::RequestFailed / 4xx / 5xx + Recovery --> Error : Event::RequestFailed + + Success --> [*] + Rejected --> [*] + Error --> [*] +``` + +--- + +## 5. Chunk Size Adjustment Rules + +Upon receiving `200 OK` from the `start` request, `Core` inspects the response headers for `X-Goog-Upload-Chunk-Granularity`. The effective chunk size (`effective_chunk_size`) stored in `State` is resolved using the following variable definitions and rules: + +### 5.1 Variable Definitions +* `DEFAULT_CHUNK_SIZE`: Default chunk size of `8_388_608` bytes (8 MB). +* `user_chunk_size`: Explicit chunk size specified in `StartUploadConfig.chunk_size` (or `nil` if unspecified). +* `chunk_granularity`: Required byte alignment modulus parsed from header `X-Goog-Upload-Chunk-Granularity` as an Integer (or `nil` if header is absent). +* `effective_chunk_size`: Final calculated byte size used by Driver for in-memory buffering and chunk transmission. + +### 5.2 Resolution Rules + +#### Rule 1: No Server Granularity Specified (`chunk_granularity` is nil or 0) +When the server does not specify a granularity requirement: +* If `user_chunk_size` is provided: `effective_chunk_size = user_chunk_size`. +* If `user_chunk_size` is omitted: `effective_chunk_size = DEFAULT_CHUNK_SIZE`. + +#### Rule 2: Default Chunk Size with Server Granularity (`user_chunk_size` is nil, `chunk_granularity > 0`) +When the user does not specify a chunk size, the default 8 MB chunk size is aligned down to the nearest multiple of `chunk_granularity`: +* `effective_chunk_size = DEFAULT_CHUNK_SIZE - (DEFAULT_CHUNK_SIZE % chunk_granularity)`. +* If `DEFAULT_CHUNK_SIZE < chunk_granularity`, `effective_chunk_size` is promoted to `chunk_granularity`. + +#### Rule 3: User Specified Chunk Size with Server Granularity (`user_chunk_size > 0`, `chunk_granularity > 0`) +When an explicit `user_chunk_size` is supplied alongside a server `chunk_granularity`: +* **Case 3A (Standard Alignment: `user_chunk_size >= chunk_granularity`)**: + * The user chunk size is aligned down to the nearest integer multiple of `chunk_granularity`: + * `effective_chunk_size = user_chunk_size - (user_chunk_size % chunk_granularity)`. + * If `user_chunk_size` is already a multiple of `chunk_granularity` (`user_chunk_size % chunk_granularity == 0`), `effective_chunk_size = user_chunk_size`. +* **Case 3B (User Size Below Granularity: `user_chunk_size < chunk_granularity`)**: + * If `user_chunk_size` is strictly less than `chunk_granularity`, downward alignment would produce `0` bytes (an invalid chunk size). + * To satisfy the server's mandatory granularity constraint, `effective_chunk_size` is promoted to `chunk_granularity`. + +### 5.3 Reference Implementation +```ruby +def self.resolve_chunk_size(user_chunk_size, chunk_granularity) + base_size = user_chunk_size || DEFAULT_CHUNK_SIZE + return base_size if chunk_granularity.nil? || chunk_granularity <= 0 + return chunk_granularity if base_size <= chunk_granularity + + base_size - (base_size % chunk_granularity) +end +``` + +--- + +## 6. Error Classification & Recovery Flows + +### 6.1 Error Categories +The implementation distinguishes three categories of network and protocol-level failures: + +#### 6.1.1 Category 1: Transient Transport Failures +* **Definition**: Standard TCP, network connection timeout, DNS, or server load-shedding errors that do not compromise the protocol session. +* **Examples**: `503 Service Unavailable`, `408 Request Timeout`, `429 Too Many Requests`, `Faraday::ConnectionFailed`, `Faraday::TimeoutError`. +* **Resolution**: The `Driver` intercepts these errors inside the physical execution wrapper and delegates directly to `Gapic::Common::RetryPolicy`. If retries succeed, `Core` receives `Event::HttpResponse`. If retries exhaust attempt/timeout limits, Driver emits `Event::RequestFailed(kind: :retries_exhausted, ...)`. + +#### 6.1.2 Category 2: Recoverable Protocol Failures +* **Definition**: Responses indicating that the client's current offset is misaligned with the server, protocol headers are missing/stripped on completed requests, or unretried transport connection failures during data transmission. +* **Conditions Producing `:response_cat2`**: + 1. **Non-200 Active Responses**: Any response with `X-Goog-Upload-Status: active` where HTTP status is non-200. + 2. **Missing or Empty `X-Goog-Upload-Status` Header**: Any response lacking `X-Goog-Upload-Status` (or empty) whose HTTP status is **not** in `FATAL_STATUS_CODES` (Section 6.1.3). This includes HTTP 200, 5xx server/gateway errors (`500`, `502`, `503`, `504`), and recoverable client errors (`400`, `408`, `409`, `412`, `416`, `429`, `499`). + 3. **Unretried Data Plane Connection Drops or Request Timeouts**: `Event::RequestFailed(kind: :connection_failed)` or `Event::RequestFailed(kind: :timeout)` (`:request_connection_failed`, `:request_timeout`) occurring during `Transmission` or `Finalizing`. +* **Missing Header Handling & Retry Policy Contract**: + * *Why Headers Go Missing*: Intermediate proxies, reverse-proxies, or Google Front End (GFE) edge proxies can strip the protocol response headers or return raw HTML/text error pages on failure. + * *Session Initiation (`start`)*: Missing `X-Goog-Upload-Status` is treated as **retriable** by `start_retry_policy` (retry predicate returns `true`) across **any response code, including 200 OK**. Driver retries transparently to smooth over transient gateway noise. If retries exhaust, `Starting` transitions to `:error` via `fail_with_request_error` or `fail_with_bad_response` (cannot recover a session before an upload URL is obtained). + * *Session Control (`query`, `cancel`)*: `control_plane_retry_policy` does **not** treat missing status headers as retriable, returning the completed `Event::HttpResponse` immediately to `Core` so it can manage protocol recovery or fail fast. + * *Data Plane (`upload`, `upload, finalize`, standalone `finalize`)*: Missing `X-Goog-Upload-Status` is treated as **unretriable** by `data_plane_retry_policy` (retry predicate returns `false`). The Driver immediately returns `Event::HttpResponse` to `Core` so it classifies as `:response_cat2` and initiates Category 2 `Recovery` via `Instruction::SendQuery` rather than blindly re-transmitting data. +* **Resolution**: Core transitions to `Recovery` and emits `Instruction::SendQuery.new(url: state.upload_url)` to obtain `server_offset`. + +#### 6.1.3 Category 3: Terminal Failures & Fatal Status Codes +* **Definition**: Irrecoverable errors where either the request is structurally invalid, unauthorized, transport retry limits are exhausted, unseekable rewind is needed, or the server has explicitly aborted/rejected the session. +* **Canonical Fatal Status Codes (`FATAL_STATUS_CODES`)**: + The following status codes indicate structural or authentication failures that cannot be resolved by querying the upload backend: + * `401 Unauthorized`: Authentication token is expired, invalid, or missing. + * `403 Forbidden`: Caller lacks required IAM permissions for the upload destination. + * `404 Not Found`: Session upload URL does not exist or has expired. + * `405 Method Not Allowed`: HTTP method is rejected by the server. + * `410 Gone`: Upload session has been permanently removed. + * `413 Payload Too Large`: Upload chunk or overall size exceeds server limit. + * `415 Unsupported Media Type`: Object content type is rejected. + Responses with these status codes are classified as `:response_fatal_bad_response` even if the `X-Goog-Upload-Status` header is absent. +* **Other Terminal Conditions**: + * **Retry Exhaustion**: Any `Event::RequestFailed(kind: :retries_exhausted)` occurring at any stage. When Category 1 transport retries are exhausted by the `RetryPolicy`, failure is immediate and terminal; it does not enter Category 2 `Recovery`. + * **Session Rejection**: Any response with `X-Goog-Upload-Status: final` and non-2xx status code (`:response_rejected` -> raises `Gapic::Rest::ResumableUpload::UploadRejectedError`). + * **Initiation Failure**: Any 4xx/5xx or `Event::RequestFailed` during `Starting` (`:error` -> raises `Gapic::Rest::ResumableUpload::BadResponseError` or source error). + * **Session Cancellation**: Cancelled upload sessions raise `Gapic::Rest::ResumableUpload::UploadCancelledError`. + * **Global Deadline Expiration**: Monotonic clock exceeding session deadline raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. + * **Unseekable Stream Rewind**: Server offset rolled back behind retained buffer (`server_offset < buffer_start_offset`) on an unseekable stream (raises `Gapic::Rest::ResumableUpload::UnseekableStreamError`). +* **Resolution**: Core transitions to `:rejected` or `:error` and emits `Instruction::TerminateFailure`. + +#### 6.1.4 Actionable Terminal Errors & Metadata Propagation +Terminal errors provide actionable context so downstream SDK callers can inspect error metadata: +* **Error Classes**: + * `BadResponseError < Gapic::Rest::Error`: Unrecoverable non-2xx HTTP responses or invalid payloads. Retains `attr_reader :response_body` returning `event.body`, and includes `HasResumeHandle`. + * `UploadRejectedError < Gapic::Rest::Error`: Backend explicitly rejected the session with `X-Goog-Upload-Status: final`. Retains `attr_reader :response_body` returning `event.body`. Does NOT include `HasResumeHandle` (session is terminated permanently). + * `UploadCancelledError < Gapic::Common::Error`: Upload session cancelled by caller. Does NOT include `HasResumeHandle` (session is terminated permanently). + * `DeadlineExceededError < Gapic::Common::Error`: Upload deadline exceeded with optional root cause (`attr_reader :root_cause`), and includes `HasResumeHandle`. + * `UnseekableStreamError < Gapic::Common::Error`: Stream rewind required on an unseekable stream; includes `HasResumeHandle`. + * `InvalidTransitionError < Gapic::Common::Error`: Unexpected event dispatched for state; includes `HasResumeHandle`. + * `StreamMismatchError < Gapic::Common::Error`: Stream content or length does not match resumed upload specifications; includes `HasResumeHandle`. + * `RequestFailedError < Gapic::Common::Error`: Terminal HTTP request failure (e.g. transport connection failure, request timeout, or retries exhausted). Retains `attr_reader :cause` returning the underlying error, preserves REST error attributes (`status_code`, `status`, `details`, `headers`) when available, and includes `HasResumeHandle`. + * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates Session lifecycle rules (e.g. attempting to start an already-bound session, resuming an unbound session without a target upload, re-binding to a different upload, resuming a finalized/dead session, or concurrent run invocations). Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. +* **Resume Handle Propagation (`HasResumeHandle`)**: + * The `HasResumeHandle` mixin exposes `attr_reader :resume_handle` returning a `ResumeHandle` (or `nil` if session initiation was incomplete or if the session was `:rejected` or `:cancelled`). + * Whenever `resume_handle` is non-nil, the uniform suffix `" (upload session is resumable: see #resume_handle)"` is automatically appended to the error message. +* **Metadata Sourcing & De-prefixing**: + * When `event.error` is present (from `Gapic::Rest::Error.wrap_faraday_error`), factories source `status_code`, `status`, `details`/`status_details`, and `headers`/`header`. + * The prefix literal `Gapic::Rest::Error::REST_ERROR_PREFIX` (`"An error has occurred when making a REST request"`) is stripped from `event.error.message` to avoid redundant prefixes. + * The resulting actionable message follows the format: + * For `UploadRejectedError`: `"Upload rejected by server with HTTP #{status_code} #{status_name}: #{inner_message}"` (e.g., `"Upload rejected by server with HTTP 403 PERMISSION_DENIED: The caller does not have permission"`). + * For `BadResponseError`: `"Resumable upload failed with HTTP #{status_code} #{status_name}: #{inner_message}"` (e.g., `"Resumable upload failed with HTTP 429 RESOURCE_EXHAUSTED: Quota limit reached"`). +* **Fallback Formatting**: + * When `event.error` is absent, factories fall back to `event.status` and `event.headers`, naming the status and including the detailed `X-Goog-Upload-Status` header: + * For `UploadRejectedError`: `"Upload rejected by server with HTTP #{event.status} #{status_name} (X-Goog-Upload-Status: 'final')"`. + * For `BadResponseError`: `"Resumable upload failed with HTTP #{event.status} #{status_name} (X-Goog-Upload-Status: #{upload_status_desc})"`. + * `response_body` on `BadResponseError` and `UploadRejectedError` returns `event.body`. + +### 6.2 Recovery and Buffer Alignment +When `Core` resolves a `query` response in the `Recovery` state, it updates `State.offset` (`protocol_state_offset`) to `server_offset` (extracted from `X-Goog-Upload-Size-Received`) and transitions to `Transmission | Reading from stream`. + +To realign the upload state, the `Driver` processes `Instruction::RealignBuffer(server_offset)` using its in-memory buffer and stream position tracking: +1. **Within-Buffer Alignment (`buffer_start_offset <= server_offset <= buffer_end_offset`)**: + * The Driver trims already-persisted bytes: `@buffer = @buffer.byteslice((server_offset - buffer_start_offset)..-1)`. + * The Driver updates `buffer_start_offset = server_offset`. + * Upon executing the accompanying `Instruction::FillBuffer(target_bytesize)`, the Driver reads `target_bytesize - @buffer.bytesize` bytes from `stream` to restore `@buffer` to full `chunk_size` before transmitting. +2. **Rewind Required (`server_offset < buffer_start_offset`)**: + * If `stream.respond_to?(:seek)`: the Driver seeks to `server_offset`, clears `@buffer = "".b`, and sets `buffer_start_offset = server_offset`. + * If `stream` is unseekable (e.g. Socket, Pipe, STDIN): the Driver raises terminal `UnseekableStreamError` (Category 3), attaching `resume_handle`. +3. **Fast-Forward Required (`server_offset > buffer_end_offset`)**: + * If total `upload_size` is known and `server_offset > upload_size`: Driver raises terminal `StreamMismatchError` with `resume_handle`. + * If `upload_size` is `nil` and `stream.respond_to?(:size)` and `server_offset > stream.size`: Driver raises terminal `StreamMismatchError` with `resume_handle`. + * The Driver clears `@buffer = "".b`. + * If `stream.respond_to?(:seek)`: seeks to `server_offset`. + * If unseekable: reads and discards `server_offset - current_stream_pos` bytes from `stream`. If the stream encounters unexpected EOF before reaching `server_offset`, Driver raises terminal `StreamMismatchError` with `resume_handle`. + * The Driver sets `buffer_start_offset = server_offset`. + +### 6.3 Sensible Defaults for Global Deadline +Every upload session executed via `Driver#run` must have a finite, guaranteed upper bound on total wall-clock execution time. Without a mandatory global deadline, a session encountering repeated Category 2 protocol recoveries or intermittent network stalls could hang indefinitely. + +To guarantee termination, `Driver#run` establishes an absolute monotonic deadline at the very start of execution: +```ruby +@deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + resolve_timeout +``` + +#### Timeout Resolution Algorithm (`resolve_timeout`) +The total session timeout is resolved in priority order: +1. **Explicit User Timeout (`config.timeout`)**: If `config.timeout` is present and strictly positive (`config.timeout&.positive?`), that value is used directly. Zero or negative values are treated as unset (`nil`). +2. **Size-Proportional Timeout (`config.upload_size`)**: If total `upload_size` is known upfront, the timeout is computed assuming a minimum sustained upload throughput of `MIN_ASSUMED_THROUGHPUT = 1_048_576` bytes/sec (1 MB/s), floored by `BASE_TIMEOUT = 3_600` seconds (1 hour): + ```ruby + [config.upload_size.fdiv(MIN_ASSUMED_THROUGHPUT), BASE_TIMEOUT].max + ``` + *Rationale*: Using `BASE_TIMEOUT` as a floor prevents sub-millisecond timeouts for small payloads while scaling linearly for multi-gigabyte uploads. +3. **Default Base Timeout (`BASE_TIMEOUT`)**: If neither a positive timeout nor `upload_size` is provided (e.g., streaming uploads of unknown length), the timeout defaults to `BASE_TIMEOUT` (`3_600` seconds). + +#### Bounding Transport Retries by Global Deadline +Transport retries and individual HTTP exchanges must never exceed the remaining global deadline. When `Driver#make_post_request` invokes `ClientStub#make_post_request`, it computes the per-request timeout from the remaining session budget (`max(deadline - monotonic_now, 0)`), additionally capped by `retry_policy.timeout`: +```ruby +remaining = [@deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max +timeout = retry_policy&.timeout ? [remaining, retry_policy.timeout].min : remaining +``` +This timeout is passed in `options[:timeout]`, ensuring that underlying Faraday requests and `Gapic::Common::RetryPolicy` evaluations always respect the remaining upload budget. + +--- + +## 7. Observability Standards + +### 7.1 Architecture & Separation of Concerns +Because `Rules` is a pure decision engine and `Core` is a side-effect-free state container, protocol decisions are encoded as immutable `Decision` data structures and logged exclusively by the `Driver` via `Driver::UploadLog`. + +Each invocation of `Driver#run` generates a fresh UUIDv4 session identifier (`uploadId`) that is attached to every log entry emitted during that run. Structured log entries are constructed using `Gapic::LoggingConcerns` (`StubLogger` yielding a `LogEntryBuilder` producing `Google::Logging::Message` instances). Machine-readable state and telemetry are stored in `Google::Logging::Message#fields`, allowing log message text to evolve independently without breaking structured queries. + +### 7.2 Log Level & Recipe Mapping +The `Driver` emits structured logs across three severity levels (`INFO`, `DEBUG`, `WARN`). High-frequency per-chunk acknowledgements (`:ack_chunk`) are suppressed from `INFO` lifecycle logs to avoid log volume bloat on multi-gigabyte uploads. + +| Severity | Category | Trigger / Recipe | Message Summary | +| :--- | :--- | :--- | :--- | +| `INFO` | Lifecycle | `:start_session` | Initiating resumable upload | +| `INFO` | Lifecycle | `:begin_transmission` | Upload session established | +| `INFO` | Lifecycle | `:send_upload_finalize` | Sending final upload chunk | +| `INFO` | Lifecycle | `:send_finalize` | Sending finalize command | +| `INFO` | Lifecycle | `:enter_recovery` | Entering upload recovery | +| `INFO` | Lifecycle | `:retry_recovery` | Retrying upload recovery query | +| `INFO` | Lifecycle | `:realign_from_recovery` | Resuming upload from server offset | +| `INFO` | Lifecycle | `:complete_upload_with_data`, `:complete_upload_finalized` | Resumable upload completed | +| `INFO` | Lifecycle | `:cancel_session` | Canceling resumable upload | +| `INFO` | Lifecycle | `:complete_cancellation` | Resumable upload canceled | +| `DEBUG` | Lifecycle | `:send_chunk` | Sending upload chunk | +| `DEBUG` | Decision | Every `Core#dispatch` transition | `Rules: + -> -> ` | +| `DEBUG` | Wire | Outbound HTTP request (`wire_send`) | `Sending request` | +| `DEBUG` | Wire | Inbound HTTP response (`wire_receive`) | `Received HTTP ` | +| `DEBUG` | Wire | Transport exception (`wire_failure`) | `Request failed: ` | +| `DEBUG` | Buffer | Stream/buffer realignment (`buffer_realign`) | `Buffer realignment: ` | +| `WARN` | Lifecycle | `:fail_with_deadline_exceeded`, `:fail_with_rejected`, `:fail_with_bad_response`, `:fail_with_request_error` | Resumable upload failed | +| `WARN` | Transition | `InvalidTransitionError` (`unmatched_transition`) | Unmatched transition | +| `WARN` | Buffer | Backward server offset rewind on unseekable stream | Server offset rewind on unseekable stream | + +### 7.3 Structured Field Glossary +All log entries emitted by `UploadLog` populate structured fields in `Google::Logging::Message#fields`: + +* **Common Context Fields** (present on all entries): + * `system`: `"gapic-common"` + * `serviceName`: `"ResumableUpload"` + * `clientId`: Object ID of the underlying `Gapic::Rest::ClientStub`. + * `uploadId`: Unique UUIDv4 identifying the specific `Driver#run` execution. +* **Decision & Lifecycle Fields**: + * `fromStatus`: Protocol status symbol prior to event dispatch. + * `toStatus`: Resulting protocol status symbol (`decision.next_state.status`). + * `shape`: Canonical event shape symbol classified by `Rules.shape_of`. + * `recipe`: Transition recipe method symbol executed by `Rules`. + * `offset`: Current server-confirmed byte offset (`Integer`). + * `inFlightLength`: Byte length of the chunk currently in flight (`Integer`). + * `instructions`: Array of abridged instruction hashes emitted by the transition. + * `uploadSize`: Total expected upload size in bytes from `config.upload_size` (on `:start_session`). + * `requestedChunkSize`: Configured chunk size in bytes from `config.chunk_size` (on `:start_session`). + * `effectiveChunkSize`: Negotiated chunk size aligned to server granularity (on `:begin_transmission`). + * `granularity`: Server chunk alignment modulus from `X-Goog-Upload-Chunk-Granularity` (on `:begin_transmission`). + * `uploadUrl`: Abridged session upload URL (on `:begin_transmission` and `:cancel_session`). + * `status`: Current protocol status symbol (on `unmatched_transition`). + * `error`: Exception message string (on `fail_with_*` and `unmatched_transition`). + * `responseBody`: Abridged error response body from `last_error.response_body` when present (on `fail_with_*`). +* **Wire & Transport Fields**: + * `method`: Always the string `"POST"`. + * `url`: Abridged request target URI. + * `headers`: Redacted HTTP header hash. + * `startAttempt`: Retry attempt counter (`Integer`). + * `command`: Value of `X-Goog-Upload-Command` request header. + * `offset`: Parsed integer value of `X-Goog-Upload-Offset` request header (`wire_send`). + * `bodySize`: Total byte length of request payload (`Integer`). + * `body`: Abridged payload or error body snippet. + * `status`: HTTP response status code (`Integer`, on `wire_receive`). + * `uploadStatus`: Value of `X-Goog-Upload-Status` response header. + * `sizeReceived`: Parsed integer value of `X-Goog-Upload-Size-Received` response header. + * `granularity`: Parsed integer value of `X-Goog-Upload-Chunk-Granularity` response header (`wire_receive`). + * `kind`: Transport failure classification symbol (`:timeout`, `:connection_failed`, `:retries_exhausted`). + * `error`: Exception message string (`wire_failure`). +* **Buffer Realignment Fields**: + * `action`: Realignment strategy string (`"within_buffer"`, `"rewind"`, or `"fast_forward"`). + * `serverOffset`: Target byte offset reported by the server (`Integer`). + * `currentOffset`: Local buffer start offset before realignment (`Integer`). + +### 7.4 Redaction & Payload Abridgement +To prevent credential leakage and ensure log volume is proportional to the number of requests and independent of payload size, `Driver::Abridge` and `ClientStub` enforce strict sanitization rules before any entry is passed to the logger: + +1. **URL Query Elision (`Abridge.url`)**: Upload session URLs contain capability tokens in their query parameters (e.g., `upload_id`, `sid`). `Abridge.url` parses the URI and replaces every query parameter value with `<...>` (e.g., `https://storage.googleapis.com/upload?upload_id=<...>`). +2. **Header Allowlisting (`Abridge.headers`)**: Only protocol control headers prefixed with `x-goog-upload-` retain their values in log entries (with `x-goog-upload-url` passed through `Abridge.url`). All other request and response headers—including `Authorization` or custom metadata—are replaced with `"<...>"`. Note that Faraday injects `Authorization` headers below the `ClientStub` logging layer; tests verify that bearer tokens never appear in logs. +3. **Binary Payload Abridgement (`Abridge.bytes` & `ClientStub#abridge_request_body`)**: + * In `Driver::Abridge.bytes`, binary payloads of 64 bytes or more are abridged to their first 32 bytes encoded in hexadecimal followed by the total byte size: `"... "`. + * In `Gapic::Rest::ClientStub#log_request`, any request body exceeding 1 KiB (1024 bytes) or containing non-UTF-8 binary data is abridged to `">"`, preventing 8 MiB upload chunks from being dumped into `DEBUG` logs. +4. **Error Body Truncation (`Abridge.error_body`)**: HTTP error response bodies (status $\ge 400$) are forced to UTF-8 encoding with invalid byte sequences scrubbed and truncated to at most 512 characters. + +### 7.5 Enabling & Configuring Logging +Logging is disabled by default (`logger: nil`) and incurs negligible allocation overhead when inactive. Users and test harnesses can enable logging via two mechanisms: + +1. **Environment Variable Opt-In (`GOOGLE_SDK_RUBY_LOGGING_GEMS`)**: + Setting the `GOOGLE_SDK_RUBY_LOGGING_GEMS` environment variable activates default `Logger` instances writing to `$stderr` at `DEBUG` level (using `Google::Logging::StructuredFormatter` when running in a Google Cloud environment): + * `GOOGLE_SDK_RUBY_LOGGING_GEMS=all` or `GOOGLE_SDK_RUBY_LOGGING_GEMS=true`: Enables logging across all Google Cloud Ruby SDK components. + * `GOOGLE_SDK_RUBY_LOGGING_GEMS=gapic-common`: Enables logging specifically for `gapic-common` (including `ResumableUpload::Driver` and `ClientStub`). + * `GOOGLE_SDK_RUBY_LOGGING_GEMS=false` or `none`: Explicitly disables SDK logging even if a default logger is configured. +2. **Explicit Logger Injection**: + Pass any Ruby `::Logger`-compatible instance directly to `Driver.new(client_stub: stub, config: config, logger: my_logger)` or configure it on the parent service client config. \ No newline at end of file diff --git a/gapic-common/design/resumable_upload/integration-test-plan.md b/gapic-common/design/resumable_upload/integration-test-plan.md new file mode 100644 index 0000000..46a5750 --- /dev/null +++ b/gapic-common/design/resumable_upload/integration-test-plan.md @@ -0,0 +1,286 @@ +# Resumable Upload Integration Test Plan + +This document outlines the integration test architecture and test suites for the Resumable Upload protocol implementation in `gapic-common`. Unlike the unit test suite under `test/gapic/rest/resumable_upload/`, which isolates protocol state transitions and driver components against test doubles, the integration test suite exercises the full stack end-to-end over real HTTP/REST connections against a live `gapic-showcase` server. + +--- + +## 1. Integration Test Architecture Overview + +```mermaid +flowchart TD + subgraph Runner["Test Runner (.toys/test-integration.rb)"] + Toys["toys test-integration"] + Lifecycle["Showcase Lifecycle Manager
(Verify version >= 0.43, allocate ports, spawn & health-check)"] + end + + subgraph Harness["Test Harness (integration/integration_helper.rb)"] + BaseClass["ShowcaseIntegrationTest
(Minitest::Test)"] + PayloadGen["payload(size)
Deterministic binary stream"] + TraceLog["StringIO Logger
(Emits debug wire traces on test failure)"] + end + + subgraph SUT["System Under Test"] + Driver["Gapic::Rest::ResumableUpload::Driver"] + Stub["Gapic::Rest::ClientStub
(raise_faraday_errors: false)"] + end + + subgraph Server["External Server"] + Showcase["gapic-showcase
(/resumable/upload/v1beta1/files:upload)"] + end + + Toys --> Lifecycle + Lifecycle --> Showcase + Toys --> BaseClass + BaseClass --> PayloadGen + BaseClass --> TraceLog + BaseClass --> Driver + Driver --> Stub + Stub <-->|"HTTP POST / PUT (REST)"| Showcase +``` + +### 1.1 Execution & Server Lifecycle (`.toys/test-integration.rb`) +* **External Endpoint Override**: If `SHOWCASE_ENDPOINT` is set in the environment, the runner connects directly to that address without spawning a local server. +* **Automatic Server Provisioning**: When `SHOWCASE_ENDPOINT` is unset, the runner locates the `gapic-showcase` binary on `PATH` (or via `SHOWCASE_BIN`), verifies that its version is at least `0.43`, allocates ephemeral ports, and spawns `gapic-showcase run`. +* **Health Check & Teardown**: Polls the TCP socket with a 10-second deadline before invoking Minitest, and guarantees process cleanup (`SIGTERM` + `waitpid`) in an `ensure` block. + +### 1.2 Test Harness (`integration/integration_helper.rb`) +* **`ShowcaseIntegrationTest`**: Base class providing helper methods for test configuration: + * `showcase_client_stub`: Instantiates a real `Gapic::Rest::ClientStub` targeting `SHOWCASE_ENDPOINT` with `raise_faraday_errors: false` and an attached `DEBUG` logger. + * `build_config(scenario: nil, scenario_config: {}, **overrides)`: Creates a `StartUploadConfig` targeting `/resumable/upload/v1beta1/files:upload`. When `scenario` is provided, injects `X-Goog-Test-Scenario` and `X-Goog-Test-Scenario-Config` (with a generated `client_uuid` merged with `scenario_config`) into `initial_headers` (these test headers are unaffected by `RESERVED_INITIAL_HEADERS` since they are not in the five reserved protocol headers). Configures fast retry policies (`FAST_RETRY = { initial_delay: 0.01, max_delay: 0.05, multiplier: 1, timeout: 2 }`), a default 10-second session timeout, a default payload of `786_432` bytes (`3 * 262_144`), default chunk size of `262_144` bytes, and an `on_progress` callback appending each `Progress` struct to `@progress_records`. + * `build_session(scenario: nil, scenario_config: {}, **overrides)` & `start_session(session, **overrides)`: Partitions overrides using `START_ONLY_KEYS` (`[:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy]`). `build_session` instantiates a `Gapic::Rest::ResumableUpload::Session` with the common members (`client_stub`, `stream`, `upload_size`, `content_type`, `timeout`, `control_plane_retry_policy`, `data_plane_retry_policy`, `on_progress`, `logger`) and stores initiation arguments in `@start_args`, while `start_session` invokes `session.start(**@start_args, **overrides)`. + * `phases` & `offsets`: Convenience accessors returning `@progress_records.map(&:phase)` and `@progress_records.map(&:bytes_uploaded)`. + * `payload(size)`: Generates deterministic binary strings of arbitrary byte length for stream uploads. + * `UnseekableStream`: Stream wrapper around `StringIO` that exposes `#read` and `#pos` while omitting `#seek` (`respond_to?(:seek)` is `false`). + * **Diagnostic Trace Capture**: Buffers `DEBUG`-level driver logs in memory during each test run and dumps the full trace to `stderr` only if a test fails (or when `SHOWCASE_LOG` is set). + +--- + +## 2. Detailed Test Suites & Cases + +### 2.1 Golden Path Suite (`integration/resumable_upload/golden_path_test.rb`) + +Tests standard, uninterrupted resumable upload workflows against `gapic-showcase`. + +#### Case 1. Multi-chunk upload with known size (`test_multi_chunk_known_size`) +* **Scenario**: Uploads a 1.5 MB (`1_500_000` bytes) stream with an explicit `upload_size: 1_500_000` and `chunk_size: 524_288` (512 KiB). +* **Protocol Flow**: + 1. `start` command initiates the session with `X-Goog-Upload-Header-Content-Length: 1500000`. + 2. Chunk 1 transmits bytes `0..524287` (`upload`). + 3. Chunk 2 transmits bytes `524288..1048575` (`upload`). + 4. Chunk 3 transmits remaining bytes `1048576..1499999` with `upload, finalize`. +* **Assertions**: + * Returned JSON body parses cleanly and reports `"size" == 1_500_000`. + * `progress_records` contains 6 `Progress` notifications across lifecycle phases: + * `Progress(phase: :initiating, bytes_uploaded: 0, total_bytes: 1_500_000)` + * `Progress(phase: :uploading, bytes_uploaded: 0, total_bytes: 1_500_000)` + * `Progress(phase: :uploading, bytes_uploaded: 524_288, total_bytes: 1_500_000)` + * `Progress(phase: :uploading, bytes_uploaded: 1_048_576, total_bytes: 1_500_000)` + * `Progress(phase: :finalizing, bytes_uploaded: 1_048_576, total_bytes: 1_500_000)` + * `Progress(phase: :completed, bytes_uploaded: 1_500_000, total_bytes: 1_500_000)` + +#### Case 2. Default chunk size on small upload (`test_small_upload_default_chunk_size`) +* **Scenario**: Uploads a ~100 KB (`100_000` bytes) payload with `upload_size: 100_000` and no `chunk_size` specified. +* **Protocol Flow**: + 1. `start` command initiates the session; chunk size defaults to 8 MiB (`8_388_608` bytes). + 2. The entire 100,000-byte payload fits within a single buffer read and is transmitted in one `upload, finalize` request. +* **Assertions**: + * Returned JSON body reports `"size" == 100_000`. + * `progress_records` contains 4 `Progress` notifications: + * `Progress(phase: :initiating, bytes_uploaded: 0, total_bytes: 100_000)` + * `Progress(phase: :uploading, bytes_uploaded: 0, total_bytes: 100_000)` + * `Progress(phase: :finalizing, bytes_uploaded: 0, total_bytes: 100_000)` + * `Progress(phase: :completed, bytes_uploaded: 100_000, total_bytes: 100_000)` + +#### Case 3. Standalone finalize on unseekable stream (`test_standalone_finalize_unseekable_stream`) +* **Scenario**: Uploads a `786_432`-byte payload (`3 * 262_144` bytes) wrapped in an `UnseekableStream`, with `chunk_size: 262_144` (256 KiB) and `upload_size` omitted (`nil`). +* **Coverage**: + * Unknown total upload size on `start` (omitted `X-Goog-Upload-Header-Content-Length`). + * End-of-exact-boundary stream reading path (payload is an exact multiple of `chunk_size`, so EOF is not detected until the subsequent buffer fill). + * Standalone `SendFinalize` instruction (`upload_command: "finalize"` with empty body). +* **Protocol Flow**: + 1. `start` command initiates the session without a total content length header. + 2. Chunk 1 transmits bytes `0..262143` (`upload`). + 3. Chunk 2 transmits bytes `262144..524287` (`upload`). + 4. Chunk 3 transmits bytes `524288..786431` (`upload`). + 5. Next buffer read returns 0 bytes at EOF (`:chunk_read_eof_empty`), emitting `SendFinalize` to send a standalone `finalize` request at offset `786432`. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `progress_records` contains 7 `Progress` notifications: + * `Progress(phase: :initiating, bytes_uploaded: 0, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 0, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 262_144, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 524_288, total_bytes: nil)` + * `Progress(phase: :uploading, bytes_uploaded: 786_432, total_bytes: nil)` + * `Progress(phase: :finalizing, bytes_uploaded: 786_432, total_bytes: nil)` + * `Progress(phase: :completed, bytes_uploaded: 786_432, total_bytes: 786_432)` + +### 2.2 Chunk Granularity Suite (`integration/resumable_upload/chunk_granularity_test.rb`) + +Tests dynamic chunk size resolution when the server mandates a byte alignment modulus via `X-Goog-Upload-Chunk-Granularity`. + +#### Case 1. Downward alignment to server granularity (`test_chunk_granularity_alignment`) +* **Scenario**: Uploads a `1_000_000`-byte payload with `scenario: "chunk_granularity"`, an explicit unaligned user `chunk_size: 300_000`, and `timeout: 5`. +* **Protocol Flow**: + 1. `start` command initiates the session; Showcase returns `X-Goog-Upload-Chunk-Granularity: 256`. + 2. Client resolves the effective chunk size down to the nearest multiple of 256: `300_000 - (300_000 % 256) = 299_776` bytes. + 3. Chunks 1, 2, and 3 transmit `299_776` bytes each (`upload`), advancing confirmed offsets to `299_776`, `599_552`, and `899_328`. + 4. Final chunk transmits the remaining `100_672` bytes (`899_328..999_999`) with `upload, finalize`. +* **Assertions**: + * Returned JSON body reports `"size" == 1_000_000`. + * `offsets` equals `[0, 0, 299_776, 599_552, 899_328, 899_328, 1_000_000]`. + * `phases` equals `[:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed]`. + +### 2.3 Error Recovery Suite (`integration/resumable_upload/error_recovery_test.rb`) + +Tests Category 1 transient transport retries and Category 2 protocol recovery workflows against `scenario: "non_fatal_error_on_chunk_upload"`. + +#### Case 1. Category 1 transient error retried transparently (`test_cat1_error_retried_transparently`) +* **Scenario**: Injects a single `503 Service Unavailable` response at offset `0` (`error_code: 503, failure_count: 1, after_offset: 0`). +* **Protocol Flow**: + 1. `start` establishes the session (`200 active`). + 2. First attempt to upload chunk 1 (`0..262143`) receives `503`. + 3. `Driver` intercepts the transient error via `data_plane_retry_policy` (`FAST_RETRY`) and retries the chunk transparently without entering protocol `Recovery`. + 4. Subsequent chunks and standalone `finalize` succeed normally. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `phases` does not include `:recovering` (`[:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed]`). + * `offsets` equals `[0, 0, 262_144, 524_288, 786_432, 786_432, 786_432]`. + +#### Case 2. Simple Category 2 error recovery at offset 0 (`test_simple_cat2_error_recovery`) +* **Scenario**: Injects a single `409 Conflict` with `X-Goog-Upload-Status: active` at offset `0` (`error_code: 409, failure_count: 1, after_offset: 0`). +* **Protocol Flow**: + 1. First upload chunk receives `409` (`:response_cat2`). + 2. `Core` transitions to `Recovery` (`:recovering`) and issues `SendQuery`. + 3. Server responds with `X-Goog-Upload-Size-Received: 0`; client realigns buffer to offset `0`, retransmits chunk 1, and completes the upload. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `phases` equals `[:initiating, :uploading, :recovering, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed]`. + * `offsets` equals `[0, 0, 0, 0, 262_144, 524_288, 786_432, 786_432, 786_432]`. + +#### Case 3. Two consecutive Category 2 recoveries on chunk 2 (`test_two_consecutive_cat2_recoveries_on_chunk_2`) +* **Scenario**: Injects two consecutive `409` errors at offset `262_144` (`error_code: 409, failure_count: 2, after_offset: 262_144`). +* **Protocol Flow**: + 1. Chunk 1 (`0..262143`) succeeds. + 2. First attempt at chunk 2 (`262144..524287`) fails with `409` -> `:recovering` -> `query` (offset `262_144`) -> `:uploading`. + 3. Second attempt at chunk 2 fails with `409` -> `:recovering` -> `query` (offset `262_144`) -> `:uploading`. + 4. Third attempt at chunk 2 succeeds; chunk 3 and `finalize` complete normally. +* **Assertions**: + * Returned JSON body reports `"size" == 786_432`. + * `phases` equals `[:initiating, :uploading, :uploading, :recovering, :uploading, :recovering, :uploading, :uploading, :uploading, :finalizing, :completed]`. + * `offsets` equals `[0, 0, 262_144, 262_144, 262_144, 262_144, 262_144, 524_288, 786_432, 786_432, 786_432]`. + +#### Case 4. Category 2 failure on finalizing chunk (`test_cat2_failure_on_finalizing_chunk`) +* **Scenario**: Uploads a `786_332`-byte payload (`3 * 262_144 - 100`) where chunk 3 (`524288..786331`) carries `upload, finalize`, injecting a `409` error at offset `524_288` (`error_code: 409, failure_count: 1, after_offset: 524_288`). +* **Protocol Flow**: + 1. Chunks 1 and 2 succeed, advancing offset to `524_288`. + 2. Client enters `:finalizing` and transmits chunk 3 with `upload, finalize`. + 3. Server returns `409` (`:response_cat2`); client transitions from `:finalizing` to `:recovering`, queries server (`X-Goog-Upload-Size-Received: 524288`), realigns buffer, re-enters `:finalizing`, and retransmits `upload, finalize` to completion. +* **Assertions**: + * Returned JSON body reports `"size" == 786_332`. + * `phases` equals `[:initiating, :uploading, :uploading, :uploading, :finalizing, :recovering, :uploading, :finalizing, :completed]`. + * `offsets` equals `[0, 0, 262_144, 524_288, 524_288, 524_288, 524_288, 524_288, 786_332]`. + +#### Case 5. Repeated no-header failures until global deadline exceeded (`test_no_headers_failure_recovers_until_deadline_exceeded`) +* **Scenario**: Configures `failure_count: 0, action_after_failures: "terminate"` with a 1-second session `timeout`. +* **Protocol Flow**: + 1. Server responds to every upload chunk with HTTP `500` and no `X-Goog-Upload-Status` header. + 2. `data_plane_retry_policy` treats missing `X-Goog-Upload-Status` as unretriable (`predicate` returns `false`), yielding `Event::HttpResponse(500)` to `Core`. + 3. `Core` classifies the response as Category 2 (`:response_cat2`), enters `:recovering`, queries the server (which returns `200 active` at offset `0`), and retries the upload. + 4. This recovery loop repeats until the 1-second global session deadline expires and `Driver#run` raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. +* **Assertions**: + * Raises `Gapic::Rest::ResumableUpload::DeadlineExceededError`. + * `phases.count(:recovering) >= 2`. + +### 2.4 Error on Start Suite (`integration/resumable_upload/error_on_start_test.rb`) + +Tests non-fatal transient retries, missing status headers, retry exhaustion, fatal errors, and session isolation during the session initiation (`start`) phase. Uses a 100-byte payload, configuring `FAST_RETRY` for non-exhaustion retry cases to minimize test execution latency while retaining default policies for exhaustion and fatal checks. + +#### Case 1. Non-fatal transient error on start (`test_non_fatal_error_on_start_503`) +* **Scenario**: Injects a single `503 Service Unavailable` on the initial `start` request (`scenario: "non_fatal_error_on_start"`, `error_code: 503, failure_count: 1`, `start_retry_policy: FAST_RETRY`). +* **Protocol Flow**: + 1. First `start` POST request receives `503`. + 2. `start_retry_policy` transparently retries the initiation request. + 3. Second attempt succeeds (`200 active`), returning session URL. + 4. 100-byte upload completes normally. +* **Assertions**: + * Returned JSON body reports `"size" == 100`. + * Exactly 1 `:initiating` notification in `phases` (`phases.count(:initiating) == 1`). + * `phases.last == :completed`. + +#### Case 2. Missing status header / 400 on start (`test_missing_header_retriable_on_start_400`) +* **Scenario**: Injects a single `400 Bad Request` without an `X-Goog-Upload-Status` header on `start` (`scenario: "non_fatal_error_on_start"`, `error_code: 400, failure_count: 1`, `start_retry_policy: FAST_RETRY`). +* **Protocol Flow**: + 1. First `start` POST request receives `400` with no upload status header. + 2. `START_PREDICATE` identifies the missing status header on start as retriable (for non-fatal status codes) and retries the initiation request. + 3. Second attempt succeeds (`200 active`). + 4. 100-byte payload is transmitted and finalized. +* **Assertions**: + * Returned JSON body reports `"size" == 100`. + * `phases.count(:initiating) == 1`. + * `phases.last == :completed`. + +#### Case 3. Retry exhaustion and session deadline on start (`test_retry_exhaustion_on_start_times_out`) +* **Scenario**: Injects repeated `503 Service Unavailable` responses (`failure_count: 10_000`) with default start retry policy and a 3-second session `timeout` (`scenario: "non_fatal_error_on_start"`). +* **Protocol Flow**: + 1. `start` command encounters continuous 503 errors. + 2. Client retries with default exponential backoff until the 3-second global session deadline expires. + 3. Client terminates failure before entering transmission. +* **Assertions**: + * Raises a `Gapic::Common::Error` (`BadResponseError` or `DeadlineExceededError`). + * Total elapsed time is close to 3 seconds (`2.5s <= elapsed <= 6.0s`, accounting for default backoff delays and network latency). + * `phases` contains no `:uploading` entries (`refute_includes phases, :uploading`). + +#### Case 4. Fatal errors on start (`test_fatal_error_on_start_raises_bad_response_immediately`) +* **Scenario**: Injects fatal HTTP status codes (`403 Forbidden` and `404 Not Found`) on `start` (`scenario: "fatal_error_on_start"`). +* **Protocol Flow**: + 1. Initial `start` request receives a fatal status code (`403` or `404`). + 2. `START_PREDICATE` refutes retry on fatal status codes (`Rules::FATAL_STATUS_CODES`). + 3. `Driver#execute_send_start` returns the fatal response directly to `Core`. + 4. `Rules` classifies the response as `:response_fatal_bad_response` and emits `:fail_with_bad_response`. + 5. Session terminates immediately with `BadResponseError`. +* **Assertions**: + * Raises `Gapic::Rest::ResumableUpload::BadResponseError` with error message containing the HTTP status code. + * Elapsed time is < 0.5s, confirming zero retries were attempted. + * Refutes any `:uploading` phases. + +#### Case 5. Sequential session isolation (`test_sequential_runs_session_isolation`) +* **Scenario**: Executes two sequential upload runs under Case 1 (`non_fatal_error_on_start`, 503, failure_count: 1, `start_retry_policy: FAST_RETRY`) with distinct client UUIDs. +* **Protocol Flow**: + 1. First run executes and succeeds. + 2. Second run executes with a newly generated `client_uuid` and independent progress tracking. +* **Assertions**: + * Both runs successfully complete uploading 100 bytes. + * Demonstrates Showcase session state isolation across sequential client sessions. + +--- + +### 2.5 Resumption Suite (`integration/resumable_upload/resume_test.rb`) + +Tests `Gapic::Rest::ResumableUpload::Session` resumption capabilities against Showcase. + +#### Case 1. Resume in-progress upload on seekable stream (`test_resume_in_progress_upload`) +* Uploads chunk 1 via `raw_upload`, then resumes with a fresh session and full stream. +* Asserts `phases == [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed]` and offsets align correctly. + +#### Case 2. Resume already finalized upload (`test_resume_finalized_upload`) +* Finalizes upload upfront via `raw_upload(finalize: true)`, then attempts to resume. +* Asserts direct completion with `phases == [:initiating, :completed]` and final payload parsed cleanly. + +#### Case 3. Non-fatal errors on query during resume +* **Case 3a (`test_resume_query_503_absorbed_by_retry`)**: 503 on resume query is absorbed transparently by `control_plane_retry_policy`. +* **Case 3b (`test_resume_query_409_triggers_retry_recovery`)**: 409 Category 2 error on query triggers `retry_recovery` re-querying without progress notification (`refute_includes phases, :recovering`). + +#### Case 4. Unseekable stream fast-forward on resume (`test_resume_unseekable_stream_fast_forwards`) +* Resumes using an `UnseekableStream` starting at byte 0. +* Verifies Driver fast-forwards by discarding bytes up to server offset, then uploads remaining chunks. + +#### Case 5. Stream mismatch errors on resume +* **Case 5a (`test_resume_wrong_stream_unseekable_mismatch`)**: Unseekable stream hitting unexpected EOF during discard raises `StreamMismatchError`. +* **Case 5b (`test_resume_wrong_stream_seekable_size_guard`)**: Seekable `StringIO` with `server_offset > stream.size` (and `upload_size: nil`) raises `StreamMismatchError` via stream size guard. + +#### Case 6. Golden user-style resume (`test_golden_user_style_resume_seekable`, `test_golden_user_style_resume_unseekable`) +* User raises exception in `on_progress` carrying `session.resume_handle` on first upload ack. +* Fresh session resumes via `resume_handle: handle` and completes the transfer. Tested on both seekable streams and fresh unseekable streams starting at byte 0. + +#### Case 7. Lifecycle and contract violations (`test_lifecycle_violations`) +* Verifies second `start` and `resume` on bound session raise `SessionStateError`, and bare `resume` raises `ArgumentError`. diff --git a/gapic-common/integration/README.md b/gapic-common/integration/README.md new file mode 100644 index 0000000..d699fd3 --- /dev/null +++ b/gapic-common/integration/README.md @@ -0,0 +1,51 @@ +# Integration Tests + +This directory contains integration tests for `gapic-common`, executed against a running `gapic-showcase` server via the `toys test-integration` command. + +## Running Integration Tests + +```bash +toys test-integration +``` + +You can pass standard Minitest flags to filter or seed test runs: + +```bash +toys test-integration --name /resumable_upload/ --seed 1234 +``` + +## Showcase Server Management & Lifecycle + +The `toys test-integration` command (`.toys/test-integration.rb`) manages the `gapic-showcase` server lifecycle automatically: + +1. **Existing Endpoint (`SHOWCASE_ENDPOINT`)**: + - If `ENV["SHOWCASE_ENDPOINT"]` is present and non-empty, `toys test-integration` skips binary resolution and runs the Minitest suite directly against that endpoint. + +2. **Binary Resolution (`SHOWCASE_BIN` / `PATH`)**: + - When `SHOWCASE_ENDPOINT` is not set, the runner checks `ENV["SHOWCASE_BIN"]` first, then searches `ENV["PATH"]` for an executable `gapic-showcase` binary. + - **Version Check**: The runner executes ` --version` and verifies that the version is at least `0.43`. If the version is older than `0.43`, it logs an error and exits with status `1`. + +3. **Missing Endpoint and Binary**: + - **CI Environment (`ENV["CI"]` set)**: Fails immediately with exit status `1`. + - **Local Environment (`ENV["CI"]` unset)**: Logs an informational message and skips integration tests cleanly (exit status `0`). + +4. **Ephemeral Port Allocation & Polling**: + - Allocates two ephemeral TCP ports on `127.0.0.1` for `--port :` and `--fallback-port :` to avoid port collisions across concurrent runs. + - Spawns `gapic-showcase run --port : --fallback-port :` in a dedicated process group (`pgroup: true`) with `stdout` and `stderr` redirected to a temporary log file in `Dir.tmpdir`. + - Polls `127.0.0.1:` with a 10-second monotonic clock budget while checking `Process.waitpid2` (`WNOHANG`) on each iteration. If the process exits prematurely or fails to accept TCP connections within 10 seconds, the runner prints the path to the log file and exits with status `1`. + +5. **Execution & Teardown**: + - Sets `ENV["SHOWCASE_ENDPOINT"] = "http://localhost:#{port}"` and runs the Minitest suite (`integration/**/*_test.rb`). + - An `ensure` block sends `SIGTERM` to the entire process group (`-TERM`) and reaps the child process so no background showcase processes are leaked. + +## Logging and Diagnostics + +Each integration test captures `DEBUG`-level client and driver logs into an in-memory buffer during execution: + +- **Automatic Failure Dump**: If a test fails or raises an unhandled exception, the captured trace is automatically dumped to `stderr` during `teardown`. +- **Force Log Dump (`SHOWCASE_LOG`)**: Set `SHOWCASE_LOG=1` (or any non-empty value) to dump the captured trace for all executed tests, including passing ones: + +```bash +SHOWCASE_LOG=1 toys test-integration +``` + diff --git a/gapic-common/integration/integration_helper.rb b/gapic-common/integration/integration_helper.rb new file mode 100644 index 0000000..0af5c0b --- /dev/null +++ b/gapic-common/integration/integration_helper.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "json" +require "logger" +require "securerandom" +require "stringio" +require "minitest/autorun" +require "minitest/focus" +require "minitest/mock" + +require "gapic/common" +require "gapic/rest" +require "gapic/rest/resumable_upload" + +## +# Base class for Showcase integration tests. +# +class ShowcaseIntegrationTest < Minitest::Test + UPLOAD_PATH = "/resumable/upload/v1beta1/files:upload" + FAST_RETRY = { initial_delay: 0.01, max_delay: 0.05, multiplier: 1, timeout: 2 }.freeze + DEFAULT_CHUNK_SIZE = 262_144 + DEFAULT_PAYLOAD_SIZE = DEFAULT_CHUNK_SIZE * 3 + + ## + # Stream double that intentionally does not implement #seek. + # + class UnseekableStream + def initialize data + @io = StringIO.new data + end + + def read length = nil + @io.read length + end + + def pos + @io.pos + end + end + + attr_reader :logger + attr_reader :progress_records + + def phases + @progress_records.map(&:phase) + end + + def offsets + @progress_records.map(&:bytes_uploaded) + end + + def showcase_endpoint + ENV["SHOWCASE_ENDPOINT"] + end + + def setup + skip "SHOWCASE_ENDPOINT is not set" if showcase_endpoint.to_s.empty? + @log_output = StringIO.new + @logger = Logger.new @log_output, level: Logger::DEBUG + super + end + + def teardown + if (!passed? || !ENV["SHOWCASE_LOG"].to_s.empty?) && @log_output && !@log_output.string.empty? + warn "\n--- Captured trace for #{name} ---\n#{@log_output.string}--- End trace ---\n" + end + super + end + + def payload size + pattern = "0123456789".b + (pattern * ((size / pattern.bytesize) + 1)).byteslice 0, size + end + + def showcase_client_stub + Gapic::Rest::ClientStub.new( + endpoint: showcase_endpoint, + credentials: :dummy_credentials, + raise_faraday_errors: false, + logger: @logger + ) + end + + def build_config scenario: nil, scenario_config: {}, **overrides + @progress_records = [] + headers = (overrides[:initial_headers] || {}).dup + if scenario + headers["X-Goog-Test-Scenario"] = scenario + headers["X-Goog-Test-Scenario-Config"] = JSON.generate( + { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) + ) + end + + defaults = { + initial_url: UPLOAD_PATH, + initial_headers: headers, + start_retry_policy: FAST_RETRY, + control_plane_retry_policy: FAST_RETRY, + data_plane_retry_policy: FAST_RETRY, + timeout: 10, + chunk_size: DEFAULT_CHUNK_SIZE, + on_progress: ->(progress) { @progress_records << progress } + } + unless overrides.key? :stream + defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) + defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE + end + + Gapic::Rest::ResumableUpload::StartUploadConfig.new(**defaults, **overrides, initial_headers: headers) + end + + START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze + + # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, + # so callers can run it with `start_session session`. + def build_session scenario: nil, scenario_config: {}, **overrides + @progress_records = [] + headers = (overrides.delete(:initial_headers) || {}).dup + if scenario + headers["X-Goog-Test-Scenario"] = scenario + headers["X-Goog-Test-Scenario-Config"] = JSON.generate( + { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) + ) + end + + @start_args = { + initial_url: UPLOAD_PATH, + initial_headers: headers, + start_retry_policy: FAST_RETRY, + chunk_size: DEFAULT_CHUNK_SIZE + }.merge(overrides.slice(*START_ONLY_KEYS)) + + defaults = { + client_stub: showcase_client_stub, + control_plane_retry_policy: FAST_RETRY, + data_plane_retry_policy: FAST_RETRY, + timeout: 10, + on_progress: ->(progress) { @progress_records << progress }, + logger: @logger + } + unless overrides.key? :stream + defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) + defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE + end + + Gapic::Rest::ResumableUpload::Session.new(**defaults, **overrides.except(*START_ONLY_KEYS)) + end + + def start_session session, **overrides + session.start(**@start_args, **overrides) + end + + def raw_start scenario: nil, scenario_config: {}, upload_size: nil, headers: {} + req_headers = { + "X-Goog-Upload-Protocol" => "resumable", + "X-Goog-Upload-Command" => "start" + } + req_headers["X-Goog-Upload-Header-Content-Length"] = upload_size.to_s if upload_size + if scenario + req_headers["X-Goog-Test-Scenario"] = scenario + req_headers["X-Goog-Test-Scenario-Config"] = JSON.generate( + { "client_uuid" => SecureRandom.uuid }.merge(scenario_config) + ) + end + req_headers.merge! headers + + response = showcase_client_stub.make_post_request( + uri: UPLOAD_PATH, + body: nil, + options: { metadata: req_headers } + ) + Gapic::Rest::ResumableUpload::Rules.header_value response.headers, "x-goog-upload-url" + end + + def raw_upload upload_url:, offset:, bytes:, finalize: false, headers: {} + req_headers = { + "X-Goog-Upload-Command" => finalize ? "upload, finalize" : "upload", + "X-Goog-Upload-Offset" => offset.to_s, + "Content-Type" => "application/octet-stream", + "Content-Length" => bytes.bytesize.to_s + } + req_headers.merge! headers + + showcase_client_stub.make_post_request( + uri: upload_url, + body: bytes, + options: { metadata: req_headers } + ) + end +end diff --git a/gapic-common/integration/resumable_upload/chunk_granularity_test.rb b/gapic-common/integration/resumable_upload/chunk_granularity_test.rb new file mode 100644 index 0000000..ea05965 --- /dev/null +++ b/gapic-common/integration/resumable_upload/chunk_granularity_test.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Integration tests for chunk granularity alignment against Showcase. +# +class ChunkGranularityTest < ShowcaseIntegrationTest + # Verifies chunk size alignment to server-specified granularity (300_000 -> 299_776) and progress notifications. + def test_chunk_granularity_alignment + size = 1_000_000 + config = build_config( + scenario: "chunk_granularity", + stream: StringIO.new(payload(size)), + upload_size: size, + chunk_size: 300_000, + timeout: 5 + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [0, 0, 299_776, 599_552, 899_328, 899_328, 1_000_000], offsets + assert_equal [:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], phases + end +end diff --git a/gapic-common/integration/resumable_upload/error_on_start_test.rb b/gapic-common/integration/resumable_upload/error_on_start_test.rb new file mode 100644 index 0000000..f43d23d --- /dev/null +++ b/gapic-common/integration/resumable_upload/error_on_start_test.rb @@ -0,0 +1,159 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "securerandom" +require "stringio" + +## +# Suite A: Integration tests for non-fatal and fatal errors on session initiation (`start`). +# +class ErrorOnStartTest < ShowcaseIntegrationTest + PAYLOAD_SIZE = 100 + + def build_start_error_config scenario:, scenario_config: {}, start_retry_policy: nil, **overrides + build_config( + scenario: scenario, + scenario_config: scenario_config, + start_retry_policy: start_retry_policy, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + stream: StringIO.new(payload(PAYLOAD_SIZE)), + upload_size: PAYLOAD_SIZE, + **overrides + ) + end + + # A1. Verifies non-fatal transient error (503) on start is retried and upload completes. + def test_non_fatal_error_on_start_503 + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 503, failure_count: 1 }, + start_retry_policy: FAST_RETRY + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal PAYLOAD_SIZE, parsed["size"] + assert_equal 1, phases.count(:initiating) + assert_equal :completed, phases.last + end + + # A2. Verifies missing status header / 400 on start is retried and upload completes. + def test_missing_header_retriable_on_start_400 + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 400, failure_count: 1 }, + start_retry_policy: FAST_RETRY + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal PAYLOAD_SIZE, parsed["size"] + assert_equal 1, phases.count(:initiating) + assert_equal :completed, phases.last + end + + # A3. Verifies retry exhaustion on start with high failure count times out within ~3s without uploading. + def test_retry_exhaustion_on_start_times_out + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 503, failure_count: 10_000 }, + timeout: 3 + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + t0 = Process.clock_gettime Process::CLOCK_MONOTONIC + err = assert_raises Gapic::Common::Error do + driver.run + end + t1 = Process.clock_gettime Process::CLOCK_MONOTONIC + + elapsed = t1 - t0 + assert_operator elapsed, :>=, 2.5 + assert_operator elapsed, :<=, 6.0 + is_expected_error = err.is_a?(Gapic::Rest::ResumableUpload::BadResponseError) || + err.is_a?(Gapic::Rest::ResumableUpload::DeadlineExceededError) + assert is_expected_error, "Expected BadResponseError or DeadlineExceededError, got #{err.class}" + refute_includes phases, :uploading + end + + # A4. Verifies fatal errors on start (403 and 404) immediately raise BadResponseError in < 0.5s without retrying. + def test_fatal_error_on_start_raises_bad_response_immediately + [403, 404].each do |code| + config = build_start_error_config( + scenario: "fatal_error_on_start", + scenario_config: { error_code: code } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + t0 = Process.clock_gettime Process::CLOCK_MONOTONIC + err = assert_raises Gapic::Rest::ResumableUpload::BadResponseError do + driver.run + end + t1 = Process.clock_gettime Process::CLOCK_MONOTONIC + + elapsed = t1 - t0 + assert_operator elapsed, :<, 0.5, "Expected failure in < 0.5s for HTTP #{code}, took #{elapsed}s" + assert_match(/#{code}/, err.message) + refute_includes phases, :uploading + end + end + + # A5. Verifies sequential executions with fresh client UUIDs remain isolated. + def test_sequential_runs_session_isolation + 2.times do + config = build_start_error_config( + scenario: "non_fatal_error_on_start", + scenario_config: { error_code: 503, failure_count: 1 }, + start_retry_policy: FAST_RETRY + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal PAYLOAD_SIZE, parsed["size"] + assert_equal 1, phases.count(:initiating) + assert_equal :completed, phases.last + end + end +end diff --git a/gapic-common/integration/resumable_upload/error_recovery_test.rb b/gapic-common/integration/resumable_upload/error_recovery_test.rb new file mode 100644 index 0000000..fd0f948 --- /dev/null +++ b/gapic-common/integration/resumable_upload/error_recovery_test.rb @@ -0,0 +1,144 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Suite B: Integration tests for Category 1 transient retries and Category 2 error recovery against Showcase. +# +class ErrorRecoveryTest < ShowcaseIntegrationTest + SCENARIO = "non_fatal_error_on_chunk_upload" + + # B1. Verifies Category 1 transient error (503) is retried transparently by FAST_RETRY without entering recovery. + def test_cat1_error_retried_transparently + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 503, failure_count: 1, after_offset: 0 } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + refute_includes phases, :recovering + assert_equal [:initiating, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], phases + assert_equal [0, 0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # B2. Verifies simple Category 2 error (409) at offset 0 triggers recovery query and resumes upload to completion. + def test_simple_cat2_error_recovery + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 409, failure_count: 1, after_offset: 0 } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal( + [:initiating, :uploading, :recovering, :uploading, :uploading, :uploading, :uploading, :finalizing, :completed], + phases + ) + assert_equal [0, 0, 0, 0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # B3. Verifies two consecutive Category 2 recoveries (409) on chunk 2 at offset 262_144. + def test_two_consecutive_cat2_recoveries_on_chunk_2 + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 409, failure_count: 2, after_offset: 262_144 } + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal( + [ + :initiating, :uploading, :uploading, + :recovering, :uploading, + :recovering, :uploading, + :uploading, :uploading, :finalizing, :completed + ], + phases + ) + assert_equal [0, 0, 262_144, 262_144, 262_144, 262_144, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # B4. Verifies Category 2 error (409) on the finalizing chunk (upload, finalize) recovers and completes. + def test_cat2_failure_on_finalizing_chunk + size = (DEFAULT_CHUNK_SIZE * 3) - 100 + config = build_config( + scenario: SCENARIO, + scenario_config: { error_code: 409, failure_count: 1, after_offset: DEFAULT_CHUNK_SIZE * 2 }, + stream: StringIO.new(payload(size)), + upload_size: size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal( + [:initiating, :uploading, :uploading, :uploading, :finalizing, :recovering, :uploading, :finalizing, :completed], + phases + ) + assert_equal [0, 0, 262_144, 524_288, 524_288, 524_288, 524_288, 524_288, size], offsets + end + + # B5. Verifies unrecoverable 500 without status header triggers repeated recovery until DeadlineExceededError. + def test_no_headers_failure_recovers_until_deadline_exceeded + config = build_config( + scenario: SCENARIO, + scenario_config: { failure_count: 0, action_after_failures: "terminate" }, + timeout: 1 + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + assert_raises Gapic::Rest::ResumableUpload::DeadlineExceededError do + driver.run + end + + assert_operator phases.count(:recovering), :>=, 2 + end +end diff --git a/gapic-common/integration/resumable_upload/golden_path_test.rb b/gapic-common/integration/resumable_upload/golden_path_test.rb new file mode 100644 index 0000000..d40c31d --- /dev/null +++ b/gapic-common/integration/resumable_upload/golden_path_test.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Golden path integration tests for ResumableUpload Driver against Showcase. +# +class GoldenPathTest < ShowcaseIntegrationTest + def test_multi_chunk_known_size + size = 1_500_000 + chunk_size = 524_288 + stream = StringIO.new payload(size) + + config = build_config( + stream: stream, + upload_size: size, + chunk_size: chunk_size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [ + Gapic::Rest::ResumableUpload::Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 524_288, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 1_048_576, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :finalizing, bytes_uploaded: 1_048_576, total_bytes: 1_500_000), + Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: 1_500_000, total_bytes: 1_500_000) + ], progress_records + end + + def test_small_upload_default_chunk_size + size = 100_000 + stream = StringIO.new payload(size) + + config = build_config( + stream: stream, + upload_size: size, + chunk_size: nil # use default chunk size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [ + Gapic::Rest::ResumableUpload::Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: size), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: size), + Gapic::Rest::ResumableUpload::Progress.new(phase: :finalizing, bytes_uploaded: 0, total_bytes: size), + Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: size, total_bytes: size) + ], progress_records + end + + def test_standalone_finalize_unseekable_stream + chunk_size = 262_144 + size = 3 * chunk_size + stream = UnseekableStream.new payload(size) + + config = build_config( + stream: stream, + chunk_size: chunk_size + ) + + driver = Gapic::Rest::ResumableUpload::Driver.new( + client_stub: showcase_client_stub, + config: config + ) + + result = driver.run + parsed = JSON.parse result + + assert_equal size, parsed["size"] + assert_equal [ + Gapic::Rest::ResumableUpload::Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 262_144, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 524_288, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :uploading, bytes_uploaded: 786_432, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :finalizing, bytes_uploaded: 786_432, total_bytes: nil), + Gapic::Rest::ResumableUpload::Progress.new(phase: :completed, bytes_uploaded: 786_432, total_bytes: 786_432) + ], progress_records + end +end diff --git a/gapic-common/integration/resumable_upload/resume_test.rb b/gapic-common/integration/resumable_upload/resume_test.rb new file mode 100644 index 0000000..5c4dc5f --- /dev/null +++ b/gapic-common/integration/resumable_upload/resume_test.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "integration_helper" +require "json" +require "stringio" + +## +# Suite D: Integration tests for Resumable Upload Session resumption against Showcase. +# +class ResumeTest < ShowcaseIntegrationTest + ## + # Custom error to simulate a user aborting an in-progress transfer from inside on_progress. + # + class UserPauseError < StandardError + attr_reader :resume_handle + + def initialize message, resume_handle + super message + @resume_handle = resume_handle + end + end + + # D1. Resume an in-progress upload on a seekable stream. + def test_resume_in_progress_upload + upload_url = raw_start upload_size: DEFAULT_PAYLOAD_SIZE + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + session = build_session + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + assert_equal [0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # D2. Resuming an already-finalized upload terminates cleanly and returns final body. + def test_resume_finalized_upload + upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: true + + session = build_session stream: StringIO.new(payload(DEFAULT_CHUNK_SIZE)), upload_size: DEFAULT_CHUNK_SIZE + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_CHUNK_SIZE, parsed["size"] + assert_equal [:initiating, :completed], phases + end + + # D3a. Non-fatal 503 error on query during resume is absorbed by control plane retry policy. + def test_resume_query_503_absorbed_by_retry + upload_url = raw_start( + scenario: "non_fatal_error_on_query", + scenario_config: { error_code: 503, failure_count: 1 }, + upload_size: DEFAULT_PAYLOAD_SIZE + ) + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + session = build_session + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + refute_includes @log_output.string, "retry_recovery" + end + + # D3b. Non-fatal 409 error on query during resume triggers protocol retry_recovery without progress notification. + def test_resume_query_409_triggers_retry_recovery + upload_url = raw_start( + scenario: "non_fatal_error_on_query", + scenario_config: { error_code: 409, failure_count: 1 }, + upload_size: DEFAULT_PAYLOAD_SIZE + ) + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + session = build_session + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + refute_includes phases, :recovering + assert_includes @log_output.string, "retry_recovery" + end + + # D4. Resume fast-forwards by discarding bytes on an unseekable stream starting at byte 0. + def test_resume_unseekable_stream_fast_forwards + upload_url = raw_start upload_size: DEFAULT_PAYLOAD_SIZE + chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false + + stream = UnseekableStream.new payload(DEFAULT_PAYLOAD_SIZE) + session = build_session stream: stream + result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert_equal [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed], phases + assert_equal [0, 262_144, 524_288, 786_432, 786_432, 786_432], offsets + end + + # D5a. Resume with unseekable stream shorter than acknowledged server offset raises StreamMismatchError. + def test_resume_wrong_stream_unseekable_mismatch + upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false + + stream = UnseekableStream.new payload(100) + session = build_session stream: stream, upload_size: nil + + assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do + session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + end + refute_includes phases, :finalizing + end + + # D5b. Resume with seekable stream shorter than server offset raises StreamMismatchError via stream.size guard. + def test_resume_wrong_stream_seekable_size_guard + upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE + raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false + + stream = StringIO.new payload(100) + session = build_session stream: stream, upload_size: nil + + assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do + session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + end + refute_includes phases, :finalizing + end + + # D6a. Golden user-style resume on a seekable stream after user abort in on_progress. + def test_golden_user_style_resume_seekable + stream = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) + session1 = nil + on_progress = lambda do |progress| + if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE + raise UserPauseError.new("user paused", session1.resume_handle) + end + end + + session1 = build_session stream: stream, on_progress: on_progress + err = assert_raises UserPauseError do + start_session session1 + end + + assert session1.bound? + assert session1.resumable? + handle = err.resume_handle + refute_nil handle + + stream.rewind + session2 = build_session stream: stream + result = session2.resume resume_handle: handle + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert session2.bound? + refute session2.resumable? + end + + # D6b. Golden user-style resume with a fresh unseekable stream starting at byte 0. + def test_golden_user_style_resume_unseekable + session1 = nil + on_progress = lambda do |progress| + if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE + raise UserPauseError.new("user paused", session1.resume_handle) + end + end + + session1 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), on_progress: on_progress + err = assert_raises UserPauseError do + start_session session1 + end + + assert session1.bound? + assert session1.resumable? + handle = err.resume_handle + refute_nil handle + + session2 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)) + result = session2.resume resume_handle: handle + parsed = JSON.parse result + + assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] + assert session2.bound? + refute session2.resumable? + end + + # D7. Lifecycle and contract violations on session runs. + def test_lifecycle_violations + session = build_session stream: StringIO.new(payload(100)), upload_size: 100 + start_session session + + assert session.bound? + + # Second start on executed session raises SessionStateError + assert_raises Gapic::Rest::ResumableUpload::SessionStateError do + start_session session + end + + # Resume on already bound/executed session raises SessionStateError + assert_raises Gapic::Rest::ResumableUpload::SessionStateError do + session.resume upload_url: "https://example.com/test", chunk_size: DEFAULT_CHUNK_SIZE + end + + # Resume without parameters on fresh session raises ArgumentError + fresh_session = build_session + assert_raises ArgumentError do + fresh_session.resume + end + end +end diff --git a/gapic-common/lib/gapic/logging_concerns.rb b/gapic-common/lib/gapic/logging_concerns.rb index 78ae13d..8be096f 100644 --- a/gapic-common/lib/gapic/logging_concerns.rb +++ b/gapic-common/lib/gapic/logging_concerns.rb @@ -67,6 +67,10 @@ def debug(&) log(Logger::DEBUG, &) end + def warn(&) + log(Logger::WARN, &) + end + ## # @private # Builder for a log entry, passed to {StubLogger#log}. diff --git a/gapic-common/lib/gapic/rest.rb b/gapic-common/lib/gapic/rest.rb index ae4ccbb..693aba2 100644 --- a/gapic-common/lib/gapic/rest.rb +++ b/gapic-common/lib/gapic/rest.rb @@ -28,6 +28,7 @@ require "gapic/rest/http_binding_override_configuration" require "gapic/rest/operation" require "gapic/rest/paged_enumerable" +require "gapic/rest/resumable_upload" require "gapic/rest/server_stream" require "gapic/rest/threaded_enumerator" require "gapic/rest/transport_operation" diff --git a/gapic-common/lib/gapic/rest/client_stub.rb b/gapic-common/lib/gapic/rest/client_stub.rb index 0ec818c..1fd05b4 100644 --- a/gapic-common/lib/gapic/rest/client_stub.rb +++ b/gapic-common/lib/gapic/rest/client_stub.rb @@ -287,17 +287,27 @@ def log_request method_name, request_id, try_number, body, metadata entry.set "requestId", request_id entry.message = "Sending request to #{entry.service}.#{method_name} (try #{try_number})" end - body = body.to_s + body_str = body.to_s metadata = metadata.to_h rescue {} - return if body.empty? && metadata.empty? + return if body_str.empty? && metadata.empty? stub_logger.debug do |entry| entry.set "requestId", request_id - entry.set "request", body + entry.set "request", abridge_request_body(body_str) entry.set "headers", metadata entry.message = "(request payload as JSON)" end end + def abridge_request_body body_str + utf8_body = body_str.dup.force_encoding Encoding::UTF_8 + if body_str.bytesize > 1024 || !utf8_body.valid_encoding? + prefix_hex = body_str.byteslice(0, 32).unpack1 "H*" + "<#{body_str.bytesize} bytes, first 32: #{prefix_hex}>" + else + utf8_body + end + end + def log_response method_name, request_id, try_number, response, is_server_streaming return unless stub_logger&.enabled? stub_logger.info do |entry| diff --git a/gapic-common/lib/gapic/rest/error.rb b/gapic-common/lib/gapic/rest/error.rb index 910604b..278ba66 100644 --- a/gapic-common/lib/gapic/rest/error.rb +++ b/gapic-common/lib/gapic/rest/error.rb @@ -22,6 +22,9 @@ module Gapic module Rest # Gapic REST exception class class Error < ::Gapic::Common::Error + # @private + REST_ERROR_PREFIX = "An error has occurred when making a REST request".freeze + # @return [Integer, nil] the http status code for the error attr_reader :status_code # @return [Object, nil] the text representation of status as parsed from the response body @@ -79,7 +82,7 @@ def parse_faraday_error err if err.response_body msg, code, status, details = try_parse_from_body err.response_body - message = "An error has occurred when making a REST request: #{msg}" unless msg.nil? + message = "#{REST_ERROR_PREFIX}: #{msg}" unless msg.nil? status_code = code unless code.nil? end diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb new file mode 100644 index 0000000..a7530e1 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload/errors" +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/events" +require "gapic/rest/resumable_upload/instructions" +require "gapic/rest/resumable_upload/retry_policies" +require "gapic/rest/resumable_upload/driver/abridge" +require "gapic/rest/resumable_upload/driver/upload_log" +require "gapic/rest/resumable_upload/rules" +require "gapic/rest/resumable_upload/core" +require "gapic/rest/resumable_upload/driver" +require "gapic/rest/resumable_upload/session" + +module Gapic + module Rest + ## + # Resumable Upload Protocol implementation for REST transport. + # + # {Session} is the primary public entry point for initiating and resuming uploads. + # It manages session initiation, chunked streaming, automatic retries, progress + # callbacks via {Progress}, and cross-session resumption via {ResumeHandle}. + # + # ### Error Types + # * {RequestFailedError} - Transport connection failure, timeout, or retries exhausted (includes {HasResumeHandle}). + # * {DeadlineExceededError} - Global upload timeout exceeded (includes {HasResumeHandle}). + # * {BadResponseError} - Unexpected or malformed HTTP response (includes {HasResumeHandle}). + # * {UnseekableStreamError} - Stream rewinding required on an unseekable stream (includes {HasResumeHandle}). + # * {StreamMismatchError} - Stream content or length does not match resumed upload (includes {HasResumeHandle}). + # * {InvalidTransitionError} - Unmatched event for the current protocol state (includes {HasResumeHandle}). + # * {UploadRejectedError} - Server explicitly rejected the upload session (final). + # * {SessionStateError} - Session lifecycle rule violation, e.g., calling `#start` twice (final). + # + # @example Initiating an upload, rescuing an error, and resuming from a fresh session + # session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: stream + # ) + # + # begin + # response = session.start( + # initial_url: "https://example.googleapis.com/resumable/upload/v1/example/upload:new" + # ) + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # handle = e.resume_handle + # raise unless handle + # + # stream.rewind + # resumed_session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: stream + # ) + # response = resumed_session.resume resume_handle: handle + # end + # + module ResumableUpload + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/core.rb b/gapic-common/lib/gapic/rest/resumable_upload/core.rb new file mode 100644 index 0000000..24b010a --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/core.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/rules" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # State machine container holding the immutable State snapshot. + # Contains zero protocol branching logic and zero side-effects. + # + # The middle tier of the three-tier design: `Driver` executes side effects, {Rules} decides transitions, + # and Core holds the {State} between the two. See {Rules} for the protocol narrative and state graph, and + # `design/resumable_upload/implementation-guide.md` section 1 for the tier boundaries. + # + class Core + # @private + # @return [State] Current immutable state snapshot + attr_reader :state + + # @private + # @return [Decision, nil] Decision emitted during the last dispatch + attr_reader :last_decision + + ## + # @private + # Initializes a Core state machine container. + # + # @param config [StartUploadConfig, ResumeUploadConfig] Upload session configuration + # + def initialize config + @config = config + @last_decision = nil + @state = State.new( + status: :initializing, + upload_url: nil, + offset: 0, + chunk_size: config.chunk_size || Rules::DEFAULT_CHUNK_SIZE, + chunk_granularity: nil, + in_flight_length: 0, + last_error: nil + ) + end + + ## + # @private + # Dispatches event to Rules and updates internal state snapshot. + # + # @param event [Object] Input event + # @return [Array] Driver instructions + # + def dispatch event + decision = Rules.decide @state, event, @config + @state = decision.next_state + @last_decision = decision + decision.instructions + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb new file mode 100644 index 0000000..d08c00d --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb @@ -0,0 +1,415 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Gapic + module Rest + # rubocop:disable Metrics/ModuleLength + module ResumableUpload + ## + # @private + # Configuration members shared by {StartUploadConfig} and {ResumeUploadConfig}, in the order both + # definitions splat them. + # + # The two config types are deliberately *flat*: {Core}, {Rules} and {Driver} read every member + # straight off `config`. Nesting the shared members inside a common object would turn every + # `config.upload_size` into `config.common.upload_size` at some thirty call sites for no behavioural + # gain, so they are spliced into each `Data.define` instead. + # + # * `stream` [IO] Binary input stream to upload. Required. + # * `upload_size` [Integer, nil] Total upload bytes if known upfront. + # * `content_type` [String, nil] MIME type of uploaded media. + # * `timeout` [Numeric, nil] Total upload timeout in seconds (zero or negative is treated as nil). + # * `control_plane_retry_policy` [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control + # commands (query, cancel). + # * `data_plane_retry_policy` [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data commands + # (upload, finalize). + # * `on_progress` [Proc, nil] Callback invoked as `->(progress)` with a {Progress} instance. + # + # Every retry policy member, here and in the per-run configs, follows the same convention: a + # {Gapic::Common::RetryPolicy} replaces the default policy outright, while a Hash overrides only the + # settings it names and leaves the remaining defaults — including retry codes and predicates — in place. + # + COMMON_MEMBERS = [ + :stream, + :upload_size, + :content_type, + :timeout, + :control_plane_retry_policy, + :data_plane_retry_policy, + :on_progress + ].freeze + + ## + # @private + # Header names a caller may not use in `initial_headers`, lowercased for comparison. + # + # These five headers are protocol machinery the driver owns: the protocol identifier, the command + # verb, the byte offset, and the content descriptors derived from `content_type` and `upload_size`. + # `x-goog-upload-offset` is included for completeness even though initiation never sets an offset: + # supplying an offset at initiation is meaningless and indicates a confused caller. Pass-through + # headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. + # + # See `Driver#start_headers`, which builds the initiation headers this list protects. + # + # @return [Array] + RESERVED_INITIAL_HEADERS = [ + "x-goog-upload-protocol", + "x-goog-upload-command", + "x-goog-upload-offset", + "x-goog-upload-header-content-type", + "x-goog-upload-header-content-length" + ].freeze + + ## + # @private + # Immutable configuration for a run that initiates a new upload session, i.e. {Session#start}. + # + # Carries {COMMON_MEMBERS} plus the members only an initiating run uses. + # + # @!attribute [r] initial_url + # @return [String] Initial endpoint URI for session initiation + # @!attribute [r] initial_body + # @return [String, nil] Request payload for session initiation + # @!attribute [r] initial_headers + # @return [Hash] Additional headers for initiation, merged over the driver's + # own headers. Keys in {RESERVED_INITIAL_HEADERS} are rejected in any casing; use + # `content_type` and `upload_size` to shape those. + # @!attribute [r] chunk_size + # @return [Integer, nil] Requested chunk size in bytes, aligned to the granularity the server + # reports during initiation. A resumed run takes its chunk size from {ResumeUploadConfig}. + # @!attribute [r] start_retry_policy + # @return [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for session initiation + # + StartUploadConfig = Data.define( + *COMMON_MEMBERS, + :initial_url, + :initial_body, + :initial_headers, + :chunk_size, + :start_retry_policy + ) do + ## + # @private + # Initializes a new upload configuration. + # + # @param initial_url [String] Initial endpoint URI for session initiation + # @param stream [IO] Binary input stream to upload + # @param initial_body [String, nil] Request payload for session initiation + # @param initial_headers [Hash] Additional headers for initiation. Keys in + # {RESERVED_INITIAL_HEADERS} are rejected in any casing; use `content_type` and `upload_size` + # to shape those. + # @param upload_size [Integer, nil] Total upload bytes if known upfront + # @param chunk_size [Integer, nil] Requested chunk size in bytes + # @param content_type [String, nil] MIME type of uploaded media + # @param timeout [Numeric, nil] Total upload timeout in seconds (zero/negative values treated as nil) + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for session initiation + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control commands + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data commands + # @param on_progress [Proc, nil] Callback invoked as `->(progress)` with a {Progress} instance + # @raise [ArgumentError] If required arguments are missing or invalid + # + def initialize initial_url:, + stream:, + initial_body: nil, + initial_headers: {}, + upload_size: nil, + chunk_size: nil, + content_type: nil, + timeout: nil, + start_retry_policy: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil + raise ArgumentError, "initial_url is required" if initial_url.nil? || initial_url.to_s.strip.empty? + raise ArgumentError, "stream is required" if stream.nil? + reserved = (initial_headers || {}).keys.find do |key| + RESERVED_INITIAL_HEADERS.include? key.to_s.downcase + end + if reserved + raise ArgumentError, + "initial_headers must not set protocol header #{reserved.inspect}; " \ + "use content_type and upload_size instead" + end + + super( + initial_url: initial_url, + initial_body: initial_body, + initial_headers: initial_headers || {}, + stream: stream, + upload_size: upload_size, + chunk_size: chunk_size, + content_type: content_type, + timeout: timeout, + start_retry_policy: start_retry_policy, + control_plane_retry_policy: control_plane_retry_policy, + data_plane_retry_policy: data_plane_retry_policy, + on_progress: on_progress + ) + end + end + + ## + # @private + # Immutable configuration for a run that resumes an existing upload session, i.e. {Session#resume}. + # + # Carries {COMMON_MEMBERS} plus the upload URL and chunk size the earlier run established. There is + # no `start_retry_policy` here: a resumed run issues no initiation request, so the member would + # always be dead. + # + # @!attribute [r] upload_url + # @return [String] Session upload URL returned by the upload backend + # @!attribute [r] chunk_size + # @return [Integer] Explicit chunk size in bytes (must be a positive integer). Server granularity is + # reported only during initiation, which a resumed run skips, so the size is carried forward from + # the earlier run rather than re-negotiated. + # + ResumeUploadConfig = Data.define( + *COMMON_MEMBERS, + :upload_url, + :chunk_size + ) do + ## + # @private + # Initializes a new upload resume configuration. + # + # @param upload_url [String] Session upload URL + # @param chunk_size [Integer] Explicit chunk size in bytes (must be a positive integer) + # @param stream [IO] Binary input stream to upload + # @param upload_size [Integer, nil] Total upload bytes if known upfront + # @param content_type [String, nil] MIME type of uploaded media + # @param timeout [Numeric, nil] Total upload timeout in seconds (zero/negative values treated as nil) + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control commands + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data commands + # @param on_progress [Proc, nil] Callback invoked as `->(progress)` with a {Progress} instance + # @raise [ArgumentError] If required arguments are missing or invalid + # + def initialize upload_url:, + chunk_size:, + stream:, + upload_size: nil, + content_type: nil, + timeout: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil + raise ArgumentError, "upload_url is required" if upload_url.nil? || upload_url.to_s.strip.empty? + unless chunk_size.is_a?(Integer) && chunk_size.positive? + raise ArgumentError, "chunk_size must be a positive integer" + end + raise ArgumentError, "stream is required" if stream.nil? + + super( + upload_url: upload_url, + chunk_size: chunk_size, + stream: stream, + upload_size: upload_size, + content_type: content_type, + timeout: timeout, + control_plane_retry_policy: control_plane_retry_policy, + data_plane_retry_policy: data_plane_retry_policy, + on_progress: on_progress + ) + end + end + + ## + # Immutable progress snapshot passed to the `on_progress` callback. + # + # The `on_progress` callback runs synchronously on the same thread as the upload protocol + # and must not block. Any exception raised inside the callback aborts the upload session + # and propagates out of {Session#start} or {Session#resume}. + # + # @!attribute [r] phase + # @return [Symbol] Current upload phase, one of {Progress::PHASES} + # @!attribute [r] bytes_uploaded + # @return [Integer] Cumulative bytes acknowledged by the server. Note that this is the + # server-confirmed offset and is not guaranteed to be monotonic — a server rewind during + # recovery can decrease this value. + # @!attribute [r] total_bytes + # @return [Integer, nil] Total upload size in bytes if known, or `nil`. Always set on the + # `:completed` phase — the total is known once the transfer finishes, even when `upload_size` + # was not supplied upfront. + # + Progress = Data.define( + :phase, + :bytes_uploaded, + :total_bytes + ) do + ## + # Initializes a new progress snapshot. + # + # @param phase [Symbol] Current upload phase, one of {Progress::PHASES} + # @param bytes_uploaded [Integer] Cumulative bytes acknowledged by the server + # @param total_bytes [Integer, nil] Total upload size in bytes if known, or nil + # @raise [ArgumentError] If the phase is not one of {Progress::PHASES} + # + def initialize phase:, bytes_uploaded:, total_bytes: nil + # Must use `self.class::` to access constants from the class scope + unless self.class::PHASES.include? phase + raise ArgumentError, "Invalid phase: #{phase.inspect}. Expected one of #{self.class::PHASES.inspect}" + end + + super( + phase: phase, + bytes_uploaded: bytes_uploaded, + total_bytes: total_bytes + ) + end + end + + ## + # Allowed lifecycle phases for an upload session. + # + # A callback observes `:initiating`, `:uploading`, `:recovering`, `:finalizing` and `:completed`. + # `:cancelling` is reserved: cancellation is not exposed on {Session}, so no phase with that value is + # currently emitted. + # + # @return [Array] + Progress::PHASES = [:initiating, :uploading, :recovering, :finalizing, :cancelling, :completed].freeze + + ## + # Immutable handle containing parameters necessary to resume an in-progress upload session. + # These parameters are provided by the server and can be persisted to resume the upload + # at a later time. + # + # @!attribute [r] upload_url + # @return [String] Upload session URL provided by the server + # @!attribute [r] chunk_size + # @return [Integer] Effective chunk size in bytes + # + ResumeHandle = Data.define( + :upload_url, + :chunk_size + ) do + ## + # Initializes a new resume handle. + # + # @param upload_url [String] Upload session URL provided by the server + # @param chunk_size [Integer] Effective chunk size in bytes + # + def initialize upload_url:, chunk_size: + super( + upload_url: upload_url, + chunk_size: chunk_size + ) + end + end + + ## + # @private + # Immutable state snapshot representing the current protocol progression. + # + # @!attribute [r] status + # @return [Symbol] Protocol lifecycle status, one of {Rules::STATUSES} + # @!attribute [r] upload_url + # @return [String, nil] Session upload URL returned by the upload backend + # @!attribute [r] offset + # @return [Integer] Contiguous bytes acknowledged by server + # @!attribute [r] chunk_size + # @return [Integer] Resolved effective chunk size in bytes + # @!attribute [r] chunk_granularity + # @return [Integer, nil] Alignment modulus returned by server + # @!attribute [r] in_flight_length + # @return [Integer] Byte length of in-flight chunk currently being transmitted + # @!attribute [r] last_error + # @return [StandardError, nil] Terminal exception if in an error or rejected status + # + State = Data.define( + :status, + :upload_url, + :offset, + :chunk_size, + :chunk_granularity, + :in_flight_length, + :last_error + ) do + ## + # @private + # Initializes a protocol state snapshot. + # + # @param status [Symbol] Protocol lifecycle status, one of {Rules::STATUSES} + # @param upload_url [String, nil] Session upload URL + # @param offset [Integer] Contiguous bytes acknowledged by server + # @param chunk_size [Integer] Resolved effective chunk size in bytes + # @param chunk_granularity [Integer, nil] Alignment modulus returned by server + # @param in_flight_length [Integer] Byte length of in-flight chunk + # @param last_error [StandardError, nil] Terminal exception + # + def initialize status: :initializing, + upload_url: nil, + offset: 0, + chunk_size: Rules::DEFAULT_CHUNK_SIZE, + chunk_granularity: nil, + in_flight_length: 0, + last_error: nil + super( + status: status, + upload_url: upload_url, + offset: offset, + chunk_size: chunk_size, + chunk_granularity: chunk_granularity, + in_flight_length: in_flight_length, + last_error: last_error + ) + end + end + + ## + # @private + # Immutable decision snapshot emitted by Rules.decide. + # + # @!attribute [r] from_status + # @return [Symbol] The protocol status before the transition + # @!attribute [r] shape + # @return [Symbol] The canonical event shape + # @!attribute [r] recipe + # @return [Symbol] Selected transition recipe method name + # @!attribute [r] next_state + # @return [State] The new protocol state snapshot after transition + # @!attribute [r] instructions + # @return [Array] Emitted instructions for the Driver + # + Decision = Data.define( + :from_status, + :shape, + :recipe, + :next_state, + :instructions + ) do + ## + # @private + # Initializes a decision snapshot. + # + # @param from_status [Symbol] The protocol status before the transition + # @param shape [Symbol] The canonical event shape + # @param recipe [Symbol] Selected transition recipe method name + # @param next_state [State] Resulting protocol state snapshot + # @param instructions [Array] Emitted instructions for the Driver + # + def initialize from_status:, shape:, recipe:, next_state:, instructions: [] + super( + from_status: from_status, + shape: shape, + recipe: recipe, + next_state: next_state, + instructions: instructions + ) + end + end + end + # rubocop:enable Metrics/ModuleLength + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb new file mode 100644 index 0000000..563171d --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -0,0 +1,780 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "uri" +require "gapic/logging_concerns" +require "gapic/rest/error" +require "gapic/rest/resumable_upload/core" +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/errors" +require "gapic/rest/resumable_upload/events" +require "gapic/rest/resumable_upload/instructions" +require "gapic/rest/resumable_upload/retry_policies" +require "gapic/rest/resumable_upload/driver/upload_log" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Synchronous execution engine for the Resumable Upload Protocol. + # Coordinates HTTP network operations, stream buffering, monotonic deadlines, + # and delegates state transitions to Core. + # + # The outer tier of the three-tier design. All side effects live here; all protocol decisions live in + # {Rules}, which carries the state graph and the error category taxonomy. Category 1 transient retries + # are absorbed here by `Gapic::Common::RetryPolicy` and never reach {Core}. See + # `design/resumable_upload/implementation-guide.md` section 2.5 for the buffer and stream position + # invariants, and section 6.3 for the deadline model. + # + # rubocop:disable Metrics/ClassLength + class Driver + include Gapic::LoggingConcerns + + ## + # @private + # Minimum assumed upload throughput in bytes per second (1 MB/s). + # @return [Integer] + MIN_ASSUMED_THROUGHPUT = 1_048_576 + + ## + # @private + # Default base timeout in seconds (1 hour). + # @return [Integer] + BASE_TIMEOUT = 3_600 + + # @private + # @return [Core] + attr_reader :core + + ## + # @private + # Returns a {ResumeHandle} representing the current upload session parameters. + # Reading this property mid-run provides a best-effort snapshot of the current session state. + # Completed uploads (`:success`), rejected uploads (`:rejected`), and cancelled uploads + # (`:cancelled`) are finalized and not resumable, returning `nil`. + # + # @return [ResumeHandle, nil] Resume handle if upload URL is established and resumable, or nil + def resume_handle + Rules.resume_handle_from @core.state + end + + ## + # @private + # Returns the raw upload session URL from protocol state, regardless of lifecycle status. + # + # @return [String, nil] Session upload URL if established, or nil + def upload_url + @core.state.upload_url + end + + ## + # @private + # Initializes a new Resumable Upload Driver. + # + # @param client_stub [Gapic::Rest::ClientStub] Underlying REST client stub + # @param config [StartUploadConfig, ResumeUploadConfig] Configuration for this upload session + # @param core [Core, nil] Optional Core state machine (defaults to new Core with config) + # @param logger [Logger, nil] Optional logger override + def initialize client_stub:, config:, core: nil, logger: nil + @client_stub = client_stub + @config = config + @core = core || Core.new(config) + @buffer = "".b + @buffer_start_offset = 0 + + endpoint = client_stub.respond_to?(:endpoint) ? client_stub.endpoint : nil + setup_logging logger: logger || (client_stub.respond_to?(:logger) ? client_stub.logger : nil), + system_name: "gapic-common", + service: "ResumableUpload", + endpoint: endpoint, + client_id: client_stub.object_id + @upload_log = UploadLog.new stub_logger, upload_id: "unstarted" + + # Only an initiating run carries a start policy; a resumed run issues no initiation request. + configured_start_policy = config.is_a?(StartUploadConfig) ? config.start_retry_policy : nil + @start_retry_policy = resolve_retry_policy configured_start_policy, RetryPolicies::START_DEFAULTS + + @control_plane_retry_policy = resolve_retry_policy config.control_plane_retry_policy, + RetryPolicies::CONTROL_PLANE_DEFAULTS + @data_plane_retry_policy = resolve_retry_policy config.data_plane_retry_policy, + RetryPolicies::DATA_PLANE_DEFAULTS + end + + ## + # @private + # Default retry policy for session initiation requests (start). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_start_retry_policy + RetryPolicies.default_start + end + + ## + # @private + # Default retry policy for control plane requests (query, cancel). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_control_plane_retry_policy + RetryPolicies.default_control_plane + end + + ## + # @private + # Default retry policy for data plane requests (upload, finalize). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_data_plane_retry_policy + RetryPolicies.default_data_plane + end + + ## + # @private + # Executes event loop until terminal state. + # Establishes a guaranteed monotonic deadline at the start of execution + # so the upload cannot stall indefinitely. + # + # Enforces the trampoline loop invariant: each dispatched instruction batch + # is validated by {#validate_batch} before any instruction executes, ensuring it + # produces either a single continuation event or terminates the session + # (via {Instruction::TerminateSuccess} or {Instruction::TerminateFailure}). + # Side-effect instructions ({Instruction::NotifyProgress}, + # {Instruction::RealignBuffer}) explicitly return `nil` by construction, + # so only {Instruction::FillBuffer} and `Send*` instructions produce + # continuation events. + # + # @return [String, nil] Final response body + def run + @upload_log = UploadLog.new stub_logger, upload_id: LoggingConcerns.random_uuid4 + @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + resolve_timeout + pending_event = initial_event + + loop do + instructions = dispatch_event pending_event + + if deadline_exceeded? && !terminal_instructions?(instructions) + instructions = dispatch_event Event::GlobalDeadlineExceeded.new + end + + pending_event, terminal_result = execute_batch instructions + return terminal_result if pending_event.nil? + end + end + + private + + ## + # @private + # Executes an instruction batch and enforces the single-continuation-event invariant. + # + # @param instructions [Array] Emitted instructions + # @return [Array] Tuple of [pending_event, terminal_result] + # + def execute_batch instructions + recipe = @core.last_decision&.recipe + validate_batch instructions, recipe + + pending_event = nil + instructions.each do |instruction| + result = dispatch_instruction instruction + return [nil, result] if instruction.is_a? Instruction::TerminateSuccess + pending_event = result if Instruction::CONTINUATION.any? { |klass| instruction.is_a? klass } + end + + unless pending_event_type? pending_event + raise InternalError, + "Resumable upload internal error: recipe :#{recipe} continuation instruction " \ + "returned #{pending_event.class} instead of an event" + end + [pending_event, nil] + end + + ## + # @private + # Validates that an instruction batch satisfies the trampoline invariant before execution. + # + # @param instructions [Array] Emitted instructions + # @param recipe [Symbol, nil] Recipe symbol from last decision + # @return [void] + # @raise [InternalError] If the batch is malformed or contains an unclassified instruction + # + def validate_batch instructions, recipe + continuation = 0 + terminal = 0 + instructions.each do |instruction| + case instruction + when *Instruction::CONTINUATION then continuation += 1 + when *Instruction::TERMINAL then terminal += 1 + when *Instruction::SIDE_EFFECT then nil + else + raise InternalError, + "Resumable upload internal error: recipe :#{recipe} emitted " \ + "unclassified instruction #{instruction.class}" + end + end + return if continuation + terminal == 1 + + raise InternalError, batch_shape_message(recipe, continuation, terminal) + end + + ## + # @private + # Formats diagnostic error message for a malformed instruction batch. + # + # @param recipe [Symbol, nil] Recipe symbol from last decision + # @param continuation [Integer] Number of continuation instructions + # @param terminal [Integer] Number of terminal instructions + # @return [String] Error message + # + def batch_shape_message recipe, continuation, terminal + reason = if continuation.zero? && terminal.zero? + "produced no continuation event and did not terminate" + elsif continuation > 1 && terminal.zero? + "produced multiple continuation events" + elsif continuation.zero? && terminal > 1 + "produced multiple terminal instructions" + else + "produced both a continuation event and a terminal instruction" + end + "Resumable upload internal error: recipe :#{recipe} #{reason}" + end + + ## + # @private + # Dispatches an event to Core, logging decisions and transitions. + # + # @param event [Object] Input event + # @return [Array] Emitted instructions + # + def dispatch_event event + instructions = begin + @core.dispatch event + rescue InvalidTransitionError => e + @upload_log.unmatched_transition @core.state, event, e + raise + end + @upload_log.decision @core.last_decision + @upload_log.lifecycle @core.last_decision, @config + instructions + end + + ## + # @private + # Checks whether an instruction execution result represents a pending event. + # + # @param obj [Object] Execution result + # @return [Boolean] + # + def pending_event_type? obj + obj.is_a?(Event::ChunkRead) || obj.is_a?(Event::HttpResponse) || + obj.is_a?(Event::RequestFailed) || obj.is_a?(Event::GlobalDeadlineExceeded) + end + + ## + # @private + # Executes an instruction emitted by the state machine. + # + # @param instruction [Object] Instruction to execute + # @return [Object, nil] Resulting event or terminal response + # + def dispatch_instruction instruction + case instruction + when Instruction::NotifyProgress then execute_notify_progress instruction + when Instruction::RealignBuffer then execute_realign_buffer instruction + when Instruction::FillBuffer then execute_fill_buffer instruction + when Instruction::SendStart then execute_send_start instruction + when Instruction::SendChunk then execute_send_chunk instruction + when Instruction::SendFinalize then execute_send_finalize instruction + when Instruction::SendQuery then execute_send_query instruction + when Instruction::SendCancel then execute_send_cancel instruction + when Instruction::TerminateSuccess + instruction.response.body + when Instruction::TerminateFailure then raise instruction.error + end + end + + ## + # @private + # Resolves a configured retry policy or applies defaults. + # + # @param value [Gapic::Common::RetryPolicy, Hash, nil] Configured policy or overrides + # @param defaults [Hash] Default policy configuration + # @return [Gapic::Common::RetryPolicy] + # + def resolve_retry_policy value, defaults + case value + when Gapic::Common::RetryPolicy + value + when Hash + Gapic::Common::RetryPolicy.new(**value).apply_defaults(defaults) + when nil + Gapic::Common::RetryPolicy.new(**defaults) + else + raise ArgumentError, "Expected RetryPolicy, Hash, or nil, got #{value.class}" + end + end + + ## + # @private + # Determines the initial event to dispatch based on configuration class. + # + # @return [Event::StartUpload, Event::ResumeUpload] + def initial_event + if @config.is_a? ResumeUploadConfig + Event::ResumeUpload.new + else + Event::StartUpload.new + end + end + + ## + # @private + # Resolves the total upload deadline timeout in seconds. + # + # @return [Numeric] Timeout in seconds + # + def resolve_timeout + return @config.timeout if @config.timeout&.positive? + + # When timeout is unset, BASE_TIMEOUT (1 hour) acts as a floor so small uploads still get + # a full hour while large uploads scale past it at MIN_ASSUMED_THROUGHPUT (1 MiB/s). + if @config.upload_size + [@config.upload_size.fdiv(MIN_ASSUMED_THROUGHPUT), BASE_TIMEOUT].max + else + BASE_TIMEOUT + end + end + + ## + # @private + # Computes the per-request timeout bounded by the global monotonic deadline. + # + # @param retry_policy [Gapic::Common::RetryPolicy, nil] Target command retry policy + # @return [Numeric] Effective per-request timeout + # + def request_timeout retry_policy + remaining = if @deadline + [@deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max + else + resolve_timeout + end + return [remaining, retry_policy.timeout].min if retry_policy&.timeout + + remaining + end + + ## + # @private + # Checks whether the monotonic clock has exceeded the session deadline. + # + # @return [Boolean] + # + def deadline_exceeded? + return false unless @deadline + Process.clock_gettime(Process::CLOCK_MONOTONIC) > @deadline + end + + ## + # @private + # Determines whether the instruction list contains a terminal instruction. + # + # @param instructions [Array] Instruction list + # @return [Boolean] + # + def terminal_instructions? instructions + instructions.any? do |i| + i.is_a?(Instruction::TerminateSuccess) || i.is_a?(Instruction::TerminateFailure) + end + end + + ## + # @private + # Invokes caller progress callback with snapshot. + # + # @param instruction [Instruction::NotifyProgress] Progress instruction + # + def execute_notify_progress instruction + @config.on_progress&.call instruction.progress + nil + end + + ## + # @private + # Realigns in-memory buffer and underlying stream to match server offset. + # + # @param instruction [Instruction::RealignBuffer] Realign instruction + # + def execute_realign_buffer instruction + server_offset = instruction.server_offset + if @config.upload_size && server_offset > @config.upload_size + raise StreamMismatchError.new( + "Server reported offset #{server_offset} exceeds total upload size #{@config.upload_size}", + resume_handle: resume_handle + ) + end + + buffer_start = @buffer_start_offset + buffer_end = @buffer_start_offset + @buffer.bytesize + + realign_case = if server_offset >= buffer_start && server_offset <= buffer_end + "within_buffer" + elsif server_offset < buffer_start + "rewind" + else + "fast_forward" + end + + unseekable = realign_case == "rewind" && !@config.stream.respond_to?(:seek) + @upload_log.buffer_realign realign_case, server_offset: server_offset, + current_offset: buffer_start, + unseekable: unseekable + + if server_offset >= buffer_start && server_offset <= buffer_end + realign_within_buffer server_offset + elsif server_offset < buffer_start + realign_rewind_stream server_offset + else + realign_fast_forward_stream server_offset, buffer_end + end + + nil + end + + ## + # @private + # Slices the in-memory buffer when server offset falls within current buffer range. + # + # @param server_offset [Integer] Target server offset + # + def realign_within_buffer server_offset + slice_index = server_offset - @buffer_start_offset + @buffer = @buffer.byteslice(slice_index..-1) || "".b + @buffer_start_offset = server_offset + end + + ## + # @private + # Rewinds seekable stream when server offset is before current buffer window. + # + # @param server_offset [Integer] Target server offset + # @raise [UnseekableStreamError] If stream does not respond to #seek + # + def realign_rewind_stream server_offset + unless @config.stream.respond_to? :seek + raise UnseekableStreamError.new( + "Cannot rewind unseekable stream to offset #{server_offset} (buffered from #{@buffer_start_offset})", + resume_handle: resume_handle + ) + end + + if @config.upload_size.nil? && @config.stream.respond_to?(:size) && server_offset > @config.stream.size + raise StreamMismatchError.new( + "Server reported offset #{server_offset} exceeds stream size #{@config.stream.size}", + resume_handle: resume_handle + ) + end + + @config.stream.seek server_offset + @buffer = "".b + @buffer_start_offset = server_offset + end + + ## + # @private + # Fast-forwards stream by seeking or discarding bytes. + # + # @param server_offset [Integer] Target server offset + # @param buffer_end [Integer] Current end offset of buffered data + # + def realign_fast_forward_stream server_offset, buffer_end + @buffer = "".b + if @config.stream.respond_to? :seek + if @config.upload_size.nil? && @config.stream.respond_to?(:size) && server_offset > @config.stream.size + raise StreamMismatchError.new( + "Server reported offset #{server_offset} exceeds stream size #{@config.stream.size}", + resume_handle: resume_handle + ) + end + @config.stream.seek server_offset + else + needed_discard = server_offset - buffer_end + while needed_discard.positive? + chunk = @config.stream.read [needed_discard, 65_536].min + if chunk.nil? || chunk.empty? + raise StreamMismatchError.new( + "Stream encountered unexpected EOF during fast-forward to offset #{server_offset} " \ + "(expected at least #{needed_discard} more bytes)", + resume_handle: resume_handle + ) + end + + needed_discard -= chunk.bytesize + end + end + @buffer_start_offset = server_offset + end + + ## + # @private + # Fills internal buffer from stream up to target byte size or EOF. + # + # @param instruction [Instruction::FillBuffer] FillBuffer instruction + # @return [Event::ChunkRead] Chunk read event + # + def execute_fill_buffer instruction + target = instruction.target_bytesize + eof = false + + while @buffer.bytesize < target + bytes_needed = target - @buffer.bytesize + chunk = @config.stream.read bytes_needed + if chunk.nil? || chunk.empty? + eof = true + break + end + @buffer << chunk.b + end + + Event::ChunkRead.new bytes_buffered: @buffer.bytesize, eof: eof + end + + ## + # @private + # Executes session initiation HTTP request. + # + # @param instruction [Instruction::SendStart] SendStart instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_start instruction + policy = @start_retry_policy.dup.start! + headers = start_headers instruction + attempt = 1 + + loop do + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + + event = make_post_request instruction.url, headers: headers, body: instruction.body, + retry_policy: policy, method_name: "ResumableUpload.start", + start_attempt: attempt + return event unless event.is_a? Event::HttpResponse + + status_hdr = Rules.header_value event.headers, "x-goog-upload-status" + return event unless status_hdr.nil? || status_hdr.empty? + return event if Rules::FATAL_STATUS_CODES.include? event.status + + err = BadResponseError.new "Missing X-Goog-Upload-Status header in start response", + event.status, + headers: event.headers + # `retry_with_deadline?` is public; its `@private` tag hides it from docs, not from callers. + can_retry = policy.retry_with_deadline? && policy.call(event) + unless can_retry + if event.status == 200 + failed_event = Event::RequestFailed.new( + kind: :retries_exhausted, message: err.message, source_error: err + ) + @upload_log.wire_failure failed_event + return failed_event + end + return event + end + attempt += 1 + end + end + + ## + # @private + # Builds initiation HTTP headers from instruction and config. + # + # Every header derived here is listed in {RESERVED_INITIAL_HEADERS}, and caller headers in + # that list are rejected when the config is built. The two sets are disjoint, so a plain merge + # cannot drop a driver header or duplicate one under a different casing. + # + # @param instruction [Instruction::SendStart] Start instruction + # @return [Hash] HTTP request headers + # + def start_headers instruction + headers = { "X-Goog-Upload-Protocol" => "resumable", "X-Goog-Upload-Command" => "start" } + headers["X-Goog-Upload-Header-Content-Type"] = @config.content_type if @config.content_type + headers["X-Goog-Upload-Header-Content-Length"] = @config.upload_size.to_s if @config.upload_size + headers.merge instruction.headers || {} + end + + ## + # @private + # Transmits a buffered chunk over HTTP. + # + # @param instruction [Instruction::SendChunk] SendChunk instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_chunk instruction + headers = { + "X-Goog-Upload-Command" => instruction.finalize ? "upload, finalize" : "upload", + "X-Goog-Upload-Offset" => instruction.offset.to_s, + "Content-Type" => @config.content_type || "application/octet-stream", + "Content-Length" => instruction.length.to_s + } + slice_index = instruction.offset - @buffer_start_offset + body = @buffer.byteslice slice_index, instruction.length + + make_post_request instruction.url, headers: headers, body: body, + retry_policy: @data_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.upload" + end + + ## + # @private + # Sends a standalone finalize command over HTTP. + # + # @param instruction [Instruction::SendFinalize] SendFinalize instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_finalize instruction + headers = { + "X-Goog-Upload-Command" => "finalize", + "X-Goog-Upload-Offset" => @core.state.offset.to_s, + "Content-Length" => "0" + } + make_post_request instruction.url, headers: headers, body: "", + retry_policy: @data_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.finalize" + end + + ## + # @private + # Sends an offset query command over HTTP. + # + # @param instruction [Instruction::SendQuery] SendQuery instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_query instruction + headers = { "X-Goog-Upload-Command" => "query", "Content-Length" => "0" } + make_post_request instruction.url, headers: headers, body: "", + retry_policy: @control_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.query" + end + + ## + # @private + # Sends a cancellation command over HTTP. + # + # @param instruction [Instruction::SendCancel] SendCancel instruction + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def execute_send_cancel instruction + headers = { "X-Goog-Upload-Command" => "cancel", "Content-Length" => "0" } + make_post_request instruction.url, headers: headers, body: "", + retry_policy: @control_plane_retry_policy.dup.start!, + method_name: "ResumableUpload.cancel" + end + + ## + # @private + # Dispatches an HTTP POST request through client stub. + # + # @param url [String] Target URL + # @param headers [Hash] Request headers + # @param body [String] Request body + # @param retry_policy [Gapic::Common::RetryPolicy] Command retry policy + # @param method_name [String, nil] RPC method name for logging + # @param start_attempt [Integer] Attempt counter + # @return [Event::HttpResponse, Event::RequestFailed, Event::GlobalDeadlineExceeded] + # + def make_post_request url, headers:, body:, retry_policy:, method_name: nil, start_attempt: 1 + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + + options = { + metadata: headers, + retry_policy: retry_policy, + timeout: request_timeout(retry_policy) + } + @upload_log.wire_send method: "POST", url: url, headers: headers, + start_attempt: start_attempt, body_size: body.to_s.bytesize, body: body + + response = @client_stub.make_post_request uri: url, body: body, params: {}, + options: options, method_name: method_name + event = Event::HttpResponse.new status: response.status, headers: response.headers || {}, body: response.body + @upload_log.wire_receive event + event + rescue StandardError => e + # If the global deadline expired during the HTTP call (e.g. Net::HTTP connection or read timeout + # triggered by request_timeout reaching 0 at @deadline), emit GlobalDeadlineExceeded rather than + # Event::RequestFailed. Otherwise, in states like Recovery where Event::RequestFailed is immediately + # terminal, the state machine would raise the underlying transport error instead of DeadlineExceededError. + return Event::GlobalDeadlineExceeded.new if deadline_exceeded? + + event = rescue_request_error e + if event.is_a? Event::HttpResponse + @upload_log.wire_receive event + else + @upload_log.wire_failure event + end + event + end + + ## + # @private + # Converts client stub transport exceptions into canonical events. + # + # @param err [StandardError] Rescued transport error + # @return [Event::HttpResponse, Event::RequestFailed] + # + def rescue_request_error err + case err + when Gapic::Rest::DeadlineExceededError + Event::RequestFailed.new kind: :timeout, message: err.message, source_error: err + when Gapic::Rest::Error + if err.status_code + Event::HttpResponse.new status: err.status_code, headers: err.headers || {}, body: err.message, + error: err + else + Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + end + when Faraday::Error + rescue_faraday_error err + else + Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + end + end + + ## + # @private + # Converts Faraday client exceptions into canonical events. + # + # @param err [Faraday::Error] Rescued Faraday error + # @return [Event::HttpResponse, Event::RequestFailed] + # + def rescue_faraday_error err + if err.response && err.response[:status] + rest_err = Gapic::Rest::Error.wrap_faraday_error err + Event::HttpResponse.new( + status: err.response[:status], + headers: err.response[:headers] || {}, + body: err.response[:body], + error: rest_err + ) + elsif err.is_a? Faraday::TimeoutError + Event::RequestFailed.new kind: :timeout, message: err.message, source_error: err + elsif err.is_a? Faraday::ConnectionFailed + Event::RequestFailed.new kind: :connection_failed, message: err.message, source_error: err + else + Event::RequestFailed.new kind: :retries_exhausted, message: err.message, source_error: err + end + end + end + # rubocop:enable Metrics/ClassLength + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb new file mode 100644 index 0000000..36cae29 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/abridge.rb @@ -0,0 +1,168 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "uri" + +module Gapic + module Rest + module ResumableUpload + class Driver + ## + # @private + # Pure functions for redacting and abridging sensitive data and large payloads in logs. + # + module Abridge + module_function + + ## + # @private + # Formats binary payload into truncated hex representation. + # + # @param data [Object, nil] Binary or string payload + # @return [String, nil] Truncated hex representation or nil + # + def bytes data + return nil if data.nil? + + str = data.to_s + if str.bytesize >= 64 + "#{str.byteslice(0, 32).unpack1('H*')}... <#{str.bytesize} bytes>" + else + str.unpack1 "H*" + end + end + + ## + # @private + # Truncates error body to a safe log length. + # + # @param data [Object, nil] Error body payload + # @return [String, nil] UTF-8 scrubbed and truncated string + # + def error_body data + return nil if data.nil? + + data.to_s.dup.force_encoding(Encoding::UTF_8).scrub[0, 512] + end + + ## + # @private + # Redacts query parameter values in URLs for safe logging. + # + # @param url [Object, nil] URL string or URI + # @return [String, nil] URL with query values elided + # + def url url + return nil if url.nil? + + uri = URI.parse url.to_s + if uri.query && !uri.query.empty? + elided = uri.query.split("&").map do |pair| + key, _val = pair.split "=", 2 + "#{key}=<...>" + end.join "&" + uri.query = nil + return "#{uri}?#{elided}" + end + uri.to_s + rescue URI::InvalidURIError + url.to_s + end + + ## + # @private + # Redacts non-protocol headers for safe logging. + # + # @param headers [Object] Headers hash + # @return [Hash] Redacted headers + # + def headers headers + return {} unless headers.is_a? Hash + + headers.each_with_object({}) do |(k, v), acc| + key_str = k.to_s + acc[key_str] = if key_str.downcase == "x-goog-upload-url" + url v + elsif key_str.downcase.start_with? "x-goog-upload-" + v + else + "<...>" + end + end + end + + ## + # @private + # Converts a list of instructions into log-safe representation hashes. + # + # @param instructions [Array] List of instructions + # @return [Array] Log-safe instruction summaries + # + def instructions instructions + instructions.map { |i| instruction i } + end + + # rubocop:disable Metrics/MethodLength + ## + # @private + # Converts an instruction into a log-safe representation hash. + # + # @param instruction [Object] Instruction object + # @return [Hash] Log-safe instruction summary + # + def instruction instruction + case instruction + when Instruction::SendStart + { "type" => "SendStart", "url" => url(instruction.url) } + when Instruction::SendChunk + { + "type" => "SendChunk", + "url" => url(instruction.url), + "offset" => instruction.offset, + "length" => instruction.length, + "finalize" => instruction.finalize + } + when Instruction::SendFinalize + { "type" => "SendFinalize", "url" => url(instruction.url) } + when Instruction::SendQuery + { "type" => "SendQuery", "url" => url(instruction.url) } + when Instruction::SendCancel + { "type" => "SendCancel", "url" => url(instruction.url) } + when Instruction::RealignBuffer + { "type" => "RealignBuffer", "serverOffset" => instruction.server_offset } + when Instruction::FillBuffer + { "type" => "FillBuffer", "targetBytesize" => instruction.target_bytesize } + when Instruction::NotifyProgress + { + "type" => "NotifyProgress", + "phase" => instruction.progress.phase.to_s, + "bytesUploaded" => instruction.progress.bytes_uploaded, + "totalBytes" => instruction.progress.total_bytes + } + when Instruction::TerminateSuccess + { "type" => "TerminateSuccess" } + when Instruction::TerminateFailure + { "type" => "TerminateFailure", "error" => instruction.error.to_s } + else + { "type" => instruction.class.name } + end + end + # rubocop:enable Metrics/MethodLength + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb new file mode 100644 index 0000000..d7864b6 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver/upload_log.rb @@ -0,0 +1,326 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "google/logging/message" +require "gapic/rest/resumable_upload/driver/abridge" + +module Gapic + module Rest + module ResumableUpload + class Driver + ## + # @private + # Structured logging helper for a single Resumable Upload run. + # + class UploadLog + ## + # @private + # Recipes omitted from INFO lifecycle logging. + # @return [Array] + SILENT_RECIPES = [ + :ack_chunk, # per-chunk transition, doesn't belong at INFO + :fail_with_unmatched_transition # raises before Decision exists, logged by #unmatched_transition + ].freeze + + ## + # @private + # Severity and message mapping for lifecycle transitions. + # @return [Hash] + LIFECYCLE = { + start_session: [:info, "Initiating resumable upload"], + resume_session: [:info, "Resuming upload session"], + begin_transmission: [:info, "Upload session established"], + send_chunk: [:debug, "Sending upload chunk"], + send_upload_finalize: [:info, "Sending final upload chunk"], + send_finalize: [:info, "Sending finalize command"], + enter_recovery: [:info, "Entering upload recovery"], + retry_recovery: [:info, "Retrying upload recovery query"], + realign_from_recovery: [:info, "Resuming upload from server offset"], + complete_upload_with_data: [:info, "Resumable upload completed"], + complete_upload_finalized: [:info, "Resumable upload completed"], + cancel_session: [:info, "Canceling resumable upload"], + complete_cancellation: [:info, "Resumable upload canceled"], + fail_with_deadline_exceeded: [:warn, "Resumable upload failed"], + fail_with_rejected: [:warn, "Resumable upload failed"], + fail_with_bad_response: [:warn, "Resumable upload failed"], + fail_with_request_error: [:warn, "Resumable upload failed"] + }.freeze + + # @private + # @return [String] + attr_reader :upload_id + + ## + # @private + # Initializes a new UploadLog logger wrapper. + # + # @param stub_logger [Logger, Object] Underlying structured logger + # @param upload_id [String] Unique session identifier + # + def initialize stub_logger, upload_id: + @stub_logger = stub_logger + @upload_id = upload_id + end + + ## + # @private + # Logs state machine transition decision at DEBUG level. + # + # @param decision [Decision] Decision snapshot + # + def decision decision + msg = "Rules: #{decision.from_status} + #{decision.shape} -> " \ + "#{decision.recipe} -> #{decision.next_state.status}" + entry( + :debug, + msg, + fromStatus: decision.from_status, + shape: decision.shape, + recipe: decision.recipe, + toStatus: decision.next_state.status, + offset: decision.next_state.offset, + inFlightLength: decision.next_state.in_flight_length, + instructions: Abridge.instructions(decision.instructions) + ) + end + + ## + # @private + # Logs high-level protocol lifecycle milestone if configured. + # + # @param decision [Decision] Decision snapshot + # @param config [StartUploadConfig, ResumeUploadConfig] Upload configuration + # + def lifecycle decision, config + return if SILENT_RECIPES.include? decision.recipe + + severity, message = LIFECYCLE[decision.recipe] + return unless severity + + extra_fields = lifecycle_fields decision, config + entry severity, message, recipe: decision.recipe, **extra_fields + end + + ## + # @private + # Logs an outgoing HTTP request at DEBUG level. + # + # @param method [String] HTTP method + # @param url [String] Request URL + # @param headers [Hash] Request headers + # @param start_attempt [Integer] Attempt index for start command + # @param body_size [Integer, nil] Byte size of request payload + # @param body [Object, nil] Request payload + # @param body_is_error [Boolean] Whether body contains an error payload + # + def wire_send method:, url:, headers:, start_attempt:, body_size: nil, body: nil, body_is_error: false + command = Rules.header_value headers, "x-goog-upload-command" + offset = Rules.header_value headers, "x-goog-upload-offset" + fields = { + method: method, + url: Abridge.url(url), + headers: Abridge.headers(headers), + startAttempt: start_attempt + } + fields[:command] = command if command + fields[:offset] = offset.to_i if offset + fields[:bodySize] = body_size if body_size + fields[:body] = body_is_error ? Abridge.error_body(body) : Abridge.bytes(body) if body + + entry :debug, "Sending #{method} request", **fields + end + + ## + # @private + # Logs a received HTTP response at DEBUG level. + # + # @param event [Event::HttpResponse] Received HTTP event + # + def wire_receive event + upload_status = Rules.header_value event.headers, "x-goog-upload-status" + size_recv = Rules.header_value event.headers, "x-goog-upload-size-received" + gran = Rules.header_value event.headers, "x-goog-upload-chunk-granularity" + err = event.error if event.respond_to? :error + fields = { + status: event.status, + headers: Abridge.headers(event.headers), + body: wire_receive_body(event, err) + } + fields[:uploadStatus] = upload_status if upload_status + fields[:errorStatus] = err.status if err&.status + fields[:sizeReceived] = size_recv.to_i if size_recv + fields[:granularity] = gran.to_i if gran + + entry :debug, "Received HTTP #{event.status}", **fields + end + + ## + # @private + # Logs a network or transport failure at DEBUG level. + # + # @param event [Event::RequestFailed] Failure event + # + def wire_failure event + entry( + :debug, + "Request failed: #{event.kind}", + kind: event.kind, + error: event.message + ) + end + + ## + # @private + # Logs buffer realignment action. + # + # @param action [String] Realignment action description + # @param server_offset [Integer] Target server offset + # @param current_offset [Integer] Current buffer start offset + # @param unseekable [Boolean] Whether rewind was attempted on an unseekable stream + # + def buffer_realign action, server_offset:, current_offset:, unseekable: false + if unseekable + entry( + :warn, + "Server offset rewind on unseekable stream", + action: action, + serverOffset: server_offset, + currentOffset: current_offset + ) + end + + entry( + :debug, + "Buffer realignment: #{action}", + action: action, + serverOffset: server_offset, + currentOffset: current_offset + ) + end + + ## + # @private + # Logs an invalid or unmatched state machine transition at WARN level. + # + # @param state [State] Current state + # @param event [Object] Triggering event + # @param error [StandardError] Resulting error + # + def unmatched_transition state, event, error + entry( + :warn, + "Unmatched transition", + status: state.status, + shape: Rules.shape_of(event), + error: error.message + ) + end + + private + + ## + # @private + # Formats response body for wire log entry. + # + # @param event [Event::HttpResponse] Response event + # @param err [Gapic::Rest::Error, nil] Error instance + # @return [String, nil] Formatted body + # + def wire_receive_body event, err + return Abridge.bytes event.body if event.status < 400 + + err&.message ? Abridge.error_body(err.message) : Abridge.error_body(event.body) + end + + ## + # @private + # Extracts relevant state fields for lifecycle logging. + # + # @param decision [Decision] Decision snapshot + # @param config [StartUploadConfig, ResumeUploadConfig] Upload configuration + # @return [Hash] Metadata fields for log entry + # + def lifecycle_fields decision, config + state = decision.next_state + case decision.recipe + when :start_session + { uploadSize: config.upload_size, requestedChunkSize: config.chunk_size } + when :resume_session + { + uploadUrl: Abridge.url(state.upload_url), + chunkSize: state.chunk_size + } + when :begin_transmission + { + effectiveChunkSize: state.chunk_size, + granularity: state.chunk_granularity, + uploadUrl: Abridge.url(state.upload_url) + } + when :send_chunk, :send_upload_finalize + { offset: state.offset, inFlightLength: state.in_flight_length } + when :send_finalize, :enter_recovery, :retry_recovery, + :realign_from_recovery, :complete_upload_with_data, :complete_upload_finalized + { offset: state.offset } + when :cancel_session + { uploadUrl: Abridge.url(state.upload_url) } + when :fail_with_deadline_exceeded, :fail_with_rejected, :fail_with_bad_response, + :fail_with_request_error + failure_fields state + else + {} + end + end + + ## + # @private + # Extracts error and response details for failure lifecycle logs. + # + # @param state [State] Current protocol state + # @return [Hash] Failure metadata fields + # + def failure_fields state + err = state.last_error + fields = { error: err&.message || err.to_s } + if err.respond_to?(:response_body) && err.response_body + fields[:responseBody] = Abridge.error_body err.response_body + end + fields + end + + ## + # @private + # Dispatches structured log entry to stub logger. + # + # @param severity [Symbol] Log severity level + # @param log_msg [String] Primary log message + # @param fields [Hash] Structured key-value fields + # + def entry severity, log_msg, **fields + @stub_logger.public_send severity do |builder| + builder.set_system_name + builder.set_service + builder.set "uploadId", @upload_id + fields.each do |k, v| + builder.set k.to_s, v + end + builder.message = log_msg + end + end + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb new file mode 100644 index 0000000..b54e114 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -0,0 +1,565 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/common/error" +require "gapic/rest/error" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # HTTP status code to reason phrase mapping. + # @return [Hash] + HTTP_STATUS_PHRASES = { + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 408 => "Request Timeout", + 409 => "Conflict", + 410 => "Gone", + 411 => "Length Required", + 412 => "Precondition Failed", + 413 => "Payload Too Large", + 415 => "Unsupported Media Type", + 416 => "Range Not Satisfiable", + 429 => "Too Many Requests", + 499 => "Client Closed Request", + 500 => "Internal Server Error", + 502 => "Bad Gateway", + 503 => "Service Unavailable", + 504 => "Gateway Timeout" + }.freeze + + ## + # @private + # Internal formatting helper for terminal error message and attribute extraction. + # + module ErrorBuilder + class << self + ## + # @private + # Formats status representation. + # + # @param status [Object, nil] Status value + # @return [String, nil] + def format_status status + return nil if status.nil? || status.to_s.empty? + + status.to_s + end + + ## + # @private + # Strips REST error prefix from message string. + # + # @param raw_message [String, nil] Raw error message + # @return [String, nil] + def clean_message raw_message + return nil if raw_message.nil? || raw_message.empty? + + prefix = Gapic::Rest::Error::REST_ERROR_PREFIX + msg = raw_message.to_s + msg = msg.sub(/\A#{Regexp.escape prefix}:\s*/, "") if msg.start_with? prefix + msg = msg.sub(/\A:\s*/, "").strip + msg.empty? ? nil : msg + end + + ## + # @private + # Builds error attributes tuple from an HTTP event or wrapped error. + # + # @param event [Object] HTTP response event or failure event + # @param prefix [String] Error message prefix + # @return [Array] Tuple of [message, status_code, status, details, headers] + def build_attributes event, prefix: "Resumable upload failed" + if event.respond_to?(:error) && event.error + build_from_wrapped_error event, prefix: prefix + else + build_from_http_event event, prefix: prefix + end + end + + private + + ## + # @private + # Builds error attributes when a wrapped REST error is available. + # + # @param event [Object] HTTP response event containing wrapped error + # @param prefix [String] Error message prefix + # @return [Array] Tuple of [message, status_code, status, details, headers] + def build_from_wrapped_error event, prefix: + err = event.error + status_code = err.status_code || (event.respond_to?(:status) ? event.status : nil) + status = err.status + status_name = format_status(status) || HTTP_STATUS_PHRASES[status_code] + status_part = status_name ? " #{status_name}" : "" + inner_msg = clean_message err.message + msg = if inner_msg + "#{prefix} with HTTP #{status_code}#{status_part}: #{inner_msg}" + else + "#{prefix} with HTTP #{status_code}#{status_part}" + end + headers = err.headers || (event.respond_to?(:headers) ? event.headers : nil) + [msg, status_code, status, err.details, headers] + end + + ## + # @private + # Builds error attributes directly from raw HTTP response event. + # + # @param event [Object] HTTP response event + # @param prefix [String] Error message prefix + # @return [Array] Tuple of [message, status_code, status, details, headers] + def build_from_http_event event, prefix: + status_code = event.status + headers = event.respond_to?(:headers) && event.headers ? event.headers : {} + upload_status = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] + status_desc = upload_status ? "'#{upload_status}'" : "missing" + status_name = HTTP_STATUS_PHRASES[status_code] + status_part = status_name ? " #{status_name}" : "" + msg = "#{prefix} with HTTP #{status_code}#{status_part} " \ + "(X-Goog-Upload-Status: #{status_desc})" + [msg, status_code, nil, nil, headers] + end + end + end + + ## + # Mixin providing {ResumeHandle} access and uniform formatting for resumable errors. + # + # Every error that may carry a resume handle includes this module, so it doubles as the rescue target + # for "this upload failed but can be retried from where it stopped": + # + # @example + # begin + # session.start initial_url: url + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # retry_later e.resume_handle if e.resume_handle + # raise + # end + # + # Included by {RequestFailedError}, {DeadlineExceededError}, {BadResponseError}, + # {UnseekableStreamError}, {StreamMismatchError} and {InvalidTransitionError}. + # + # Deliberately **not** included by {UploadRejectedError} or {SessionStateError}: the first means the + # server refused the upload outright and the second is a caller misuse, so neither is retryable. Note + # also that `resume_handle` may still be `nil` on an including error, for instance when the failure + # happened before initiation established an upload URL. + # + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated upload session resume handle + # + module HasResumeHandle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] + attr_reader :resume_handle + + ## + # Suffix appended to error message when a resume handle is present. + # @return [String] + RESUMABLE_SUFFIX = " (upload session is resumable: see #resume_handle)" + + ## + # Appends the uniform resumable suffix if resume_handle is non-nil. + # + # @param message [String, nil] Error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Resume handle + # @return [String, nil] + def self.append_suffix message, resume_handle + return message if resume_handle.nil? + return RESUMABLE_SUFFIX.strip if message.nil? || message.to_s.strip.empty? + return message if message.end_with? RESUMABLE_SUFFIX + + "#{message}#{RESUMABLE_SUFFIX}" + end + end + + ## + # Raised when an invalid or unmatched event is dispatched for the current protocol state. + # + # @!attribute [r] response + # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] Associated HTTP response + # @!attribute [r] state + # @return [Symbol, nil] Current protocol state + # @!attribute [r] event + # @return [Object, nil] Received event + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class InvalidTransitionError < Gapic::Common::Error + include HasResumeHandle + + # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] + attr_reader :response + + # @return [Symbol, nil] Current protocol state + attr_reader :state + + # @return [Object, nil] Received event + attr_reader :event + + ## + # Initializes a new InvalidTransitionError. + # + # @param message [String] Descriptive error message + # @param state [Symbol, nil] Current protocol state + # @param event [Object, nil] Received event + # @param response [Gapic::Rest::ResumableUpload::Event::HttpResponse, Object, nil] Associated HTTP response + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message, state: nil, event: nil, response: nil, resume_handle: nil + @state = state + @event = event + @response = response || (event if defined?(Event::HttpResponse) && event.is_a?(Event::HttpResponse)) + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle) + end + + ## + # Creates an InvalidTransitionError from an event. + # + # @param event [Object] Received event + # @param state [Symbol, nil] Current protocol state + # @param message [String, nil] Descriptive error message + # @param response [Object, nil] Associated HTTP response + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [InvalidTransitionError] + def self.from event, state: nil, message: nil, response: nil, resume_handle: nil + new( + message || "Invalid transition for event #{event.inspect}", + state: state, + event: event, + response: response, + resume_handle: resume_handle + ) + end + end + + ## + # Raised when stream rewinding is required but the stream does not support seeking. + # + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class UnseekableStreamError < Gapic::Common::Error + include HasResumeHandle + + ## + # Initializes a new UnseekableStreamError. + # + # @param message [String, nil] Descriptive error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = nil, resume_handle: nil + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle) + end + + ## + # Creates an UnseekableStreamError with optional resume handle. + # + # @param message [String, nil] Descriptive error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [UnseekableStreamError] + def self.from message = nil, resume_handle: nil + new message, resume_handle: resume_handle + end + end + + ## + # Raised when stream content or length does not match resumed upload specifications. + # + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class StreamMismatchError < Gapic::Common::Error + include HasResumeHandle + + ## + # Initializes a new StreamMismatchError. + # + # @param message [String] Error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = "Stream content or length does not match resumed upload", resume_handle: nil + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle) + end + + ## + # Creates a StreamMismatchError with optional resume handle. + # + # @param message [String, nil] Error message + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [StreamMismatchError] + def self.from message = nil, resume_handle: nil + msg = message || "Stream content or length does not match resumed upload" + new msg, resume_handle: resume_handle + end + end + + ## + # Raised when an unrecoverable HTTP response is received. + # + # @!attribute [r] response_body + # @return [String, nil] Response body from backend + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class BadResponseError < Gapic::Rest::Error + include HasResumeHandle + + # @return [String, nil] Response body from backend + attr_reader :response_body + + ## + # Initializes a new BadResponseError. + # + # @param message [String, nil] Error message + # @param status_code [Integer, nil] HTTP status code + # @param status [String, nil] Status description + # @param details [Object, nil] Error details + # @param headers [Object, nil] Response headers + # @param response_body [String, nil] Response body + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = nil, status_code = nil, status: nil, details: nil, headers: nil, + response_body: nil, resume_handle: nil + @response_body = response_body + @resume_handle = resume_handle + super HasResumeHandle.append_suffix(message, resume_handle), + status_code, status: status, details: details, headers: headers + end + + ## + # Creates a BadResponseError from an HTTP response event. + # + # @param event [Object] HTTP response event + # @param response_body [String, nil] Optional response body override + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Optional resume handle + # @return [BadResponseError] + def self.from event, response_body: nil, resume_handle: nil + body = response_body || (event.respond_to?(:body) ? event.body : nil) + message, status_code, status, details, headers = ErrorBuilder.build_attributes event + new message, status_code, status: status, details: details, headers: headers, + response_body: body, resume_handle: resume_handle + end + end + + ## + # Raised when the resumable upload backend explicitly rejects the + # upload session (returns non-2xx with X-Goog-Upload-Status: final). + # + # @!attribute [r] response_body + # @return [String, nil] Response body from backend + # + class UploadRejectedError < Gapic::Rest::Error + # @return [String, nil] Response body from backend + attr_reader :response_body + + ## + # Initializes a new UploadRejectedError. + # + # @param message [String, nil] Error message + # @param status_code [Integer, nil] HTTP status code + # @param status [String, nil] Status description + # @param details [Object, nil] Error details + # @param headers [Object, nil] Response headers + # @param response_body [String, nil] Response body + def initialize message = nil, status_code = nil, status: nil, details: nil, headers: nil, response_body: nil + @response_body = response_body + super message, status_code, status: status, details: details, headers: headers + end + + ## + # Creates an UploadRejectedError from an HTTP response event. + # + # @param event [Object] HTTP response event + # @param response_body [String, nil] Optional response body override + # @return [UploadRejectedError] + def self.from event, response_body: nil + body = response_body || (event.respond_to?(:body) ? event.body : nil) + message, status_code, status, details, headers = + ErrorBuilder.build_attributes event, prefix: "Upload rejected by server" + new message, status_code, status: status, details: details, headers: headers, response_body: body + end + end + + ## + # @private + # Raised when the upload session is cancelled. + # Cancellation is not public yet. + # + class UploadCancelledError < Gapic::Common::Error + ## + # Initializes a new UploadCancelledError. + # + # @param message [String] Cancellation message + def initialize message = "Upload session was cancelled" + super message + end + + ## + # Creates an UploadCancelledError from a source event or message string. + # + # @param source [Object, String, nil] Source event or message + # @return [UploadCancelledError] + def self.from source = nil + if source.is_a?(String) && !source.empty? + new source + else + new + end + end + end + + ## + # Raised when an upload exceeds its global monotonic deadline. + # + # @!attribute [r] root_cause + # @return [Object, nil] Root cause exception if deadline exceeded during a retry loop + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # + class DeadlineExceededError < Gapic::Common::Error + include HasResumeHandle + + # @return [Object, nil] Root cause exception if deadline exceeded during a retry loop + attr_reader :root_cause + + ## + # Initializes a new DeadlineExceededError. + # + # @param message [String] Deadline exceeded message + # @param root_cause [Object, nil] Root cause exception + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + def initialize message = "Upload deadline exceeded", root_cause: nil, resume_handle: nil + super HasResumeHandle.append_suffix(message, resume_handle) + @root_cause = root_cause + @resume_handle = resume_handle + end + + ## + # Creates a DeadlineExceededError with optional resume handle. + # + # @param message [String, nil] Deadline exceeded message + # @param root_cause [Object, nil] Root cause exception + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [DeadlineExceededError] + def self.from message = "Upload deadline exceeded", root_cause: nil, resume_handle: nil + new message, root_cause: root_cause, resume_handle: resume_handle + end + end + + ## + # Raised when an HTTP request fails (e.g. transport connection failure, request timeout, or retries exhausted). + # + # @!attribute [r] cause + # @return [StandardError, nil] Underlying cause exception + # @!attribute [r] resume_handle + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @!attribute [r] status_code + # @return [Integer, nil] HTTP status code if cause was a REST error + # @!attribute [r] status + # @return [String, nil] Status description if cause was a REST error + # @!attribute [r] details + # @return [Object, nil] Error details if cause was a REST error + # @!attribute [r] headers + # @return [Object, nil] Response headers if cause was a REST error + # + class RequestFailedError < Gapic::Common::Error + include HasResumeHandle + + # @return [Integer, nil] + attr_reader :status_code + + # @return [String, nil] + attr_reader :status + + # @return [Object, nil] + attr_reader :details + + # @return [Object, nil] + attr_reader :headers + + ## + # Initializes a new RequestFailedError. + # + # @param message [String, nil] Error message + # @param cause [StandardError, nil] Underlying cause exception + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @param status_code [Integer, nil] HTTP status code + # @param status [String, nil] Status description + # @param details [Object, nil] Error details + # @param headers [Object, nil] Response headers + def initialize message = nil, cause: nil, resume_handle: nil, + status_code: nil, status: nil, details: nil, headers: nil + @cause = cause + @resume_handle = resume_handle + @status_code = status_code || (cause.respond_to?(:status_code) ? cause.status_code : nil) + @status = status || (cause.respond_to?(:status) ? cause.status : nil) + @details = details || (cause.respond_to?(:details) ? cause.details : nil) + @headers = headers || (cause.respond_to?(:headers) ? cause.headers : nil) + msg = message || cause&.message || "Request failed" + super HasResumeHandle.append_suffix(msg, resume_handle) + end + + ## + # Returns the underlying cause exception. + # + # @return [StandardError, nil] + def cause + @cause || super + end + + ## + # Creates a RequestFailedError from a failure event or error. + # + # @param event_or_error [Event::RequestFailed, StandardError] Source event or error + # @param message [String, nil] Optional message override + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Associated resume handle + # @return [RequestFailedError] + def self.from event_or_error, message: nil, resume_handle: nil + if event_or_error.respond_to? :source_error + cause = event_or_error.source_error + msg = message || event_or_error.message || cause&.message || "Request failed" + new msg, cause: cause, resume_handle: resume_handle + elsif event_or_error.is_a? Exception + msg = message || event_or_error.message || "Request failed" + new msg, cause: event_or_error, resume_handle: resume_handle + else + new message || event_or_error.to_s, resume_handle: resume_handle + end + end + end + + ## + # Raised when an operation violates the Session lifecycle rules + # (e.g. calling a `start` method more than once). + # + class SessionStateError < Gapic::Common::Error + end + + ## + # @private + # Raised when an internal state machine or driver invariant is violated. + # Produced by {Rules} when an unlisted shape or recipe is encountered, and by + # {Driver} when a recipe emits a malformed instruction batch. + # + class InternalError < Gapic::Common::Error + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/events.rb b/gapic-common/lib/gapic/rest/resumable_upload/events.rb new file mode 100644 index 0000000..6a8df2c --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/events.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Event vocabulary emitted by the Driver and dispatched to Core/Rules. + # Events are `outside-in` signaling. Something happened, e.g. a chunk of data + # was successfully read, and the Driver is reporting that to Core/Rules. + # + module Event + ## + # @private + # Signals the start of the upload session. + # + StartUpload = Data.define + + ## + # @private + # Signals the resumption of an existing upload session. + # + ResumeUpload = Data.define + + ## + # @private + # Signals that binary data was read from the stream into the Driver's buffer. + # + # @!attribute [r] bytes_buffered + # @return [Integer] Number of bytes currently held in the Driver buffer + # @!attribute [r] eof + # @return [Boolean] Whether stream EOF was encountered during the read + # + ChunkRead = Data.define :bytes_buffered, :eof do + ## + # @private + # Initializes a ChunkRead event. + # + # @param bytes_buffered [Integer] Number of bytes currently held in buffer + # @param eof [Boolean] Whether stream EOF was encountered + # + def initialize bytes_buffered: 0, eof: false + super bytes_buffered: bytes_buffered, eof: eof + end + end + + ## + # @private + # Signals a completed HTTP exchange over the wire (status, headers, body, error). + # + # @!attribute [r] status + # @return [Integer] HTTP status code + # @!attribute [r] headers + # @return [Hash] Response headers + # @!attribute [r] body + # @return [String, nil] Response body + # @!attribute [r] error + # @return [Gapic::Rest::Error, nil] Wrapped REST error if status >= 400 + # + HttpResponse = Data.define :status, :headers, :body, :error do + ## + # @private + # Initializes an HttpResponse event. + # + # @param status [Integer] HTTP status code + # @param headers [Hash] Response headers + # @param body [String, nil] Response body + # @param error [Gapic::Rest::Error, nil] Wrapped REST error + # + def initialize status:, headers: {}, body: nil, error: nil + super status: status, headers: headers || {}, body: body, error: error + end + end + + ## + # @private + # Signals an HTTP request failure (e.g. request timeout, transport connection failure, or retries exhausted). + # + # @!attribute [r] kind + # @return [Symbol] Failure kind: `:timeout`, `:connection_failed`, or `:retries_exhausted` + # @!attribute [r] message + # @return [String, nil] Human-readable failure summary + # @!attribute [r] source_error + # @return [StandardError, nil] Original underlying exception + # + RequestFailed = Data.define :kind, :message, :source_error do + ## + # @private + # Initializes a RequestFailed event. + # + # @param kind [Symbol] Failure kind (`:timeout`, `:connection_failed`, `:retries_exhausted`) + # @param message [String, nil] Human-readable failure summary + # @param source_error [StandardError, nil] Original underlying exception + # + def initialize kind:, message: nil, source_error: nil + super kind: kind, message: message, source_error: source_error + end + end + + ## + # @private + # Signals a caller-requested session cancellation. + # + Cancel = Data.define + + ## + # @private + # Signals that the global monotonic clock exceeded the configured deadline. + # + GlobalDeadlineExceeded = Data.define + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb b/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb new file mode 100644 index 0000000..57d02bd --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/instructions.rb @@ -0,0 +1,265 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Instruction vocabulary emitted by Rules/Core to be executed by Driver. + # The vocabulary is partitioned three ways ({CONTINUATION}, {TERMINAL}, {SIDE_EFFECT}), + # and every instruction class must join exactly one list. + # + module Instruction + ## + # @private + # Execute initiation request to establish upload session. + # + # @!attribute [r] url + # @return [String] Initial endpoint URI + # @!attribute [r] headers + # @return [Hash] Additional headers for initiation request + # @!attribute [r] body + # @return [String, nil] Request payload for session initiation + # + SendStart = Data.define :url, :headers, :body do + ## + # @private + # Initializes a SendStart instruction. + # + # @param url [String] Initial endpoint URI + # @param headers [Hash] Additional headers + # @param body [String, nil] Request payload + # + def initialize url:, headers: {}, body: nil + super url: url, headers: headers || {}, body: body + end + end + + ## + # @private + # Transmit buffered chunk starting at offset for length bytes. + # + # @!attribute [r] url + # @return [String] Session upload URL + # @!attribute [r] offset + # @return [Integer] Byte offset within the full upload stream + # @!attribute [r] length + # @return [Integer] Number of bytes to transmit from buffer + # @!attribute [r] finalize + # @return [Boolean] Whether to append finalize command to upload request + # + SendChunk = Data.define :url, :offset, :length, :finalize do + ## + # @private + # Initializes a SendChunk instruction. + # + # @param url [String] Session upload URL + # @param offset [Integer] Byte offset within upload stream + # @param length [Integer] Number of bytes to transmit + # @param finalize [Boolean] Whether to combine upload and finalize commands + # + def initialize url:, offset:, length:, finalize: false + super url: url, offset: offset, length: length, finalize: finalize + end + end + + ## + # @private + # Send standalone finalize command when all data bytes were already uploaded. + # + # @!attribute [r] url + # @return [String] Session upload URL + # + SendFinalize = Data.define :url do + ## + # @private + # Initializes a SendFinalize instruction. + # + # @param url [String] Session upload URL + # + def initialize url: + super url: url + end + end + + ## + # @private + # Query backend for current acknowledged offset. + # + # @!attribute [r] url + # @return [String] Session upload URL + # + SendQuery = Data.define :url do + ## + # @private + # Initializes a SendQuery instruction. + # + # @param url [String] Session upload URL + # + def initialize url: + super url: url + end + end + + ## + # @private + # Cancel upload session on backend. + # + # @!attribute [r] url + # @return [String] Session upload URL + # + SendCancel = Data.define :url do + ## + # @private + # Initializes a SendCancel instruction. + # + # @param url [String] Session upload URL + # + def initialize url: + super url: url + end + end + + ## + # @private + # Realign Driver in-memory buffer and stream position to match server_offset. + # + # @!attribute [r] server_offset + # @return [Integer] Acknowledged byte offset reported by server + # + RealignBuffer = Data.define :server_offset do + ## + # @private + # Initializes a RealignBuffer instruction. + # + # @param server_offset [Integer] Target server byte offset + # + def initialize server_offset: + super server_offset: server_offset + end + end + + ## + # @private + # Read from stream until in-memory buffer reaches target_bytesize or stream hits EOF. + # + # @!attribute [r] target_bytesize + # @return [Integer] Target buffer size in bytes + # + FillBuffer = Data.define :target_bytesize do + ## + # @private + # Initializes a FillBuffer instruction. + # + # @param target_bytesize [Integer] Target buffer size in bytes + # + def initialize target_bytesize: + super target_bytesize: target_bytesize + end + end + + ## + # @private + # Invoke user progress callback with a Progress instance. + # + # @!attribute [r] progress + # @return [Gapic::Rest::ResumableUpload::Progress] Progress notification snapshot + # + NotifyProgress = Data.define :progress do + ## + # @private + # Initializes a NotifyProgress instruction. + # + # @param progress [Gapic::Rest::ResumableUpload::Progress] Progress notification snapshot + # + def initialize progress: + super progress: progress + end + end + + ## + # @private + # Upload finalized cleanly; return response. + # + # @!attribute [r] response + # @return [Gapic::Rest::ResumableUpload::Event::HttpResponse] Final response object + # + TerminateSuccess = Data.define :response do + ## + # @private + # Initializes a TerminateSuccess instruction. + # + # @param response [Gapic::Rest::ResumableUpload::Event::HttpResponse] Final response object + # + def initialize response: + super response: response + end + end + + ## + # @private + # Terminate upload with error. + # + # @!attribute [r] error + # @return [StandardError] Terminal exception to raise + # + TerminateFailure = Data.define :error do + ## + # @private + # Initializes a TerminateFailure instruction. + # + # @param error [StandardError] Terminal exception to raise + # + def initialize error: + super error: error + end + end + + ## + # @private + # Instruction classes that produce a continuation event for the next step of the trampoline loop. + # @return [Array] + CONTINUATION = [ + FillBuffer, + SendStart, + SendChunk, + SendFinalize, + SendQuery, + SendCancel + ].freeze + + ## + # @private + # Instruction classes that terminate the upload run. + # @return [Array] + TERMINAL = [ + TerminateSuccess, + TerminateFailure + ].freeze + + ## + # @private + # Instruction classes that perform side effects without producing continuation events or terminating. + # @return [Array] + SIDE_EFFECT = [ + NotifyProgress, + RealignBuffer + ].freeze + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb new file mode 100644 index 0000000..deb92bf --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/common/retry_policy" +require "gapic/rest/resumable_upload/rules" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Default retry policy generators for control plane and data plane requests. + # + module RetryPolicies + ## + # @private + # Retry predicate determining retriability for session initiation requests. + # Retries missing status header on non-fatal codes. + # @return [Proc] + START_PREDICATE = lambda do |error_or_response| + status = extract_status_code error_or_response + return false if Rules::FATAL_STATUS_CODES.include? status + + headers = extract_headers error_or_response + if headers + status_hdr = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] + return true if status_hdr.nil? || status_hdr.empty? + end + nil + end + + ## + # @private + # Retry predicate determining retriability for data plane requests. + # Disallows retry when upload status header is missing. + # @return [Proc] + DATA_PLANE_PREDICATE = lambda do |error_or_response| + headers = extract_headers error_or_response + if headers + status_hdr = headers["x-goog-upload-status"] || headers["X-Goog-Upload-Status"] + return false if status_hdr.nil? || status_hdr.empty? + end + nil + end + + ## + # @private + # Default options for start command retry policy. + # Keep in sync with the "Retry Policies" section of Session's class doc. + # @return [Hash] + START_DEFAULTS = { + retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3, + retry_predicate: START_PREDICATE + }.freeze + + ## + # @private + # Default options for query and cancel commands retry policy. + # Keep in sync with the "Retry Policies" section of Session's class doc. + # @return [Hash] + CONTROL_PLANE_DEFAULTS = { + retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3 + }.freeze + + ## + # @private + # Default options for upload and finalize commands retry policy. + # Keep in sync with the "Retry Policies" section of Session's class doc. + # @return [Hash] + DATA_PLANE_DEFAULTS = { + retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, + initial_delay: 1.0, + max_delay: 15.0, + multiplier: 1.3, + retry_predicate: DATA_PLANE_PREDICATE + }.freeze + + ## + # @private + # Default retry policy for session initiation requests (start). + # Missing X-Goog-Upload-Status header is retriable across any response code, + # including 200 (predicate returns true). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_start + Gapic::Common::RetryPolicy.new(**START_DEFAULTS) + end + + ## + # @private + # Default retry policy for session control requests (query, cancel). + # Does not retry on missing X-Goog-Upload-Status header. + # + # @return [Gapic::Common::RetryPolicy] + def self.default_control_plane + Gapic::Common::RetryPolicy.new(**CONTROL_PLANE_DEFAULTS) + end + + ## + # @private + # Default retry policy for data plane requests (upload, finalize). + # Missing X-Goog-Upload-Status header is unretriable (predicate returns false). + # + # @return [Gapic::Common::RetryPolicy] + def self.default_data_plane + Gapic::Common::RetryPolicy.new(**DATA_PLANE_DEFAULTS) + end + + ## + # @private + # Extracts headers hash from Faraday response or error object. + # + # @param error_or_response [Object] Response, error, or hash object + # @return [Hash, nil] + def self.extract_headers error_or_response + if error_or_response.respond_to? :headers + error_or_response.headers + elsif error_or_response.respond_to? :response_headers + error_or_response.response_headers + elsif error_or_response.respond_to?(:response) && error_or_response.response.is_a?(Hash) + error_or_response.response[:headers] + end + end + + ## + # @private + # Extracts HTTP status code from Faraday response, error, or event object. + # + # @param error_or_response [Object] Response, error, or event object + # @return [Integer, nil] + def self.extract_status_code error_or_response + if error_or_response.respond_to?(:status_code) && error_or_response.status_code.is_a?(Integer) + error_or_response.status_code + elsif error_or_response.respond_to?(:response) && error_or_response.response.is_a?(Hash) && + error_or_response.response[:status].is_a?(Integer) + error_or_response.response[:status] + elsif error_or_response.respond_to?(:response_status) && error_or_response.response_status.is_a?(Integer) + error_or_response.response_status + elsif error_or_response.respond_to?(:status) && error_or_response.status.is_a?(Integer) + error_or_response.status + end + end + end + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/rules.rb b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb new file mode 100644 index 0000000..90cc14f --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/rules.rb @@ -0,0 +1,1016 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/common/error" +require "gapic/rest/resumable_upload/errors" +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/events" +require "gapic/rest/resumable_upload/instructions" + +module Gapic + module Rest + module ResumableUpload + ## + # @private + # Pure functional transition engine for the Resumable Upload Protocol. + # Contains zero side-effects and zero persistent state. + # + # ### Model + # + # {Rules.decide} is the protocol. It is a total function of `[state.status, shape_of(event)]` returning a + # {Decision} that carries the next {State} and the instructions for the Driver to execute. Three + # vocabularies define it, each published as a frozen constant: + # + # * {STATUSES} - protocol lifecycle statuses a {State} may hold. + # * {SHAPES} - canonical event shapes that {Rules.shape_of} reduces raw events to. + # * {RECIPES} - transition handlers that {Rules.decide} may select. + # + # Every router arm maps one (status, shape) pair to exactly one recipe, and every recipe returns + # `[next_state, instructions]`. Adding a protocol behaviour means adding a shape, a recipe and an arm. + # It never means adding branching to the Driver. + # + # ### Trampoline invariant + # + # {Driver#run} executes as a synchronous trampoline loop, so every recipe in {RECIPES} must return an + # instruction batch that yields either: + # + # 1. **Exactly one** event-producing instruction (`FillBuffer` or `Send*`) and zero terminal instructions, or + # 2. **Exactly one** terminal instruction (`TerminateSuccess` or `TerminateFailure`) and zero event-producing + # instructions. + # + # A recipe returning zero event-producing instructions without terminating stalls the loop, and a recipe + # returning multiple event-producing instructions discards continuation events. The Driver validates each + # batch against {Instruction::CONTINUATION}, {Instruction::TERMINAL} and {Instruction::SIDE_EFFECT} before + # executing any instruction, rejecting malformed batches up front. Side-effect instructions + # ({Instruction::NotifyProgress}, {Instruction::RealignBuffer}) explicitly return `nil` in the Driver by + # construction, so only {Instruction::FillBuffer} and `Send*` instructions produce continuation events. + # + # ### State transition graph + # + # ```mermaid + # stateDiagram-v2 + # [*] --> initializing + # initializing --> starting : start_upload + # initializing --> recovery : resume_upload + # starting --> transmission_reading : response_active + # transmission_reading --> transmission_sending : chunk_read_full + # transmission_sending --> transmission_reading : response_active + # transmission_reading --> finalizing_sending_upload : chunk_read_eof_with_data + # transmission_reading --> finalizing_sending_finalize : chunk_read_eof_empty + # finalizing_sending_upload --> success : response_final + # finalizing_sending_finalize --> success : response_final + # transmission_sending --> recovery : response_cat2 / connection_failed / timeout + # finalizing_sending_upload --> recovery : response_cat2 / connection_failed / timeout + # finalizing_sending_finalize --> recovery : response_cat2 / connection_failed / timeout + # recovery --> recovery : response_cat2 + # recovery --> transmission_reading : response_active + # recovery --> success : response_final + # starting --> error : response_cat2 / response_fatal_bad_response / request_* + # recovery --> error : request_* + # transmission_sending --> rejected : response_rejected + # recovery --> rejected : response_rejected + # cancelling --> cancelled : response_cancelled + # success --> [*] + # rejected --> [*] + # cancelled --> [*] + # error --> [*] + # ``` + # + # The graph shows the protocol's intended path and its recoverable detours. Failure edges are largely omitted + # to keep it readable: every non-terminal status can also reach `error` (on `:global_deadline_exceeded`, on an + # unretriable request failure, on a fatally bad response, or on any unmatched event) and `rejected` (on + # `:response_rejected`), and every status listed in the `:user_cancel` arm can reach `cancelling`. `cancelling` + # in particular has only its success edge drawn; it fails like any other in-flight state. {Rules.decide} is the + # authoritative enumeration. + # + # ### Router ordering + # + # Arms are evaluated top to bottom, so their order encodes precedence and is load-bearing: + # + # * `enter_recovery` and `fail_with_request_error` both match + # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize,` + # `:request_connection_failed | :request_timeout]`. Recovery wins purely because its arm precedes + # `fail_with_request_error`. + # * `[:starting, :response_cat2]` fails instead of recovering, unlike the same shape during transmission + # and finalizing. There is no upload to recover to until initiation yields an upload URL. + # * `recovery` re-queries on `:response_cat2` with no attempt cap. Termination is guaranteed only by the + # global deadline the Driver enforces, not by anything in this module. + # + # See `design/resumable_upload/implementation-guide.md` section 4 for the transition specification and + # section 6.1 for the error category taxonomy this module implements. + # + # rubocop:disable Metrics/ModuleLength + module Rules + ## + # @private + # Default chunk size in bytes (8 MB). + # @return [Integer] + DEFAULT_CHUNK_SIZE = 8_388_608 # 8 MB + + # Failures are classified into three categories, which the rest of this module is written in terms of: + # + # * **Category 1 (transient transport)** - connection resets, DNS failures, load shedding. Handled + # entirely inside the Driver by `Gapic::Common::RetryPolicy`; Core never sees them. Only their + # exhaustion reaches this module, as `:request_retries_exhausted`. + # * **Category 2 (recoverable protocol)** - the client offset may be misaligned with the server, or a + # proxy stripped the protocol headers. Resolved by querying the server for its acknowledged offset + # and realigning, never by blindly retransmitting. Shape: `:response_cat2`. + # * **Category 3 (terminal)** - structurally invalid, unauthorized, rejected, or out of budget. + # Resolved by transitioning to `:error` or `:rejected` and emitting `Instruction::TerminateFailure`. + # + # See `design/resumable_upload/implementation-guide.md` section 6.1 for the full classification. + + ## + # @private + # HTTP status codes eligible for Category 2 (recovery) handling. + # + # Descriptive rather than load-bearing: {Rules.classify_http_response} routes any non-fatal status with a + # missing or empty `X-Goog-Upload-Status` to `:response_cat2`, so this list does not gate the decision. + # It records the codes the upload backend is expected to produce in that situation, and is asserted against + # {Rules.classify_http_response} by the classification tests. + # + # @return [Array] + CAT2_STATUS_CODES = [400, 408, 409, 412, 416, 429, 499].freeze + + ## + # @private + # HTTP status codes that are immediately fatal and non-retriable (Category 3). + # + # Unlike {CAT2_STATUS_CODES} this list is load-bearing: {Rules.classify_http_response} consults it to decide + # between `:response_fatal_bad_response` and `:response_cat2` when the upload status header is absent, + # and {RetryPolicies::START_PREDICATE} consults it to refuse retries outright. + # + # @return [Array] + FATAL_STATUS_CODES = [401, 403, 404, 405, 410, 413, 415].freeze + + ## + # @private + # Human-readable state descriptions for error reporting. + # @return [Hash] + STATE_DESCRIPTIONS = { + initializing: "initializing upload", + starting: "initiating upload session", + transmission_reading: "reading chunk from stream", + transmission_sending: "sending a chunk of data", + finalizing_sending_upload: "sending final data chunk", + finalizing_sending_finalize: "sending finalize command", + recovery: "querying upload offset for recovery", + cancelling: "cancelling upload session", + success: "in completed upload state", + cancelled: "in cancelled upload state", + error: "in error state", + rejected: "in rejected upload state" + }.freeze + + ## + # @private + # Canonical list of protocol lifecycle statuses a {State} may hold. Guaranteed to match the keys of + # {STATE_DESCRIPTIONS} by the classification test suite. + # + # * `:initializing` - nothing dispatched yet; awaits `:start_upload` or `:resume_upload`. + # * `:starting` - initiation request in flight; no upload URL yet. + # * `:transmission_reading` - filling the buffer from the stream. + # * `:transmission_sending` - a non-final chunk is in flight. + # * `:finalizing_sending_upload` - the last chunk is in flight, combined with the finalize command. + # * `:finalizing_sending_finalize` - a standalone finalize is in flight; all data bytes were already sent. + # * `:recovery` - offset query in flight, either after a recoverable failure or as the first step of a + # resume. + # * `:cancelling` - cancel command in flight. Not reachable from the public API. + # * `:success` - terminal; the upload finalized. + # * `:cancelled` - terminal; the server acknowledged cancellation. + # * `:rejected` - terminal; the server refused the upload. + # * `:error` - terminal for this run; `last_error` holds the exception. + # + # `:success`, `:cancelled` and `:rejected` are finalized and yield no {ResumeHandle}. `:error` ends the + # run but may still be resumable from a fresh session; see {Rules.resume_handle_from}. + # + # @return [Array] + STATUSES = [ + :initializing, + :starting, + :transmission_reading, + :transmission_sending, + :finalizing_sending_upload, + :finalizing_sending_finalize, + :recovery, + :cancelling, + :success, + :cancelled, + :rejected, + :error + ].freeze + + ## + # @private + # Terminal protocol lifecycle statuses in {STATUSES}. + # @return [Array] + TERMINAL_STATUSES = [:success, :cancelled, :rejected, :error].freeze + + ## + # @private + # Canonical list of event shapes produced by {Rules.shape_of} and matched by {Rules.decide}, + # grouped by the event family each is reduced from. + # + # Lifecycle signals, one shape each from {Event::StartUpload}, {Event::ResumeUpload}, {Event::Cancel} + # and {Event::GlobalDeadlineExceeded}: `:start_upload`, `:resume_upload`, `:user_cancel`, + # `:global_deadline_exceeded`. + # + # Stream reads, from {Event::ChunkRead} split by EOF and buffer occupancy. The three-way split is what + # lets a zero-length tail finalize without sending an empty chunk: `:chunk_read_full`, + # `:chunk_read_eof_with_data`, `:chunk_read_eof_empty`. + # + # Request failures, from {Event::RequestFailed} split by `kind`: `:request_timeout`, + # `:request_retries_exhausted`, `:request_connection_failed`, `:request_failed_unknown`. + # + # HTTP responses, from {Event::HttpResponse} split by `X-Goog-Upload-Status` and HTTP status: + # `:response_active`, `:response_final`, `:response_cancelled`, `:response_rejected`, `:response_cat2`, + # `:response_fatal_bad_response`. + # + # `:unknown` is a live shape rather than an error sentinel. It is what {Rules.shape_of} returns for anything + # it does not recognise, and it routes to {Rules.fail_with_unmatched_transition}. + # + # @return [Array] + SHAPES = [ + :start_upload, + :resume_upload, + :user_cancel, + :global_deadline_exceeded, + :chunk_read_full, + :chunk_read_eof_with_data, + :chunk_read_eof_empty, + :request_timeout, + :request_retries_exhausted, + :request_connection_failed, + :request_failed_unknown, + :response_active, + :response_final, + :response_cancelled, + :response_rejected, + :response_cat2, + :response_fatal_bad_response, + :unknown + ].freeze + + ## + # @private + # Canonical list of recipe symbols emitted by {Rules.decide}. + # @return [Array] + RECIPES = [ + :start_session, + :resume_session, + :begin_transmission, + :send_chunk, + :send_upload_finalize, + :send_finalize, + :ack_chunk, + :enter_recovery, + :retry_recovery, + :realign_from_recovery, + :complete_upload_with_data, + :complete_upload_finalized, + :cancel_session, + :complete_cancellation, + :fail_with_deadline_exceeded, + :fail_with_rejected, + :fail_with_bad_response, + :fail_with_request_error, + :fail_with_unmatched_transition + ].freeze + + ## + # @private + # Mapping of notifying recipes to their emitted {Progress} phase. + # @return [Hash] + RECIPE_PHASES = { + start_session: :initiating, + resume_session: :initiating, + begin_transmission: :uploading, + ack_chunk: :uploading, + realign_from_recovery: :uploading, + enter_recovery: :recovering, + send_upload_finalize: :finalizing, + send_finalize: :finalizing, + complete_upload_with_data: :completed, + complete_upload_finalized: :completed, + cancel_session: :cancelling + }.freeze + + ## + # @private + # Recipes that do not emit {Instruction::NotifyProgress}. + # @return [Array] + NON_NOTIFYING_RECIPES = [ + :send_chunk, + :retry_recovery, + :complete_cancellation, + :fail_with_deadline_exceeded, + :fail_with_rejected, + :fail_with_bad_response, + :fail_with_request_error, + :fail_with_unmatched_transition + ].freeze + + ## + # @private + # Classifies incoming event into a canonical shape symbol. + # + # @param event [Object] Input event + # @return [Symbol] Canonical event shape + def self.shape_of event + case event + when Event::StartUpload, Event::StartUpload.singleton_class + :start_upload + when Event::ResumeUpload, Event::ResumeUpload.singleton_class + :resume_upload + when Event::ChunkRead + classify_chunk_read event + when Event::Cancel, Event::Cancel.singleton_class + :user_cancel + when Event::GlobalDeadlineExceeded, Event::GlobalDeadlineExceeded.singleton_class + :global_deadline_exceeded + when Event::RequestFailed + classify_request_failed event + when Event::HttpResponse + classify_http_response event + when Class + classify_event_class event + else + :unknown + end + end + + ## + # @private + # Top-level transition decision engine. Matches [state.status, shape]. + # + # @param state [State] Current state + # @param event [Object] Input event + # @param config [StartUploadConfig, ResumeUploadConfig] Static configuration + # @return [Decision] Decision snapshot + # + # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength + def self.decide state, event, config + shape = shape_of event + unless SHAPES.include? shape + raise InternalError, "Resumable upload internal error: shape_of returned unknown shape #{shape.inspect}" + end + + recipe = case [state.status, shape] + in [:initializing, :start_upload] + :start_session + in [:initializing, :resume_upload] + :resume_session + in [:starting, :response_active] + :begin_transmission + in [:transmission_reading, :chunk_read_full] + :send_chunk + in [:transmission_reading, :chunk_read_eof_with_data] + :send_upload_finalize + in [:transmission_reading, :chunk_read_eof_empty] + :send_finalize + in [:transmission_sending, :response_active] + :ack_chunk + # Order matters: `enter_recovery` and `fail_with_request_error` below both match + # `[:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, + # :request_connection_failed | :request_timeout]`. Recovery wins purely because this arm comes first. + in [:transmission_sending | :finalizing_sending_upload | :finalizing_sending_finalize, + :response_cat2 | :request_connection_failed | :request_timeout] + :enter_recovery + in [:finalizing_sending_upload, :response_final] + :complete_upload_with_data + in [:finalizing_sending_finalize | :recovery, :response_final] + :complete_upload_finalized + in [:recovery, :response_active] + :realign_from_recovery + # Re-query with no attempt cap. Only the Driver's global deadline guarantees termination. + in [:recovery, :response_cat2] + :retry_recovery + in [:cancelling, :response_cancelled] + :complete_cancellation + in [_, :global_deadline_exceeded] + :fail_with_deadline_exceeded + in [:transmission_reading | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery, :user_cancel] + :cancel_session + in [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, :response_rejected] + :fail_with_rejected + # `:starting` fails on `:response_cat2` rather than entering recovery, unlike the + # transmission and finalizing states above: there is no upload to recover to until + # initiation has returned an upload URL. + in [:starting | :cancelling, :response_cat2] | + [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, :response_fatal_bad_response] + :fail_with_bad_response + in [:starting | :transmission_sending | :finalizing_sending_upload | + :finalizing_sending_finalize | :recovery | :cancelling, + :request_retries_exhausted | :request_connection_failed | :request_timeout | + :request_failed_unknown] + :fail_with_request_error + else + :fail_with_unmatched_transition + end + + unless RECIPES.include? recipe + raise InternalError, "Resumable upload internal error: decide selected unknown recipe #{recipe.inspect}" + end + + next_state, instructions = public_send recipe, state, event, config + Decision.new( + from_status: state.status, + shape: shape, + recipe: recipe, + next_state: next_state, + instructions: instructions + ) + end + # rubocop:enable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength + + ## + # @private + # Top-level transition router. Matches [state.status, shape]. + # + # @param state [State] Current state + # @param event [Object] Input event + # @param config [StartUploadConfig, ResumeUploadConfig] Static configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.step state, event, config + decision = decide state, event, config + [decision.next_state, decision.instructions] + end + + ## + # @private + # Initiates the upload session. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig] Session configuration; only an initiating run reaches this recipe + # @return [Array>] Tuple of [next_state, instructions] + def self.start_session state, _event, config + next_state = state.with status: :starting + progress = Progress.new phase: :initiating, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendStart.new( + url: config.initial_url, + headers: config.initial_headers, + body: config.initial_body + ) + ] + [next_state, instructions] + end + + ## + # @private + # Resumes an existing upload session by transitioning to recovery and querying backend offset. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [ResumeUploadConfig] Resume session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.resume_session state, _event, config + next_state = state.with( + status: :recovery, + upload_url: config.upload_url, + chunk_size: config.chunk_size, + offset: 0 + ) + progress = Progress.new( + phase: :initiating, + bytes_uploaded: 0, + total_bytes: config.upload_size + ) + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendQuery.new(url: config.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Processes initiation response and begins data reading. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Initiation response + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.begin_transmission state, event, config + granularity_str = header_value event.headers, "x-goog-upload-chunk-granularity" + granularity = granularity_str&.to_i + chunk_size = resolve_chunk_size config.chunk_size, granularity + upload_url = header_value event.headers, "x-goog-upload-url" + next_state = state.with( + status: :transmission_reading, + upload_url: upload_url, + chunk_granularity: granularity, + chunk_size: chunk_size, + offset: 0, + in_flight_length: 0 + ) + progress = Progress.new phase: :uploading, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::FillBuffer.new(target_bytesize: chunk_size) + ] + [next_state, instructions] + end + + ## + # @private + # Emits instruction to transmit a filled data chunk. + # + # @param state [State] Current state + # @param event [Event::ChunkRead] Chunk read event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.send_chunk state, event, _config + next_state = state.with( + status: :transmission_sending, + in_flight_length: event.bytes_buffered + ) + instructions = [ + Instruction::SendChunk.new( + url: state.upload_url, + offset: state.offset, + length: event.bytes_buffered, + finalize: false + ) + ] + [next_state, instructions] + end + + ## + # @private + # Emits instruction to transmit the final data chunk with finalize. + # + # @param state [State] Current state + # @param event [Event::ChunkRead] Chunk read event with EOF + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.send_upload_finalize state, event, config + next_state = state.with( + status: :finalizing_sending_upload, + in_flight_length: event.bytes_buffered + ) + progress = Progress.new phase: :finalizing, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendChunk.new( + url: state.upload_url, + offset: state.offset, + length: event.bytes_buffered, + finalize: true + ) + ] + [next_state, instructions] + end + + ## + # @private + # Emits instruction to send a zero-length finalize command. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.send_finalize state, _event, config + next_state = state.with( + status: :finalizing_sending_finalize, + in_flight_length: 0 + ) + progress = Progress.new phase: :finalizing, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendFinalize.new(url: state.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Acknowledges transmitted chunk and advances offset. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.ack_chunk state, _event, config + new_offset = state.offset + state.in_flight_length + next_state = state.with( + status: :transmission_reading, + offset: new_offset, + in_flight_length: 0 + ) + progress = Progress.new phase: :uploading, bytes_uploaded: new_offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::RealignBuffer.new(server_offset: new_offset), + Instruction::FillBuffer.new(target_bytesize: state.chunk_size) + ] + [next_state, instructions] + end + + ## + # @private + # Transitions to recovery state to query backend byte offset. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.enter_recovery state, _event, config + next_state = state.with( + status: :recovery, + in_flight_length: 0 + ) + progress = Progress.new phase: :recovering, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendQuery.new(url: state.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Retries offset query during recovery. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.retry_recovery state, _event, _config + next_state = state.with( + status: :recovery, + in_flight_length: 0 + ) + [next_state, [Instruction::SendQuery.new(url: state.upload_url)]] + end + + ## + # @private + # Completes upload when final chunk transmission succeeds. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Final HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.complete_upload_with_data state, event, _config + new_offset = state.offset + state.in_flight_length + next_state = state.with( + status: :success, + offset: new_offset, + in_flight_length: 0 + ) + # `total_bytes` is set from `new_offset` even when `config.upload_size` is nil (see `Progress#total_bytes`). + progress = Progress.new phase: :completed, bytes_uploaded: new_offset, total_bytes: new_offset + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::TerminateSuccess.new(response: event) + ] + [next_state, instructions] + end + + ## + # @private + # Completes upload when standalone finalize succeeds. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Final HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.complete_upload_finalized state, event, _config + next_state = state.with( + status: :success, + in_flight_length: 0 + ) + # `total_bytes` is set from `next_state.offset` even when `config.upload_size` is nil + # (see `Progress#total_bytes`). + progress = Progress.new phase: :completed, bytes_uploaded: next_state.offset, total_bytes: next_state.offset + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::TerminateSuccess.new(response: event) + ] + [next_state, instructions] + end + + ## + # @private + # Realigns buffer and resumes transmission from recovered offset. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Query response containing acknowledged offset + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.realign_from_recovery state, event, config + server_offset_str = header_value event.headers, "x-goog-upload-size-received" + server_offset = server_offset_str.to_i + next_state = state.with( + status: :transmission_reading, + offset: server_offset, + in_flight_length: 0 + ) + progress = Progress.new phase: :uploading, bytes_uploaded: server_offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::RealignBuffer.new(server_offset: server_offset), + Instruction::FillBuffer.new(target_bytesize: state.chunk_size) + ] + [next_state, instructions] + end + + ## + # @private + # Completes session cancellation and emits failure instruction. + # + # @param state [State] Current state + # @param event [Object] Cancellation response event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.complete_cancellation state, event, _config + err = UploadCancelledError.from event + next_state = state.with status: :cancelled, in_flight_length: 0, last_error: err + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Initiates session cancellation request. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.cancel_session state, _event, config + next_state = state.with status: :cancelling + progress = Progress.new phase: :cancelling, bytes_uploaded: next_state.offset, total_bytes: config.upload_size + instructions = [ + Instruction::NotifyProgress.new(progress: progress), + Instruction::SendCancel.new(url: state.upload_url) + ] + [next_state, instructions] + end + + ## + # @private + # Extracts a {ResumeHandle} from current protocol state. + # Completed uploads (`:success`), rejected uploads (`:rejected`), and cancelled uploads + # (`:cancelled`) are finalized and not resumable, returning `nil`. + # + # @param state [State] Protocol state + # @return [ResumeHandle, nil] Resume handle if upload URL is established and resumable, or nil + def self.resume_handle_from state + return nil if state.nil? || state.upload_url.nil? || [:rejected, :cancelled, :success].include?(state.status) + + ResumeHandle.new upload_url: state.upload_url, chunk_size: state.chunk_size + end + + ## + # @private + # Fails upload due to exceeded execution deadline. + # + # @param state [State] Current state + # @param _event [Object] Dispatched event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_deadline_exceeded state, _event, _config + handle = resume_handle_from state + err = DeadlineExceededError.new resume_handle: handle + next_state = state.with( + status: :error, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Fails upload when backend explicitly rejects session. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Rejected HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_rejected state, event, _config + err = UploadRejectedError.from event + next_state = state.with( + status: :rejected, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Fails upload when an unrecoverable HTTP response is encountered. + # + # @param state [State] Current state + # @param event [Event::HttpResponse] Fatal HTTP response + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_bad_response state, event, _config + handle = resume_handle_from state + err = BadResponseError.from event, resume_handle: handle + next_state = state.with( + status: :error, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Fails upload when an unrecoverable network or request error occurs. + # + # @param state [State] Current state + # @param event [Event::RequestFailed] Request failure event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @return [Array>] Tuple of [next_state, instructions] + def self.fail_with_request_error state, event, _config + handle = resume_handle_from state + err = RequestFailedError.from event, resume_handle: handle + next_state = state.with( + status: :error, + in_flight_length: 0, + last_error: err + ) + [next_state, [Instruction::TerminateFailure.new(error: err)]] + end + + ## + # @private + # Raises InvalidTransitionError for unmatched state and event pair. + # + # @param state [State] Current state + # @param event [Object] Dispatched event + # @param _config [StartUploadConfig, ResumeUploadConfig] Session configuration + # @raise [InvalidTransitionError] + def self.fail_with_unmatched_transition state, event, _config + shape = shape_of event + action = STATE_DESCRIPTIONS[state.status] || "processing #{state.status}" + happened = describe_event event, shape + message = "Resumable upload failed while #{action}: #{happened}." + response = event.is_a?(Event::HttpResponse) ? event : nil + handle = resume_handle_from state + raise InvalidTransitionError.new( + message, + state: state.status, + event: event, + response: response, + resume_handle: handle + ) + end + + ## + # @private + # Formats human-readable summary of an event. + # + # @param event [Object] Event instance + # @param shape [Symbol] Event shape symbol + # @return [String] Formatted description + def self.describe_event event, shape + case event + when Event::HttpResponse + upload_status = event.headers["x-goog-upload-status"] || event.headers["X-Goog-Upload-Status"] + status_desc = upload_status ? "'#{upload_status}'" : "missing" + "received an unexpected HTTP #{event.status} response (X-Goog-Upload-Status: #{status_desc})" + when Event::ChunkRead + "received unexpected stream chunk read (#{event.bytes_buffered} bytes, eof: #{event.eof})" + when Event::RequestFailed + "encountered unexpected request failure (#{event.kind}: #{event.message})" + else + "received unexpected event #{shape} (#{event.class.name})" + end + end + + ## + # @private + # Resolves effective chunk size given user specification and backend granularity. + # + # @param user_chunk_size [Integer, nil] Configured chunk size + # @param chunk_granularity [Integer, nil] Backend alignment granularity + # @return [Integer] Effective chunk size in bytes + def self.resolve_chunk_size user_chunk_size, chunk_granularity + base_size = user_chunk_size || DEFAULT_CHUNK_SIZE + return base_size if chunk_granularity.nil? || chunk_granularity <= 0 + return chunk_granularity if base_size <= chunk_granularity + + base_size - (base_size % chunk_granularity) + end + + ## + # @private + # Classifies an HTTP response into a canonical response shape. + # + # @param response [Event::HttpResponse] Response event + # @return [Symbol] Canonical response shape + def self.classify_http_response response + status_header = header_value(response.headers, "x-goog-upload-status")&.downcase + + case status_header + when "active" + response.status == 200 ? :response_active : :response_cat2 + when "final" + response.status == 200 ? :response_final : :response_rejected + when "cancelled" + response.status == 200 ? :response_cancelled : :response_fatal_bad_response + when nil, "" + if FATAL_STATUS_CODES.include? response.status + :response_fatal_bad_response + else + :response_cat2 + end + else + :response_fatal_bad_response + end + end + + ## + # @private + # Case-insensitive header lookup helper. + # + # @param headers [Hash, Object] Headers collection + # @param key [String] Target header key + # @return [String, nil] Header value + def self.header_value headers, key + return nil unless headers.is_a? Hash + return headers[key] if headers.key? key + + target = key.downcase + _, val = headers.find { |k, _| k.to_s.downcase == target } + val + end + + ## + # @private + # Classifies chunk read event by buffer size and EOF flag. + # + # @param event [Event::ChunkRead] Chunk read event + # @return [Symbol] Canonical chunk shape + def self.classify_chunk_read event + if !event.eof + :chunk_read_full + elsif event.bytes_buffered.positive? + :chunk_read_eof_with_data + else + :chunk_read_eof_empty + end + end + + ## + # @private + # Classifies request failure event by failure kind. + # + # @param event [Event::RequestFailed] Request failed event + # @return [Symbol] Canonical failure shape + def self.classify_request_failed event + case event.kind + when :timeout then :request_timeout + when :retries_exhausted then :request_retries_exhausted + when :connection_failed then :request_connection_failed + else :request_failed_unknown + end + end + + ## + # @private + # Classifies raw event class objects. + # + # @param event_class [Class] Event class + # @return [Symbol] Canonical shape + def self.classify_event_class event_class + if event_class == Event::StartUpload + :start_upload + elsif event_class == Event::ResumeUpload + :resume_upload + elsif event_class == Event::Cancel + :user_cancel + elsif event_class == Event::GlobalDeadlineExceeded + :global_deadline_exceeded + else + :unknown + end + end + end + # rubocop:enable Metrics/ModuleLength + end + end +end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/session.rb b/gapic-common/lib/gapic/rest/resumable_upload/session.rb new file mode 100644 index 0000000..ac0e431 --- /dev/null +++ b/gapic-common/lib/gapic/rest/resumable_upload/session.rb @@ -0,0 +1,546 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload/data_types" +require "gapic/rest/resumable_upload/driver" +require "gapic/rest/resumable_upload/errors" + +module Gapic + module Rest + module ResumableUpload + ## + # Coordinates a resumable upload across its lifecycle. + # + # A Session performs exactly one run (`start` or `resume`), never both, never twice. + # + # ### Two-State Model + # 1. **Unbound** (`!bound?`): Fresh session prior to execution. Permitted operations: `start` + # or `resume(...)`. + # 2. **Bound** (`bound?`): Session has executed or bound to an upload URL. Permitted operations: + # none (`start` and `resume` both raise {SessionStateError}). + # + # Calling {#resumable?} reports whether a new session can resume the upload (`!resume_handle.nil?`). + # Completed uploads (`:success`) and rejected uploads are finalized and not resumable + # (`resumable?` returns `false`, `resume_handle` returns `nil`). + # + # ### Execution Model + # + # {#start} and {#resume} are synchronous: they block the calling thread for the entire duration of the + # upload and return only on completion or failure. The `on_progress` callback runs on that same thread. + # + # The remaining readers ({#upload_url}, {#bound?}, {#resume_handle}, {#resumable?}, {#running?}) are + # guarded by an internal mutex and may be called from another thread while a run is in progress. Values + # read mid-run are a best-effort snapshot of a state the upload thread is still advancing. + # + # ### Where Arguments Live + # + # The constructor takes what both run types share: the client stub, the stream, `upload_size`, + # `content_type`, `timeout`, the control- and data-plane retry policies, `on_progress` and `logger`. + # Arguments that belong to one run live on the method performing it — `initial_url`, `initial_body`, + # `initial_headers`, `chunk_size` and `start_retry_policy` on {#start}; `upload_url` and `chunk_size`, + # or a {ResumeHandle}, on {#resume}. + # + # ### Recovering From a Failure + # + # A bound session never runs again, so recovery means constructing a new Session. Errors that carry a + # resume handle include the {HasResumeHandle} mixin, which can be rescued directly to catch all of them: + # + # @example Resuming after a recoverable failure + # begin + # session.start initial_url: url + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # raise unless e.resume_handle + # Session.new(client_stub: client_stub, stream: File.open(path, "rb")) + # .resume(resume_handle: e.resume_handle) + # end + # + # The replacement session needs a stream positioned at byte 0 of the whole object, not at the server's + # acknowledged offset; {#resume} fast-forwards on its own. For an unseekable stream that means opening a + # fresh one, since it cannot be rewound. + # + # ### Defaults + # + # * `chunk_size` defaults to 8 MB, then rounds down to a multiple of any chunk granularity the server + # requires. + # * `timeout` defaults to `upload_size / 1 MB per second` when `upload_size` is known, floored at one + # hour, and to one hour flat when it is not. + # + # ### Retry Policies + # + # Retry behavior is partitioned across three policies: `start_retry_policy` on {#start}, and + # `control_plane_retry_policy` and `data_plane_retry_policy` on {#initialize}. + # + # Passing a {Gapic::Common::RetryPolicy} replaces the corresponding default policy outright. Passing a + # Hash overrides only the keys it names and leaves the remaining defaults — including `retry_codes` and + # any status-header predicates — in place. + # + # All three policies share the same default retry codes and exponential backoff settings: + # + # | Setting | Default | + # |---|---| + # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | + # | `initial_delay` | `1.0` s | + # | `max_delay` | `15.0` s | + # | `multiplier` | `1.3` | + # + # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response + # header is treated: + # + # | Policy | Governs | Missing status header | + # |---|---|---| + # | `start_retry_policy` | session initiation | **Retriable** on any status (incl. `200`), unless fatal | + # | `control_plane_retry_policy` | `query` and `cancel` | No predicate; decided on `retry_codes` alone | + # | `data_plane_retry_policy` | `upload` and `finalize` | **Not** retriable | + # + # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data + # plane treats it as a response it cannot interpret and refuses to replay bytes against it. + # + class Session + # @return [Gapic::Rest::ClientStub] Underlying REST client stub + attr_reader :client_stub + + ## + # Binary input stream to upload. The stream is assumed to be positioned at byte 0 + # (it is not rewound prior to reading) and is not closed after use. + # + # @return [IO] + attr_reader :stream + + # @return [Integer, nil] Total upload bytes if known upfront + attr_reader :upload_size + + # @return [String, nil] MIME type of uploaded media + attr_reader :content_type + + ## + # Total upload timeout in seconds, covering the whole run rather than any single request. When `nil`, + # it resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to + # one hour flat otherwise. Zero and negative values are treated as `nil`. + # + # @return [Numeric, nil] + attr_reader :timeout + + ## + # Retry policy for control commands (query, cancel). A {Gapic::Common::RetryPolicy} replaces the + # default policy outright; a Hash overrides only the settings it names. + # + # @return [Gapic::Common::RetryPolicy, Hash, nil] + attr_reader :control_plane_retry_policy + + ## + # Retry policy for data commands (upload, finalize). A {Gapic::Common::RetryPolicy} replaces the + # default policy outright; a Hash overrides only the settings it names. + # + # @return [Gapic::Common::RetryPolicy, Hash, nil] + attr_reader :data_plane_retry_policy + + ## + # Callback invoked with {Progress} snapshots during upload execution. + # Executed synchronously on the thread running the upload protocol; it must not block. + # Exceptions raised inside the callback immediately abort the upload session and + # propagate out of {#start} or {#resume}. + # + # @return [Proc, nil] + attr_reader :on_progress + + # @return [Logger, nil] Logger instance + attr_reader :logger + + ## + # Initializes a new Resumable Upload Session. + # + # The constructor takes only what both run types share. Arguments specific to a single run live on + # the method that performs it: initiation details on {#start}, the upload URL and chunk size on + # {#resume}. + # + # @param client_stub [Gapic::Rest::ClientStub] Underlying REST client stub + # @param stream [IO] Binary input stream to upload. Precondition: assumed to be positioned at byte 0 + # (not rewound prior to reading) and not closed after use. + # @param upload_size [Integer, nil] Total upload bytes if known upfront + # @param content_type [String, nil] MIME type of uploaded media + # @param timeout [Numeric, nil] Total upload timeout in seconds covering the whole run. When `nil`, + # resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to + # one hour flat otherwise. + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control + # commands (`query` and `cancel`). See the "Retry Policies" section in the class documentation. + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data + # commands (`upload` and `finalize`). See the "Retry Policies" section in the class documentation. + # @param on_progress [Proc, nil] Progress callback invoked as `->(progress)` with a {Progress} instance. + # Executed synchronously on the upload protocol thread; it must not block. + # Exceptions raised inside the callback abort the session and propagate out of {#start} or {#resume}. + # @param logger [Logger, nil] Logger instance + # + def initialize client_stub:, + stream:, + upload_size: nil, + content_type: nil, + timeout: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil, + logger: nil + @client_stub = client_stub + @stream = stream + @upload_size = upload_size + @content_type = content_type + @timeout = timeout + @control_plane_retry_policy = control_plane_retry_policy + @data_plane_retry_policy = data_plane_retry_policy + @on_progress = on_progress + @logger = logger + + @mutex = Mutex.new + @running = false + @executed = false + @upload_url = nil + @last_driver = nil + end + + ## + # Returns the raw upload session URL if established. + # + # @return [String, nil] + def upload_url + @mutex.synchronize { upload_url_internal } + end + + ## + # Returns whether the session is bound to a server-side upload. + # + # @return [Boolean] + def bound? + @mutex.synchronize { bound_internal? } + end + + ## + # Returns the current {ResumeHandle} if the session is alive and resumable. + # Completed uploads are not resumable (returns nil). Rejected uploads and + # cancelled uploads are also finalized and not resumable, returning nil. + # + # @return [ResumeHandle, nil] + def resume_handle + @mutex.synchronize { resume_handle_internal } + end + + ## + # Returns whether a new session can resume the upload. + # Completed uploads are not resumable (returns false). Rejected uploads and + # cancelled uploads are also finalized and not resumable (returns false). + # + # @return [Boolean] + def resumable? + @mutex.synchronize { !resume_handle_internal.nil? } + end + + ## + # Returns whether a run is currently executing. + # + # @return [Boolean] + def running? + @mutex.synchronize { @running } + end + + ## + # Starts a new upload session on the server. + # + # A session performs exactly one run (`start` or `resume`). Calling `start` on an already-bound + # or executed session raises {SessionStateError}. Precondition: the stream is assumed to be + # positioned at byte 0 (the session does not rewind it before reading) and is not closed after use. + # + # Blocks the calling thread until the upload completes or fails. + # + # @example Uploading a file with progress reporting + # session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: File.open("movie.mp4", "rb"), + # upload_size: File.size("movie.mp4"), + # content_type: "video/mp4", + # on_progress: ->(progress) { puts "#{progress.phase}: #{progress.bytes_uploaded} bytes" } + # ) + # response = session.start initial_url: "https://example.googleapis.com/upload/v1/media" + # + # @param initial_url [String] Initial endpoint URI for session initiation + # @param initial_body [String, nil] Request payload for session initiation + # @param initial_headers [Hash] Additional headers for the initiation request. + # The five reserved protocol headers (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, + # `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, + # `X-Goog-Upload-Header-Content-Length`) are rejected with an `ArgumentError` in any casing — + # they carry protocol mechanics the session owns. Use the constructor's `content_type` and + # `upload_size` to shape the media descriptors. Pass-through headers such as + # `X-Goog-Upload-Header-Content-Disposition` are permitted. + # @param chunk_size [Integer, nil] Requested chunk size in bytes, defaulting to 8 MB. The effective + # size is rounded down to a multiple of any chunk granularity the server requires, or raised to that + # granularity if it exceeds the requested size. + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for the initiation + # request (`start`). See the "Retry Policies" section in the class documentation. + # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the + # response carried no body), typically the JSON resource the backend created that the caller + # parses. A client stub carrying response-decoding middleware is outside the contract. + # @raise [ArgumentError] If `initial_url` is missing or blank, if `initial_headers` sets a + # reserved protocol header, or if a retry policy argument is neither a + # {Gapic::Common::RetryPolicy}, a Hash, nor `nil` + # @raise [SessionStateError] If already bound/executed or if a run is currently in progress + # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs + # @raise [DeadlineExceededError] If the global upload timeout is exceeded + # @raise [BadResponseError] If an unexpected or malformed HTTP response is received + # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream + # @raise [StreamMismatchError] If stream content or length does not match protocol expectations + # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state + # @raise [UploadRejectedError] If the server explicitly rejects the upload session + # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug + # in this library rather than a caller or server error + def start initial_url:, + initial_body: nil, + initial_headers: {}, + chunk_size: nil, + start_retry_policy: nil + config = build_start_config initial_url: initial_url, + initial_body: initial_body, + initial_headers: initial_headers, + chunk_size: chunk_size, + start_retry_policy: start_retry_policy + + driver = nil + @mutex.synchronize do + raise SessionStateError, "A run is already in progress for this session" if @running + raise SessionStateError, "Session has already executed a run" if bound_internal? + + driver = Driver.new client_stub: @client_stub, config: config, logger: @logger + @executed = true + @running = true + end + + execute_run driver + end + + ## + # Resumes an upload session using one of two explicit keyword forms: + # 1. `resume(upload_url:, chunk_size:)`: Resumes with explicit URL and chunk size. + # 2. `resume(resume_handle:)`: Resumes via {ResumeHandle}. + # + # A session performs exactly one run (`start` or `resume`). Resuming must be executed on a + # fresh, unexecuted session. Blocks the calling thread until the upload completes or fails. + # + # A resumed run targets an upload the server has already created, so it takes no initiation + # arguments; everything it needs beyond the constructor is on this method. + # + # ### Chunk size + # + # A chunk size must be given explicitly because the server reports chunk granularity during + # initiation, which a resumed run skips. {ResumeHandle} carries the effective value from the original + # run for exactly this reason. + # + # ### Stream position + # + # The stream must be positioned at byte 0 of the whole object, not at the server's acknowledged + # offset, and is not closed after use. The Driver fast-forwards on its own, by seeking on seekable + # streams or by reading and discarding on unseekable ones. An unseekable stream therefore has to be + # freshly opened rather than rewound. + # + # A completed upload is finalized: {#resume_handle} returns `nil` and {#resumable?} returns + # `false`, so there is no handle to resume from. Calling `#resume` on the session that completed + # the run raises {SessionStateError}, as it would after any run. Resuming a *fresh* session + # against a finalized `upload_url` is undefined behavior: it queries the server and might return + # the response body or raise an error, depending on the server response. + # + # @example Resuming from a handle persisted by an earlier process + # handle = Gapic::Rest::ResumableUpload::ResumeHandle.new( + # upload_url: row[:upload_url], + # chunk_size: row[:chunk_size] + # ) + # session = Gapic::Rest::ResumableUpload::Session.new( + # client_stub: client_stub, + # stream: File.open("movie.mp4", "rb"), + # upload_size: File.size("movie.mp4") + # ) + # response = session.resume resume_handle: handle + # + # @param upload_url [String, nil] Explicit upload URL + # @param chunk_size [Integer, nil] Explicit chunk size + # @param resume_handle [ResumeHandle, nil] Explicit resume handle + # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the + # response carried no body), typically the JSON resource the backend created that the caller + # parses. A client stub carrying response-decoding middleware is outside the contract. + # @raise [ArgumentError] If argument shape is invalid, target upload is missing, or stream.pos != 0 + # @raise [SessionStateError] If already bound/executed or if a run is currently in progress + # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs + # @raise [DeadlineExceededError] If the global upload timeout is exceeded + # @raise [BadResponseError] If an unexpected or malformed HTTP response is received + # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream + # @raise [StreamMismatchError] If stream content or length does not match the resumed upload + # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state + # @raise [UploadRejectedError] If the server explicitly rejects the upload session + # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug + # in this library rather than a caller or server error + def resume upload_url: nil, + chunk_size: nil, + resume_handle: nil + target_url, target_chunk_size = resolve_resume_args( + upload_url: upload_url, + chunk_size: chunk_size, + resume_handle: resume_handle + ) + + driver = nil + @mutex.synchronize do + raise SessionStateError, "A run is already in progress for this session" if @running + raise SessionStateError, "Session has already executed a run" if bound_internal? + + if @stream.respond_to?(:pos) && !@stream.pos.zero? + raise ArgumentError, "Stream must be positioned at byte 0 to resume an upload (got pos #{@stream.pos})" + end + + config = build_resume_config target_url, target_chunk_size + driver = Driver.new client_stub: @client_stub, config: config, logger: @logger + @executed = true + @running = true + @upload_url = target_url + end + + execute_run driver + end + + private + + ## + # @private + # Returns the established upload URL without locking. + # + # @return [String, nil] + def upload_url_internal + @upload_url || @last_driver&.upload_url + end + + ## + # @private + # Returns whether the session is bound without locking. + # + # @return [Boolean] + def bound_internal? + @executed || !upload_url_internal.nil? + end + + ## + # @private + # Returns the current resume handle from the driver without locking. + # + # @return [ResumeHandle, nil] + def resume_handle_internal + @last_driver&.resume_handle + end + + ## + # @private + # Returns the configuration members shared by both run types, mirroring + # {ResumableUpload::COMMON_MEMBERS}. + # + # @return [Hash{Symbol=>Object}] + def common_config_args + { + stream: @stream, + upload_size: @upload_size, + content_type: @content_type, + timeout: @timeout, + control_plane_retry_policy: @control_plane_retry_policy, + data_plane_retry_policy: @data_plane_retry_policy, + on_progress: @on_progress + } + end + + ## + # @private + # Builds configuration for a new upload session. + # + # @param initial_url [String] Initial endpoint URI for session initiation + # @param initial_body [String, nil] Request payload for session initiation + # @param initial_headers [Hash, nil] Additional headers for initiation + # @param chunk_size [Integer, nil] Requested chunk size in bytes + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Initiation retry policy + # @return [StartUploadConfig] + def build_start_config initial_url:, initial_body:, initial_headers:, chunk_size:, start_retry_policy: + StartUploadConfig.new( + initial_url: initial_url, + initial_body: initial_body, + initial_headers: initial_headers || {}, + chunk_size: chunk_size, + start_retry_policy: start_retry_policy, + **common_config_args + ) + end + + ## + # @private + # Builds configuration for resuming an upload session. + # + # @param target_url [String] Target upload session URL + # @param target_chunk_size [Integer] Effective chunk size in bytes + # @return [ResumeUploadConfig] + def build_resume_config target_url, target_chunk_size + ResumeUploadConfig.new( + upload_url: target_url, + chunk_size: target_chunk_size, + **common_config_args + ) + end + + ## + # @private + # Validates and extracts target upload URL and chunk size from resume keyword arguments. + # + # @param upload_url [String, nil] Explicit upload URL + # @param chunk_size [Integer, nil] Explicit chunk size + # @param resume_handle [ResumeHandle, nil] Explicit resume handle + # @return [Array] Tuple of [upload_url, chunk_size] + # @raise [ArgumentError] If arguments are missing or mutually exclusive + def resolve_resume_args upload_url:, chunk_size:, resume_handle: + if resume_handle + raise ArgumentError, "Cannot pass both resume_handle and upload_url/chunk_size" if upload_url || chunk_size + [resume_handle.upload_url, resume_handle.chunk_size] + elsif upload_url + raise ArgumentError, "Must provide chunk_size with upload_url" if chunk_size.nil? + [upload_url, chunk_size] + elsif chunk_size + raise ArgumentError, "Cannot pass chunk_size without upload_url" + else + raise ArgumentError, "Must provide either resume_handle or upload_url and chunk_size" + end + end + + ## + # @private + # Executes the driver run and records the final upload URL and state. + # + # @param driver [Driver] Driver instance to run + # @return [String, nil] Final response body upon completion + def execute_run driver + @mutex.synchronize { @last_driver = driver } + result = driver.run + @mutex.synchronize do + @upload_url ||= driver.upload_url + @running = false + end + result + rescue StandardError + @mutex.synchronize do + @upload_url ||= driver.upload_url + @running = false + end + raise + end + end + end + end +end diff --git a/gapic-common/test/gapic/rest/client_stub_test.rb b/gapic-common/test/gapic/rest/client_stub_test.rb index f60c9aa..f6c2420 100644 --- a/gapic-common/test/gapic/rest/client_stub_test.rb +++ b/gapic-common/test/gapic/rest/client_stub_test.rb @@ -173,4 +173,45 @@ def test_universe_domain_credentials_mismatch credentials: creds end end + + def test_log_request_retains_valid_utf8_under_1kib + recording = RecordingLogger.new + client_stub = ::Gapic::Rest::ClientStub.new endpoint: "google.example.com", + credentials: :dummy_credentials, + logger: recording + payload = '{"hello":"world"}' + client_stub.send :log_request, "MyMethod", "req-1", 1, payload, { "x-test" => "val" } + + debug_entry = recording.entries.find { |e| e.severity == Logger::DEBUG } + refute_nil debug_entry + assert_equal payload, debug_entry.message.fields["request"] + end + + def test_log_request_abridges_body_over_1kib + recording = RecordingLogger.new + client_stub = ::Gapic::Rest::ClientStub.new endpoint: "google.example.com", + credentials: :dummy_credentials, + logger: recording + payload = "A" * 1025 + expected_prefix = ("A" * 32).unpack1 "H*" + client_stub.send :log_request, "MyMethod", "req-1", 1, payload, {} + + debug_entry = recording.entries.find { |e| e.severity == Logger::DEBUG } + refute_nil debug_entry + assert_equal "<1025 bytes, first 32: #{expected_prefix}>", debug_entry.message.fields["request"] + end + + def test_log_request_abridges_invalid_utf8_body + recording = RecordingLogger.new + client_stub = ::Gapic::Rest::ClientStub.new endpoint: "google.example.com", + credentials: :dummy_credentials, + logger: recording + payload = "\xFF\xFE\x00\x01binary".b + expected_prefix = payload.unpack1 "H*" + client_stub.send :log_request, "MyMethod", "req-1", 1, payload, {} + + debug_entry = recording.entries.find { |e| e.severity == Logger::DEBUG } + refute_nil debug_entry + assert_equal "<10 bytes, first 32: #{expected_prefix}>", debug_entry.message.fields["request"] + end end diff --git a/gapic-common/test/gapic/rest/error_test.rb b/gapic-common/test/gapic/rest/error_test.rb index 6593db1..96c16d9 100644 --- a/gapic-common/test/gapic/rest/error_test.rb +++ b/gapic-common/test/gapic/rest/error_test.rb @@ -323,4 +323,17 @@ def test_surface_absent_details assert_nil gapic_err.details end + + def test_rest_error_prefix_constant + assert_equal "An error has occurred when making a REST request", ::Gapic::Rest::Error::REST_ERROR_PREFIX + + faraday_err = OpenStruct.new( + message: "raw", + response_body: JSON.dump({ "error" => { "message" => "Quota exceeded", "code" => 429 } }), + response_headers: {}, + response_status: 429 + ) + gapic_err = ::Gapic::Rest::Error.wrap_faraday_error faraday_err + assert_equal "#{::Gapic::Rest::Error::REST_ERROR_PREFIX}: Quota exceeded", gapic_err.message + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/core_test.rb b/gapic-common/test/gapic/rest/resumable_upload/core_test.rb new file mode 100644 index 0000000..7a7a205 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/core_test.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +class CoreTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("test content"), + upload_size: 2048, + chunk_size: 1024 + ) + @core = Core.new @config + end + + def test_initial_state + state = @core.state + assert_equal :initializing, state.status + assert_nil state.upload_url + assert_equal 0, state.offset + assert_equal 1024, state.chunk_size + assert_nil state.chunk_granularity + assert_equal 0, state.in_flight_length + assert_nil state.last_error + assert_nil @core.last_decision + end + + def test_dispatch_updates_state_and_returns_instructions + instructions = @core.dispatch Event::StartUpload.new + assert_equal :starting, @core.state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal :initiating, instructions[0].progress.phase + assert_instance_of Instruction::SendStart, instructions[1] + assert_instance_of Decision, @core.last_decision + assert_equal :initializing, @core.last_decision.from_status + assert_equal :start_upload, @core.last_decision.shape + assert_equal :start_session, @core.last_decision.recipe + assert_equal @core.state, @core.last_decision.next_state + assert_equal instructions, @core.last_decision.instructions + + resp = Event::HttpResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Chunk-Granularity" => "512", + "X-Goog-Upload-Status" => "active" + } + ) + instructions = @core.dispatch resp + assert_equal :transmission_reading, @core.state.status + assert_equal "https://example.com/session/1", @core.state.upload_url + assert_equal 512, @core.state.chunk_granularity + assert_equal 1024, @core.state.chunk_size + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal :uploading, instructions[0].progress.phase + assert_instance_of Instruction::FillBuffer, instructions[1] + assert_equal 1024, instructions[1].target_bytesize + assert_instance_of Decision, @core.last_decision + assert_equal :starting, @core.last_decision.from_status + assert_equal :response_active, @core.last_decision.shape + assert_equal :begin_transmission, @core.last_decision.recipe + assert_equal @core.state, @core.last_decision.next_state + assert_equal instructions, @core.last_decision.instructions + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb b/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb new file mode 100644 index 0000000..e78d849 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/data_types_test.rb @@ -0,0 +1,231 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for data types in resumable upload. +# +class DataTypesTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def test_start_upload_config_defaults + stream = StringIO.new "content" + config = StartUploadConfig.new( + initial_url: "https://example.com", + stream: stream + ) + + assert_equal "https://example.com", config.initial_url + assert_same stream, config.stream + assert_nil config.initial_body + assert_equal({}, config.initial_headers) + assert_nil config.upload_size + assert_nil config.chunk_size + assert_nil config.content_type + assert_nil config.timeout + assert_nil config.start_retry_policy + assert_nil config.control_plane_retry_policy + assert_nil config.data_plane_retry_policy + assert_nil config.on_progress + end + + def test_start_upload_config_validations + stream = StringIO.new "content" + + assert_raises ArgumentError do + StartUploadConfig.new initial_url: nil, stream: stream + end + + assert_raises ArgumentError do + StartUploadConfig.new initial_url: " ", stream: stream + end + + assert_raises ArgumentError do + StartUploadConfig.new initial_url: "https://example.com", stream: nil + end + end + + def test_start_upload_config_rejects_reserved_initial_headers + stream = StringIO.new "content" + reserved = ["X-Goog-Upload-Command", "x-goog-upload-command", "X-GOOG-UPLOAD-COMMAND", + "X-Goog-Upload-Protocol", "x-goog-upload-offset", + "X-Goog-Upload-Header-Content-Type", "x-goog-upload-header-content-length"] + + reserved.each do |header| + error = assert_raises ArgumentError do + StartUploadConfig.new initial_url: "https://example.com", stream: stream, initial_headers: { header => "x" } + end + assert_match(/must not set protocol header/, error.message) + assert_includes error.message, header + end + end + + def test_reserved_initial_headers_constant_is_lowercase_and_duplicate_free + assert_equal RESERVED_INITIAL_HEADERS.map(&:downcase), RESERVED_INITIAL_HEADERS + assert_equal RESERVED_INITIAL_HEADERS.uniq, RESERVED_INITIAL_HEADERS + assert_predicate RESERVED_INITIAL_HEADERS, :frozen? + end + + def test_start_upload_config_allows_caller_owned_initial_headers + stream = StringIO.new "content" + headers = { + "X-Goog-Test-Scenario" => "chunk_granularity", + "X-Goog-Upload-Header-Content-Disposition" => 'attachment; filename="movie.mp4"', + "X-Custom" => "value" + } + + config = StartUploadConfig.new initial_url: "https://example.com", stream: stream, initial_headers: headers + + assert_equal headers, config.initial_headers + end + + def test_state_defaults_and_with + state = State.new + + assert_equal :initializing, state.status + assert_nil state.upload_url + assert_equal 0, state.offset + assert_equal 8_388_608, state.chunk_size + assert_nil state.chunk_granularity + assert_equal 0, state.in_flight_length + assert_nil state.last_error + + modified = state.with status: :starting, upload_url: "https://example.com/upload" + assert_equal :starting, modified.status + assert_equal "https://example.com/upload", modified.upload_url + assert_equal :initializing, state.status + end + + def test_event_instantiation + start_event = Event::StartUpload.new + assert_instance_of Event::StartUpload, start_event + + chunk = Event::ChunkRead.new bytes_buffered: 100, eof: true + assert_equal 100, chunk.bytes_buffered + assert chunk.eof + + http = Event::HttpResponse.new status: 200, headers: { "a" => "b" }, body: "body" + assert_equal 200, http.status + assert_equal({ "a" => "b" }, http.headers) + assert_equal "body", http.body + + req_fail = Event::RequestFailed.new kind: :connection_failed, message: "err" + assert_equal :connection_failed, req_fail.kind + assert_equal "err", req_fail.message + end + + def test_instruction_instantiation + start = Instruction::SendStart.new url: "https://example.com" + assert_equal "https://example.com", start.url + assert_equal({}, start.headers) + assert_nil start.body + + chunk = Instruction::SendChunk.new url: "https://example.com", offset: 0, length: 100 + assert_equal 0, chunk.offset + assert_equal 100, chunk.length + refute chunk.finalize + + realign = Instruction::RealignBuffer.new server_offset: 500 + assert_equal 500, realign.server_offset + end + + def test_progress_instantiation + Progress::PHASES.each do |phase| + progress = Progress.new phase: phase, bytes_uploaded: 512, total_bytes: 2048 + assert_equal phase, progress.phase + assert_equal 512, progress.bytes_uploaded + assert_equal 2048, progress.total_bytes + end + + progress_unknown = Progress.new phase: :uploading, bytes_uploaded: 1024 + assert_equal :uploading, progress_unknown.phase + assert_equal 1024, progress_unknown.bytes_uploaded + assert_nil progress_unknown.total_bytes + + assert_raises ArgumentError do + Progress.new bytes_uploaded: 512, total_bytes: 2048 + end + + assert_raises ArgumentError do + Progress.new phase: :invalid_phase, bytes_uploaded: 512, total_bytes: 2048 + end + end + + def test_resume_handle_instantiation + handle = ResumeHandle.new upload_url: "https://upload.example.com/session123", chunk_size: 1_048_576 + assert_equal "https://upload.example.com/session123", handle.upload_url + assert_equal 1_048_576, handle.chunk_size + + assert_raises ArgumentError do + ResumeHandle.new upload_url: "https://upload.example.com/session123" + end + + assert_raises NoMethodError do + handle.upload_url = "https://mutated.com" + end + end + + def test_resume_upload_config_defaults + stream = StringIO.new "content" + config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session1", + chunk_size: 1024, + stream: stream + ) + + assert_equal "https://upload.example.com/session1", config.upload_url + assert_equal 1024, config.chunk_size + assert_same stream, config.stream + assert_nil config.upload_size + assert_nil config.content_type + assert_nil config.timeout + refute_respond_to config, :start_retry_policy + assert_nil config.control_plane_retry_policy + assert_nil config.data_plane_retry_policy + assert_nil config.on_progress + end + + def test_resume_upload_config_validations + stream = StringIO.new "content" + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: nil, chunk_size: 1024, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: " ", chunk_size: 1024, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: 0, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: -10, stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: "1024", stream: stream + end + + assert_raises ArgumentError do + ResumeUploadConfig.new upload_url: "https://example.com", chunk_size: 1024, stream: nil + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb new file mode 100644 index 0000000..14e1e80 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver/abridge_test.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" + +## +# Unit tests for Gapic::Rest::ResumableUpload::Driver::Abridge. +# +class AbridgeTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def test_bytes_hex_encodes_short_payloads + assert_nil Driver::Abridge.bytes(nil) + + short = "hello" + assert_equal short.unpack1("H*"), Driver::Abridge.bytes(short) + + boundary = "A" * 63 + assert_equal boundary.unpack1("H*"), Driver::Abridge.bytes(boundary) + end + + def test_bytes_abridges_and_hex_encodes_large_payloads + large = "A" * 100 + expected_prefix = ("A" * 32).unpack1 "H*" + assert_equal "#{expected_prefix}... <100 bytes>", Driver::Abridge.bytes(large) + end + + def test_error_body_truncates_at_512_bytes_and_scrubs_invalid_utf8 + assert_nil Driver::Abridge.error_body(nil) + + long_err = "E" * 600 + assert_equal 512, Driver::Abridge.error_body(long_err).bytesize + + invalid_utf8 = "error \xFF\xFE message".b + scrubbed = Driver::Abridge.error_body invalid_utf8 + assert scrubbed.valid_encoding? + assert_includes scrubbed, "error " + assert_includes scrubbed, " message" + end + + def test_url_elides_query_parameter_values + assert_nil Driver::Abridge.url(nil) + + url = "https://storage.googleapis.com/upload/storage/v1/b/bucket/o?uploadType=resumable&sid=SECRET123" + assert_equal "https://storage.googleapis.com/upload/storage/v1/b/bucket/o?uploadType=<...>&sid=<...>", + Driver::Abridge.url(url) + end + + def test_headers_retains_x_goog_upload_and_redacts_others + headers = { + "X-Goog-Upload-Command" => "upload, finalize", + "X-Goog-Upload-Offset" => "0", + "Authorization" => "Bearer SECRET123", + "Content-Type" => "application/octet-stream" + } + + abridged = Driver::Abridge.headers headers + assert_equal "upload, finalize", abridged["X-Goog-Upload-Command"] + assert_equal "0", abridged["X-Goog-Upload-Offset"] + assert_equal "<...>", abridged["Authorization"] + assert_equal "<...>", abridged["Content-Type"] + end + + def test_instructions_summarizes_without_bodies + instructions = [ + Instruction::SendStart.new(url: "https://example.com/upload?key=SECRET", headers: {}, body: "secret_body"), + Instruction::SendChunk.new(url: "https://example.com/session?id=123", offset: 0, length: 64, finalize: true), + Instruction::NotifyProgress.new(progress: Progress.new(phase: :uploading, bytes_uploaded: 64, total_bytes: 1024)) + ] + + summary = Driver::Abridge.instructions instructions + assert_equal "SendStart", summary[0]["type"] + assert_equal "https://example.com/upload?key=<...>", summary[0]["url"] + refute summary[0].key?("body") + + assert_equal "SendChunk", summary[1]["type"] + assert_equal "https://example.com/session?id=<...>", summary[1]["url"] + assert_equal 0, summary[1]["offset"] + assert_equal 64, summary[1]["length"] + assert_equal true, summary[1]["finalize"] + + assert_equal "NotifyProgress", summary[2]["type"] + assert_equal "uploading", summary[2]["phase"] + assert_equal 64, summary[2]["bytesUploaded"] + assert_equal 1024, summary[2]["totalBytes"] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb new file mode 100644 index 0000000..49a651b --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver/upload_log_test.rb @@ -0,0 +1,216 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" + +## +# Unit tests for Gapic::Rest::ResumableUpload::Driver::UploadLog. +# +class UploadLogTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: @recording, service: "ResumableUpload" + @upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-id" + @config = StartUploadConfig.new initial_url: "https://example.com/upload", + initial_body: nil, + initial_headers: {}, + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 256, + content_type: "text/plain", + timeout: nil, + start_retry_policy: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + on_progress: nil + end + + def test_decision_logs_debug_with_fields_from_rules_decide + decision = Rules.decide State.new(status: :initializing), Event::StartUpload.new, @config + + @upload_log.decision decision + + assert_equal 1, @recording.entries.size + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "test-upload-id", fields["uploadId"] + assert_equal "initializing", fields["fromStatus"] + assert_equal "start_upload", fields["shape"] + assert_equal "start_session", fields["recipe"] + assert_equal "starting", fields["toStatus"] + assert_equal 0, fields["offset"] + assert_equal 0, fields["inFlightLength"] + assert_equal [ + { "type" => "NotifyProgress", "phase" => "initiating", "bytesUploaded" => 0, "totalBytes" => 1024 }, + { "type" => "SendStart", "url" => "https://example.com/upload" } + ], fields["instructions"] + end + + def test_lifecycle_start_session_logs_info + decision = Rules.decide State.new(status: :initializing), Event::StartUpload.new, @config + + @upload_log.lifecycle decision, @config + + entry = @recording.entries.first + assert_equal Logger::INFO, entry.severity + fields = entry.message.fields + assert_equal "start_session", fields["recipe"] + assert_equal 1024, fields["uploadSize"] + assert_equal 256, fields["requestedChunkSize"] + end + + def test_lifecycle_send_chunk_logs_debug + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 0, chunk_size: 256 + event = Event::ChunkRead.new bytes_buffered: 256, eof: false + decision = Rules.decide state, event, @config + + @upload_log.lifecycle decision, @config + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "send_chunk", fields["recipe"] + assert_equal 0, fields["offset"] + assert_equal 256, fields["inFlightLength"] + end + + def test_lifecycle_terminal_failure_logs_warn + state = State.new status: :starting + event = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Forbidden" + decision = Rules.decide state, event, @config + + @upload_log.lifecycle decision, @config + + entry = @recording.entries.first + assert_equal Logger::WARN, entry.severity + fields = entry.message.fields + assert_equal "fail_with_rejected", fields["recipe"] + assert_includes fields["error"], "Forbidden" + end + + def test_lifecycle_silent_recipes_emit_no_logs + state = State.new status: :transmission_sending + active = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "active" } + decision = Rules.decide state, active, @config + assert_equal :ack_chunk, decision.recipe + + @upload_log.lifecycle decision, @config + + assert_empty @recording.entries + end + + def test_lifecycle_table_matches_rules_recipes + lifecycle_keys = Driver::UploadLog::LIFECYCLE.keys + silent_keys = Driver::UploadLog::SILENT_RECIPES + all_upload_log_recipes = lifecycle_keys + silent_keys + + assert_empty Rules::RECIPES - all_upload_log_recipes, + "Rules recipes not covered by UploadLog::LIFECYCLE or SILENT_RECIPES" + assert_empty all_upload_log_recipes - Rules::RECIPES, + "Extra recipes in UploadLog::LIFECYCLE or SILENT_RECIPES not in Rules::RECIPES" + assert_empty lifecycle_keys & silent_keys, + "Recipes present in both UploadLog::LIFECYCLE and SILENT_RECIPES" + end + + def test_wire_send_logs_debug_with_start_attempt_and_hex_body + @upload_log.wire_send method: "POST", + url: "https://example.com/session?key=SECRET", + headers: { + "X-Goog-Upload-Command" => "upload", + "X-Goog-Upload-Offset" => "256", + "Authorization" => "Bearer SECRET" + }, + start_attempt: 2, + body_size: 4, + body: "test" + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "POST", fields["method"] + assert_equal "upload", fields["command"] + assert_equal 256, fields["offset"] + assert_equal "https://example.com/session?key=<...>", fields["url"] + assert_equal 2, fields["startAttempt"] + assert_equal 4, fields["bodySize"] + assert_equal "74657374", fields["body"] + assert_equal "<...>", fields["headers"]["Authorization"] + assert_equal "upload", fields["headers"]["X-Goog-Upload-Command"] + end + + def test_wire_receive_logs_debug + event = Event::HttpResponse.new status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "256", + "X-Goog-Upload-Chunk-Granularity" => "256" + }, + body: "ok" + + @upload_log.wire_receive event + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal 200, fields["status"] + assert_equal "active", fields["uploadStatus"] + assert_equal 256, fields["sizeReceived"] + assert_equal 256, fields["granularity"] + assert_equal "6f6b", fields["body"] + end + + def test_wire_failure_logs_debug + event = Event::RequestFailed.new kind: :timeout, message: "timed out" + + @upload_log.wire_failure event + + entry = @recording.entries.first + assert_equal Logger::DEBUG, entry.severity + fields = entry.message.fields + assert_equal "timeout", fields["kind"] + end + + def test_buffer_realign_logs_warn_and_debug_on_unseekable_rewind + @upload_log.buffer_realign "rewind", server_offset: 0, current_offset: 256, unseekable: true + + assert_equal 2, @recording.entries.size + warn_entry, debug_entry = @recording.entries + assert_equal Logger::WARN, warn_entry.severity + assert_equal "rewind", warn_entry.message.fields["action"] + assert_equal 0, warn_entry.message.fields["serverOffset"] + assert_equal 256, warn_entry.message.fields["currentOffset"] + + assert_equal Logger::DEBUG, debug_entry.severity + end + + def test_unmatched_transition_logs_warn + state = State.new status: :initializing + event = Event::HttpResponse.new status: 200, headers: {}, body: "" + err = InvalidTransitionError.new "no transition" + + @upload_log.unmatched_transition state, event, err + + entry = @recording.entries.first + assert_equal Logger::WARN, entry.severity + fields = entry.message.fields + assert_equal "initializing", fields["status"] + assert_equal "no transition", fields["error"] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb new file mode 100644 index 0000000..a7e110d --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_buffer_test.rb @@ -0,0 +1,378 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for Driver stream reading and buffer realignment mechanics. +# +class DriverBufferTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + ## + # Stream double that returns at most max_chunk_size bytes per read call. + # + class ChunkedStream + def initialize data, max_chunk_size + @io = StringIO.new data + @max_chunk_size = max_chunk_size + end + + def read length = nil + return @io.read if length.nil? + + actual_length = [length, @max_chunk_size].min + @io.read actual_length + end + + def seek offset + @io.seek offset + end + + def pos + @io.pos + end + end + + ## + # Stream double that intentionally does not implement #seek. + # + class UnseekableStream + def initialize data + @io = StringIO.new data + end + + def read length = nil + @io.read length + end + + def pos + @io.pos + end + end + + def setup + @dummy_client = Object.new + end + + # ============================================================================ + # execute_fill_buffer tests + # ============================================================================ + + def test_fill_buffer_short_reads_accumulates_until_target + data = "abcdefghijklmnopqrstuvwxyz" * 4 # 104 bytes + stream = ChunkedStream.new data, 20 + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 100, event.bytes_buffered + refute event.eof + assert_equal data.byteslice(0, 100), driver.instance_variable_get(:@buffer) + end + + def test_fill_buffer_eof_exactly_at_target_boundary + data = "0123456789" * 10 # exactly 100 bytes + stream = StringIO.new data + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 100, event.bytes_buffered + # eof stays false until a subsequent read attempts to read past boundary + refute event.eof + assert_equal 100, stream.pos + + # Subsequent fill detects EOF + second_event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 101) + assert_equal 100, second_event.bytes_buffered + assert second_event.eof + end + + def test_fill_buffer_eof_mid_fill + data = "short data of 45 bytes......................." # 45 bytes + stream = StringIO.new data + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 45, event.bytes_buffered + assert event.eof + assert_equal data, driver.instance_variable_get(:@buffer) + end + + def test_fill_buffer_empty_stream + stream = StringIO.new "" + driver = build_driver stream: stream + + event = driver.send :execute_fill_buffer, Instruction::FillBuffer.new(target_bytesize: 100) + + assert_instance_of Event::ChunkRead, event + assert_equal 0, event.bytes_buffered + assert event.eof + assert_equal "".b, driver.instance_variable_get(:@buffer) + end + + # ============================================================================ + # execute_realign_buffer tests: trim within buffer + # ============================================================================ + + def test_realign_buffer_trim_exact_beginning + driver = build_driver stream: StringIO.new + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1000) + + assert_equal 1000, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "0123456789".b, driver.instance_variable_get(:@buffer) + end + + def test_realign_buffer_trim_middle + driver = build_driver stream: StringIO.new + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1004) + + assert_equal 1004, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "456789".b, driver.instance_variable_get(:@buffer) + end + + def test_realign_buffer_trim_exact_end + driver = build_driver stream: StringIO.new + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1010) + + assert_equal 1010, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + end + + # ============================================================================ + # execute_realign_buffer tests: rewind stream + # ============================================================================ + + def test_realign_buffer_rewind_seekable_stream + stream = StringIO.new "0123456789" * 100 + stream.seek 1000 + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "buffered".b + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) + + assert_equal 500, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + assert_equal 500, stream.pos + end + + def test_realign_buffer_rewind_unseekable_stream_raises_error + stream = UnseekableStream.new "0123456789" * 100 + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "buffered".b + + err = assert_raises UnseekableStreamError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) + end + + assert_includes err.message, "offset 500" + assert_includes err.message, "buffered from 1000" + assert_nil err.resume_handle + refute_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_realign_buffer_rewind_unseekable_stream_with_resume_handle + stream = UnseekableStream.new "0123456789" * 100 + driver = build_driver stream: stream + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_1", chunk_size: 256) + ) + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "buffered".b + + err = assert_raises UnseekableStreamError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 500) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_1", err.resume_handle.upload_url + assert_equal 256, err.resume_handle.chunk_size + assert_includes err.message, "offset 500" + assert_includes err.message, "buffered from 1000" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_driver_resume_handle_property + stream = StringIO.new "test" + driver = build_driver stream: stream + assert_nil driver.resume_handle + + driver.core.instance_variable_set( + :@state, + driver.core.state.with(upload_url: "https://upload.example.com/session_2", chunk_size: 512) + ) + handle = driver.resume_handle + refute_nil handle + assert_equal "https://upload.example.com/session_2", handle.upload_url + assert_equal 512, handle.chunk_size + + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/session_2", chunk_size: 512) + ) + assert_nil driver.resume_handle + + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/session_2", chunk_size: 512) + ) + assert_nil driver.resume_handle + end + + # ============================================================================ + # execute_realign_buffer tests: fast forward stream + # ============================================================================ + + def test_realign_buffer_fast_forward_seekable_stream + stream = StringIO.new "0123456789" * 200 + stream.seek 1010 + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 1000 + driver.instance_variable_set :@buffer, "0123456789".b # buffer ends at 1010 + + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1050) + + assert_equal 1050, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + assert_equal 1050, stream.pos + end + + def test_realign_buffer_fast_forward_unseekable_stream + # Stream contains 1000 bytes. Buffer has consumed up to 10 bytes (buffer_start=0, length=10 -> buffer_end=10). + # Unseekable stream pos is currently at 10. + stream = UnseekableStream.new "0123456789" * 100 + stream.read 10 # advance stream to match buffer_end + assert_equal 10, stream.pos + + driver = build_driver stream: stream + driver.instance_variable_set :@buffer_start_offset, 0 + driver.instance_variable_set :@buffer, "0123456789".b # ends at offset 10 + + # Fast forward to 50 (discards 40 bytes from stream) + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 50) + + assert_equal 50, driver.instance_variable_get(:@buffer_start_offset) + assert_equal "".b, driver.instance_variable_get(:@buffer) + assert_equal 50, stream.pos + assert_equal "0123456789", stream.read(10) + end + + def test_fast_forward_unseekable_stream_raises_stream_mismatch_on_unexpected_eof + # Stream has only 20 bytes total + stream = UnseekableStream.new "01234567890123456789" + resume_config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session_resume", + chunk_size: 256, + stream: stream + ) + driver = Driver.new client_stub: @dummy_client, config: resume_config + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_resume", chunk_size: 256) + ) + driver.instance_variable_set :@buffer, "".b + driver.instance_variable_set :@buffer_start_offset, 0 + + # Server offset is 50, but stream only has 20 bytes -> EOF hit during discard + err = assert_raises StreamMismatchError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 50) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_resume", err.resume_handle.upload_url + assert_includes err.message, "unexpected EOF during fast-forward" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_realign_buffer_raises_stream_mismatch_when_server_offset_exceeds_upload_size + stream = StringIO.new "data" + resume_config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session_resume", + chunk_size: 256, + stream: stream, + upload_size: 500 + ) + driver = Driver.new client_stub: @dummy_client, config: resume_config + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_resume", chunk_size: 256) + ) + + err = assert_raises StreamMismatchError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 600) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_resume", err.resume_handle.upload_url + assert_includes err.message, "Server reported offset 600 exceeds total upload size 500" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_realign_buffer_fast_forward_seekable_stream_raises_stream_mismatch_when_exceeding_stream_size + stream = StringIO.new "hello" + resume_config = ResumeUploadConfig.new( + upload_url: "https://upload.example.com/session_resume", + chunk_size: 256, + stream: stream, + upload_size: nil + ) + driver = Driver.new client_stub: @dummy_client, config: resume_config + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :recovery, upload_url: "https://upload.example.com/session_resume", chunk_size: 256) + ) + + err = assert_raises StreamMismatchError do + driver.send :execute_realign_buffer, Instruction::RealignBuffer.new(server_offset: 1000) + end + + refute_nil err.resume_handle + assert_equal "https://upload.example.com/session_resume", err.resume_handle.upload_url + assert_includes err.message, "Server reported offset 1000 exceeds stream size 5" + assert_includes err.message, "(upload session is resumable: see #resume_handle)" + end + + private + + def build_driver stream: + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: stream, + upload_size: 2000, + chunk_size: 100 + ) + Driver.new client_stub: @dummy_client, config: config + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb new file mode 100644 index 0000000..64f859e --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_config_test.rb @@ -0,0 +1,295 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver configuration and deadline resolution. +# +class DriverConfigTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + # Fake client stub recording calls and yielding scripted responses. + class FakeClientStub + attr_reader :requests + + def initialize responses = [], on_request: nil + @responses = responses + @requests = [] + @on_request = on_request + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + @on_request&.call + raise "Unexpected request: no scripted response left" if @responses.empty? + + resp = @responses.shift + raise resp if resp.is_a? Exception + + resp.respond_to?(:call) ? resp.call : resp + end + end + + FakeResponse = Data.define :status, :headers, :body + + def test_resolve_timeout_prefers_positive_config_timeout + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 10 * 1_048_576, + timeout: 42 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal 42, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_treats_zero_timeout_same_as_nil + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + timeout: 0 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_treats_negative_timeout_same_as_nil + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + timeout: -10 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_calculates_from_upload_size_above_base_timeout + stub = FakeClientStub.new + large_size = 7_200 * Driver::MIN_ASSUMED_THROUGHPUT # 7200 seconds at 1MB/s + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: large_size + ) + driver = Driver.new client_stub: stub, config: config + + assert_in_delta 7_200.0, driver.send(:resolve_timeout), 0.001 + end + + def test_resolve_timeout_uses_base_timeout_floor_for_small_upload_size + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 1_048_576 # 1 second at 1MB/s < 3600 + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_resolve_timeout_defaults_to_base_timeout_when_upload_size_nil + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123") + ) + driver = Driver.new client_stub: stub, config: config + + assert_equal Driver::BASE_TIMEOUT, driver.send(:resolve_timeout) + end + + def test_run_raises_deadline_exceeded_when_timeout_expires + stub = FakeClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + timeout: 5 + ) + driver = Driver.new client_stub: stub, config: config + + # Stub monotonic clock so that initial check sets deadline at t=105, and subsequent checks read t=110 + clock_ticks = [100.0, 110.0, 110.0] + Process.stub :clock_gettime, ->(_clock_id) { clock_ticks.shift || 110.0 } do + assert_raises DeadlineExceededError do + driver.run + end + end + assert_empty stub.requests + end + + def test_run_raises_deadline_exceeded_when_clock_advances_past_deadline_mid_batch + current_time = 100.0 + responses = [ + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-URL" => "https://example.com/session" }, + body: "" + ) + ] + stub = FakeClientStub.new responses + # Advance clock past deadline (105.0) mid-batch during NotifyProgress(:finalizing) before SendChunk + on_progress = lambda do |progress| + current_time = 110.0 if progress.phase == :finalizing + end + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + timeout: 5, + on_progress: on_progress + ) + driver = Driver.new client_stub: stub, config: config + + Process.stub :clock_gettime, ->(_clock_id) { current_time } do + assert_raises DeadlineExceededError do + driver.run + end + end + + # Only the start request was made; SendChunk hit deadline_exceeded? inside make_post_request + assert_equal 1, stub.requests.size + end + + def test_make_post_request_passes_timeout_close_to_remaining_budget_and_decreases_across_calls + current_time = 1000.0 + stub = FakeClientStub.new(scripted_recovery_responses, on_request: -> { current_time += 10.0 }) + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + timeout: 100.0, + data_plane_retry_policy: Gapic::Common::RetryPolicy.new(timeout: 85.0) + ) + driver = Driver.new client_stub: stub, config: config + + Process.stub :clock_gettime, ->(_clock_id) { current_time } do + assert_equal "done", driver.run + end + + timeouts = stub.requests.map { |req| req[:options][:timeout] } + assert_equal [100.0, 85.0, 80.0, 70.0], timeouts + timeouts.each_cons 2 do |prev_timeout, next_timeout| + assert_operator prev_timeout, :>, next_timeout + end + end + + def test_start_headers_derives_content_descriptors_from_config + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + content_type: "application/octet-stream" + ) + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new url: "https://example.com/upload" + + headers = driver.send :start_headers, instruction + + assert_equal "application/octet-stream", headers["X-Goog-Upload-Header-Content-Type"] + assert_equal "4", headers["X-Goog-Upload-Header-Content-Length"] + end + + def test_start_headers_passes_unrelated_caller_headers_through + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123") + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new( + url: "https://example.com/upload", + headers: { "X-Custom" => "value" } + ) + + headers = driver.send :start_headers, instruction + + assert_equal "value", headers["X-Custom"] + assert_equal "resumable", headers["X-Goog-Upload-Protocol"] + assert_equal "start", headers["X-Goog-Upload-Command"] + end + + def test_start_headers_without_caller_headers_is_unchanged + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("0123") + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new url: "https://example.com/upload" + + headers = driver.send :start_headers, instruction + + assert_equal({ "X-Goog-Upload-Protocol" => "resumable", "X-Goog-Upload-Command" => "start" }, headers) + end + + def test_start_headers_merges_caller_pass_through_upload_header + caller_headers = { + "X-Goog-Upload-Header-Content-Disposition" => 'attachment; filename="movie.mp4"', + "X-Custom" => "value" + } + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("content"), + initial_headers: caller_headers, + content_type: "video/mp4", + upload_size: 7 + ) + driver = Driver.new client_stub: FakeClientStub.new, config: config + instruction = Instruction::SendStart.new url: "https://example.com/upload", headers: caller_headers + + headers = driver.send :start_headers, instruction + + assert_equal 'attachment; filename="movie.mp4"', headers["X-Goog-Upload-Header-Content-Disposition"] + assert_equal "value", headers["X-Custom"] + assert_equal "resumable", headers["X-Goog-Upload-Protocol"] + assert_equal "start", headers["X-Goog-Upload-Command"] + assert_equal "video/mp4", headers["X-Goog-Upload-Header-Content-Type"] + assert_equal "7", headers["X-Goog-Upload-Header-Content-Length"] + end + + private + + def scripted_recovery_responses + [ + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-URL" => "https://example.com/session" }, + body: "" + ), + Gapic::Rest::Error.new( + "Service Unavailable", + 503, + headers: { "X-Goog-Upload-Status" => "active" } + ), + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "active", "X-Goog-Upload-Size-Received" => "0" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "final" }, + body: "done" + ) + ] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb new file mode 100644 index 0000000..b7b5e1b --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_error_mapping_test.rb @@ -0,0 +1,209 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" +require "faraday" + +## +# Tests for Driver network error mapping to protocol events. +# +class DriverErrorMappingTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + ## + # Integration fake client stub that raises configured errors on make_post_request. + # + class FailingClientStub + attr_accessor :error_to_raise + + def make_post_request uri:, body: nil, params: {}, options: {}, method_name: nil + raise @error_to_raise if @error_to_raise + + raise "No error configured" + end + end + + def setup + @client_stub = FailingClientStub.new + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4 + ) + @driver = Driver.new client_stub: @client_stub, config: @config + end + + # ============================================================================ + # SUT: rescue_request_error + # ============================================================================ + + def test_rescue_request_error_rest_deadline_exceeded + err = Gapic::Rest::DeadlineExceededError.new "RPC deadline exceeded", 504 + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :timeout, event.kind + assert_equal "RPC deadline exceeded", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :timeout, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_request_error_rest_error_with_status_code + err = Gapic::Rest::Error.new "Service Unavailable", 503, headers: { "Retry-After" => "15" } + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::HttpResponse, event + assert_equal 503, event.status + assert_equal({ "Retry-After" => "15" }, event.headers) + assert_equal "Service Unavailable", event.body + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::HttpResponse, integration_event + assert_equal 503, integration_event.status + assert_equal({ "Retry-After" => "15" }, integration_event.headers) + assert_equal "Service Unavailable", integration_event.body + end + + def test_rescue_request_error_rest_error_without_status_code + err = Gapic::Rest::Error.new "Client network error", nil + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_equal "Client network error", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_request_error_standard_error + err = RuntimeError.new "Unexpected low-level runtime error" + event = @driver.send :rescue_request_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_equal "Unexpected low-level runtime error", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + assert_same err, integration_event.source_error + end + + # ============================================================================ + # SUT: rescue_faraday_error + # ============================================================================ + + def test_rescue_faraday_error_with_response + response_env = { + status: 400, + headers: { "x-goog-upload-status" => "final" }, + body: '{"error":{"message":"Bad Request"}}' + } + err = Faraday::ClientError.new "the server responded with status 400", response_env + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::HttpResponse, event + assert_equal 400, event.status + assert_equal({ "x-goog-upload-status" => "final" }, event.headers) + assert_equal '{"error":{"message":"Bad Request"}}', event.body + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::HttpResponse, integration_event + assert_equal 400, integration_event.status + assert_equal '{"error":{"message":"Bad Request"}}', integration_event.body + end + + def test_rescue_faraday_error_timeout + err = Faraday::TimeoutError.new "Net::ReadTimeout with https://example.com" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :timeout, event.kind + assert_equal "Net::ReadTimeout with https://example.com", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :timeout, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_faraday_error_connection_failed + err = Faraday::ConnectionFailed.new "Connection refused - connect(2)" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :connection_failed, event.kind + assert_equal "Connection refused - connect(2)", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :connection_failed, integration_event.kind + assert_same err, integration_event.source_error + end + + def test_rescue_faraday_error_generic_without_response + err = Faraday::Error.new "Generic transport error without response env" + event = @driver.send :rescue_faraday_error, err + + assert_instance_of Event::RequestFailed, event + assert_equal :retries_exhausted, event.kind + assert_equal "Generic transport error without response env", event.message + assert_same err, event.source_error + + # End-to-end via make_post_request + @client_stub.error_to_raise = err + integration_event = @driver.send :make_post_request, "https://example.com", headers: {}, body: "", + retry_policy: nil + assert_instance_of Event::RequestFailed, integration_event + assert_equal :retries_exhausted, integration_event.kind + assert_same err, integration_event.source_error + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb new file mode 100644 index 0000000..4cf93d2 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb @@ -0,0 +1,574 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" +require "google/rpc/error_details_pb" + +## +# Integration and unit tests for Driver logging concerns. +# +class DriverLoggingTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body + + class FakeStub + attr_reader :method_names + + def initialize responses + @responses = responses + @method_names = [] + end + + def endpoint + "https://storage.googleapis.com" + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + _ = uri + _ = body + _ = params + _ = options + @method_names << method_name + resp = @responses.shift + raise resp if resp.is_a?(Exception) || (resp.is_a?(Class) && resp < Exception) + + resp + end + end + + def test_all_entries_share_upload_id_and_pass_method_names + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + + refute_empty recording.entries + upload_ids = recording.entries.map { |e| e.message.fields["uploadId"] }.uniq + assert_equal 1, upload_ids.size + refute_nil upload_ids.first + + assert_equal ["ResumableUpload.start", "ResumableUpload.upload"], stub.method_names + + info_recipes = recording.entries.select { |e| e.severity == Logger::INFO }.map { |e| e.message.fields["recipe"] } + assert_includes ["complete_upload_with_data", "complete_upload_finalized"], info_recipes.last + end + + def test_multi_chunk_upload_logs_lifecycle_entries + recording = RecordingLogger.new + run_two_chunk_upload_with_secret recording + + info_recipes = recording.entries.select { |e| e.severity == Logger::INFO }.map { |e| e.message.fields["recipe"] } + refute_includes info_recipes, "ack_chunk" + assert_includes info_recipes, "start_session" + assert_includes info_recipes, "begin_transmission" + assert(info_recipes.any? { |r| ["complete_upload_with_data", "complete_upload_finalized"].include? r }) + end + + def test_recovery_scenario_logs_enter_recovery_and_realign + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" + }, + "" + ), + FakeResponse.new(503, {}, "Service Unavailable"), + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "0" + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + + info_recipes = recording.entries.select { |e| e.severity == Logger::INFO }.map { |e| e.message.fields["recipe"] } + assert_includes info_recipes, "enter_recovery" + assert_includes info_recipes, "realign_from_recovery" + end + + def test_resume_upload_logs_resume_session_entry + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "0" + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = ResumeUploadConfig.new( + upload_url: "https://storage.googleapis.com/session?id=123", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + + resume_entry = recording.entries.find do |e| + e.severity == Logger::INFO && e.message.fields["recipe"] == "resume_session" + end + refute_nil resume_entry + assert_equal Logger::INFO, resume_entry.severity + assert_equal "Resuming upload session", resume_entry.message.message + assert_equal 256, resume_entry.message.fields["chunkSize"] + assert_includes resume_entry.message.fields["uploadUrl"], "session?id=" + end + + def test_fatal_failure_logs_warn_with_fail_with_recipe + recording = RecordingLogger.new + responses = [ + FakeResponse.new( + 403, + { "X-Goog-Upload-Status" => "final" }, + "Forbidden" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + assert_raises UploadRejectedError do + driver.run + end + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + refute_empty warn_entries + assert(warn_entries.any? { |e| e.message.fields["recipe"]&.start_with? "fail_with_" }) + end + + def test_unmatched_transition_logs_warn_and_reraises + recording = RecordingLogger.new + stub = FakeStub.new [] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello"), + upload_size: 5, + chunk_size: 256 + ) + + failing_core = Minitest::Mock.new + failing_core.expect :dispatch, nil do |_event| + raise InvalidTransitionError, "unmatched transition in state" + end + failing_core.expect :state, State.new(status: :initializing) + + driver = Driver.new client_stub: stub, config: config, core: failing_core, logger: recording + + assert_raises InvalidTransitionError do + driver.run + end + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + assert_equal 1, warn_entries.size + fields = warn_entries.first.message.fields + assert_equal "initializing", fields["status"] + assert_equal "unmatched transition in state", fields["error"] + end + + def test_full_log_corpus_redacts_secrets + recording = RecordingLogger.new + run_two_chunk_upload_with_secret recording + + corpus = log_corpus recording + refute_includes corpus, "SECRET-123456" + end + + def test_full_log_corpus_size_under_64kib + recording = RecordingLogger.new + run_two_chunk_upload_with_secret recording + + corpus = log_corpus recording + assert_operator corpus.bytesize, :<, 65_536 + end + + def test_wire_receive_logs_error_status_and_abridged_error_message + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-1" + + err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Permission denied on resource", + 403, + status: "PERMISSION_DENIED" + ) + event = Event::HttpResponse.new( + status: 403, + headers: { "x-goog-upload-status" => "final" }, + body: '{"raw":"error"}', + error: err + ) + + upload_log.wire_receive event + + debug_entries = recording.entries.select { |e| e.severity == Logger::DEBUG && e.message.message.include?("403") } + refute_empty debug_entries + fields = debug_entries.first.message.fields + assert_equal 403, fields["status"] + assert_equal "PERMISSION_DENIED", fields["errorStatus"] + assert_equal "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Permission denied on resource", fields["body"] + end + + def test_wire_receive_fallback_body_when_error_absent + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-2" + + event = Event::HttpResponse.new( + status: 503, + headers: {}, + body: "Server unavailable", + error: nil + ) + + upload_log.wire_receive event + + debug_entries = recording.entries.select { |e| e.severity == Logger::DEBUG && e.message.message.include?("503") } + refute_empty debug_entries + fields = debug_entries.first.message.fields + assert_equal 503, fields["status"] + assert_nil fields["errorStatus"] + assert_equal "Server unavailable", fields["body"] + end + + def test_lifecycle_warn_carries_rich_message_when_driver_fails + recording = RecordingLogger.new + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Bucket access denied", + 403, + status: "PERMISSION_DENIED", + headers: { "x-goog-upload-status" => "final" } + ) + stub = FakeStub.new [wrapped_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises UploadRejectedError do + driver.run + end + + assert_equal "Upload rejected by server with HTTP 403 PERMISSION_DENIED: Bucket access denied", err.message + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + refute_empty warn_entries + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_rejected" } + refute_nil fail_warn + assert_equal "Upload rejected by server with HTTP 403 PERMISSION_DENIED: Bucket access denied", + fail_warn.message.fields["error"] + + debug_entries = recording.entries.select { |e| e.severity == Logger::DEBUG && e.message.message.include?("403") } + refute_empty debug_entries + wire_recv = debug_entries.first + assert_equal "PERMISSION_DENIED", wire_recv.message.fields["errorStatus"] + end + + def test_driver_error_mapping_populates_error_on_rescue + stub = FakeStub.new [] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + driver = Driver.new client_stub: stub, config: config + + rest_err = Gapic::Rest::Error.new "Forbidden", 403, status: "PERMISSION_DENIED" + event = driver.send :rescue_request_error, rest_err + assert_instance_of Event::HttpResponse, event + assert_equal rest_err, event.error + + faraday_err = Faraday::ClientError.new "Client error", { + status: 400, + headers: { "content-type" => "application/json" }, + body: '{"error":{"message":"Bad input","code":400,"status":"INVALID_ARGUMENT"}}' + } + faraday_event = driver.send :rescue_faraday_error, faraday_err + assert_instance_of Event::HttpResponse, faraday_event + assert_instance_of Gapic::Rest::Error, faraday_event.error + assert_equal 400, faraday_event.error.status_code + assert_equal "INVALID_ARGUMENT", faraday_event.error.status + end + + def test_lifecycle_warn_includes_response_body_for_rejected_error + recording = RecordingLogger.new + raw_body = '{"error":{"code":403,"message":"Rejected by backend"}}' + faraday_err = Faraday::ClientError.new "Client error", { + status: 403, + headers: { "x-goog-upload-status" => "final" }, + body: raw_body + } + stub = FakeStub.new [faraday_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises UploadRejectedError do + driver.run + end + + assert_equal raw_body, err.response_body + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_rejected" } + refute_nil fail_warn + assert_equal raw_body, fail_warn.message.fields["responseBody"] + end + + def test_lifecycle_warn_includes_response_body_for_bad_response_error + recording = RecordingLogger.new + raw_body = '{"error":{"message":"Invalid input","code":400}}' + faraday_err = Faraday::ClientError.new "Client error", { + status: 400, + headers: { "x-goog-upload-status" => "active" }, + body: raw_body + } + stub = FakeStub.new [faraday_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises BadResponseError do + driver.run + end + + assert_equal raw_body, err.response_body + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_bad_response" } + refute_nil fail_warn + assert_equal raw_body, fail_warn.message.fields["responseBody"] + end + + def test_lifecycle_warn_omits_response_body_when_error_lacks_it + recording = RecordingLogger.new + stub_logger = Gapic::LoggingConcerns::StubLogger.new logger: recording, service: "ResumableUpload" + upload_log = Driver::UploadLog.new stub_logger, upload_id: "test-upload-no-body" + state = State.new( + status: :error, + last_error: DeadlineExceededError.new("Upload deadline exceeded") + ) + decision = Decision.new( + from_status: :transferring, + shape: :deadline_exceeded, + recipe: :fail_with_deadline_exceeded, + next_state: state, + instructions: [] + ) + + upload_log.lifecycle decision, StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_deadline_exceeded" } + refute_nil fail_warn + assert_equal "Upload deadline exceeded", fail_warn.message.fields["error"] + assert_nil fail_warn.message.fields["responseBody"] + end + + def test_error_info_reason_in_details_survives_in_error_and_logs + recording = RecordingLogger.new + error_info = Google::Rpc::ErrorInfo.new( + reason: "SERVICE_DISABLED", + domain: "googleapis.com", + metadata: { "consumer" => "projects/12345", "service" => "storage.googleapis.com" } + ) + error_info_any = Google::Protobuf::Any.pack error_info + raw_body = JSON.dump( + { + "error" => { + "code" => 403, + "message" => "Google Cloud Storage API has not been used in project 12345 or it is disabled.", + "status" => "PERMISSION_DENIED", + "details" => [JSON.parse(error_info_any.to_json)] + } + } + ) + faraday_err = Faraday::ClientError.new "Client error", { + status: 403, + headers: { "x-goog-upload-status" => "final" }, + body: raw_body + } + stub = FakeStub.new [faraday_err] + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + err = assert_raises UploadRejectedError do + driver.run + end + + refute_nil err.details + unpacked_info = err.details.find { |d| d.is_a? Google::Rpc::ErrorInfo } + refute_nil unpacked_info + assert_equal "SERVICE_DISABLED", unpacked_info.reason + assert_equal "googleapis.com", unpacked_info.domain + assert_equal "projects/12345", unpacked_info.metadata["consumer"] + + expected_msg = "Upload rejected by server with HTTP 403 PERMISSION_DENIED: " \ + "Google Cloud Storage API has not been used in project 12345 or it is disabled." + assert_equal expected_msg, err.message + assert_equal 403, err.status_code + assert_equal "PERMISSION_DENIED", err.status + assert_equal raw_body, err.response_body + + warn_entries = recording.entries.select { |e| e.severity == Logger::WARN } + fail_warn = warn_entries.find { |e| e.message.fields["recipe"] == "fail_with_rejected" } + refute_nil fail_warn + assert_equal expected_msg, fail_warn.message.fields["error"] + assert_equal raw_body, fail_warn.message.fields["responseBody"] + end + + private + + def run_two_chunk_upload_with_secret recording + chunk_size = 8 * 1024 * 1024 + secret = "SECRET-123456" + binary_prefix = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09".b + + half = (chunk_size / 2) - 10 + chunk1 = binary_prefix + ("A" * half) + secret + ("A" * (chunk_size - 10 - half - secret.bytesize)) + chunk2 = binary_prefix + ("A" * (chunk_size - 10)) + stream_data = chunk1 + chunk2 + + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?sid=#{secret}" + }, + "" + ), + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => chunk_size.to_s + }, + "" + ), + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => (chunk_size * 2).to_s + }, + "" + ), + FakeResponse.new( + 200, + { "X-Goog-Upload-Status" => "final" }, + "done" + ) + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload?token=#{secret}", + initial_headers: { "Authorization" => "Bearer #{secret}" }, + stream: StringIO.new(stream_data), + upload_size: stream_data.bytesize, + chunk_size: chunk_size + ) + + driver = Driver.new client_stub: stub, config: config, logger: recording + driver.run + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb new file mode 100644 index 0000000..54cf579 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_progress_test.rb @@ -0,0 +1,178 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for Driver progress notification dispatching and callback error propagation. +# +class DriverProgressTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + CustomCallbackError = Class.new StandardError + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + class ScriptedClientStub + def initialize responses + @responses = responses + end + + def make_post_request uri:, body: nil, params: {}, options: {}, method_name: nil + raise "No scripted response" if @responses.empty? + + @responses.shift + end + end + + def test_execute_notify_progress_without_callback_does_not_raise + driver = build_driver on_progress: nil + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 1024, total_bytes: 4096) + # Must not raise when callback is nil + driver.send :execute_notify_progress, instruction + end + + def test_execute_notify_progress_happy_path_invoked_once + calls = [] + callback = ->(progress) { calls << progress } + driver = build_driver on_progress: callback + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 500, total_bytes: 1000) + driver.send :execute_notify_progress, instruction + + assert_equal 1, calls.size + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 500, total_bytes: 1000), calls.first + end + + def test_execute_notify_progress_total_bytes_nil_passes_through + calls = [] + callback = ->(progress) { calls << progress } + driver = build_driver on_progress: callback + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 250, total_bytes: nil) + driver.send :execute_notify_progress, instruction + + assert_equal 1, calls.size + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 250, total_bytes: nil), calls.first + end + + def test_execute_notify_progress_raises_error_to_caller_when_callback_fails + callback = ->(_progress) { raise CustomCallbackError, "User UI crashed in progress callback" } + driver = build_driver on_progress: callback + + instruction = Instruction::NotifyProgress.new progress: Progress.new(phase: :uploading, bytes_uploaded: 100, total_bytes: 1000) + err = assert_raises CustomCallbackError do + driver.send :execute_notify_progress, instruction + end + + assert_equal "User UI crashed in progress callback", err.message + end + + def test_driver_run_propagates_callback_error_end_to_end + # Script responses: 1. start response -> 2. chunk response (triggers NotifyProgress) + responses = [ + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-url" => "https://example.com/upload/123", "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ) + ] + stub = ScriptedClientStub.new responses + + callback = ->(_progress) { raise CustomCallbackError, "Terminal failure in user progress handler" } + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: callback + ) + driver = Driver.new client_stub: stub, config: config + + err = assert_raises CustomCallbackError do + driver.run + end + assert_equal "Terminal failure in user progress handler", err.message + end + + def test_completed_progress_reports_total_bytes_when_upload_size_unknown + responses = [ + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-url" => "https://example.com/upload/123", "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: "{\"done\":true}" + ) + ] + stub = ScriptedClientStub.new responses + + progress_events = [] + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: nil, + chunk_size: 4, + on_progress: ->(progress) { progress_events << progress } + ) + driver = Driver.new client_stub: stub, config: config + + result = driver.run + assert_equal "{\"done\":true}", result + + uploading_snapshots = progress_events.select { |p| p.phase == :uploading } + refute_empty uploading_snapshots + assert uploading_snapshots.all? { |p| p.total_bytes.nil? } + + completed_snapshot = progress_events.last + assert_equal :completed, completed_snapshot.phase + assert_equal 10, completed_snapshot.bytes_uploaded + assert_equal 10, completed_snapshot.total_bytes + end + + private + + def build_driver on_progress: nil + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: on_progress + ) + Driver.new client_stub: Object.new, config: config + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb new file mode 100644 index 0000000..76af4c0 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_policy_test.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver retry policy resolution and configuration overrides. +# +class DriverRetryPolicyTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123") + ) + @driver = Driver.new client_stub: Object.new, config: @config + end + + def test_resolve_retry_policy_with_nil_returns_default_policy + policy = @driver.send :resolve_retry_policy, nil, RetryPolicies::START_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, policy + assert_equal RetryPolicies.default_start.retry_codes, policy.retry_codes + assert_in_delta 1.0, policy.initial_delay + assert_in_delta 15.0, policy.max_delay + assert_in_delta 1.3, policy.multiplier + assert_same RetryPolicies::START_PREDICATE, policy.retry_predicate + end + + def test_resolve_retry_policy_with_policy_instance_returns_as_is + custom_policy = Gapic::Common::RetryPolicy.new initial_delay: 5.0 + resolved = @driver.send :resolve_retry_policy, custom_policy, RetryPolicies::START_DEFAULTS + + assert_same custom_policy, resolved + assert_nil resolved.retry_predicate + assert_empty resolved.retry_codes + end + + def test_resolve_retry_policy_with_hash_applies_defaults_and_preserves_codes_and_predicate + hash_override = { initial_delay: 0.25, max_delay: 2.0 } + resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::START_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, resolved + assert_in_delta 0.25, resolved.initial_delay + assert_in_delta 2.0, resolved.max_delay + assert_in_delta 1.3, resolved.multiplier + assert_equal RetryPolicies.default_start.retry_codes, resolved.retry_codes + assert_same RetryPolicies::START_PREDICATE, resolved.retry_predicate + end + + def test_resolve_retry_policy_with_data_plane_hash_preserves_data_plane_predicate + hash_override = { timeout: 60.0 } + resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::DATA_PLANE_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, resolved + assert_in_delta 60.0, resolved.timeout + assert_equal RetryPolicies.default_data_plane.retry_codes, resolved.retry_codes + assert_same RetryPolicies::DATA_PLANE_PREDICATE, resolved.retry_predicate + end + + def test_resolve_retry_policy_with_empty_retry_codes_array_honors_empty_array + hash_override = { retry_codes: [] } + resolved = @driver.send :resolve_retry_policy, hash_override, RetryPolicies::START_DEFAULTS + + assert_kind_of Gapic::Common::RetryPolicy, resolved + assert_empty resolved.retry_codes + assert_same RetryPolicies::START_PREDICATE, resolved.retry_predicate + end + + def test_resolve_retry_policy_with_unknown_hash_key_raises_argument_error + err = assert_raises ArgumentError do + @driver.send :resolve_retry_policy, { unknown_key: 123 }, RetryPolicies::START_DEFAULTS + end + assert_match(/unknown keyword: :unknown_key/, err.message) + end + + def test_resolve_retry_policy_with_invalid_type_raises_argument_error + err = assert_raises ArgumentError do + @driver.send :resolve_retry_policy, "invalid", RetryPolicies::START_DEFAULTS + end + assert_match(/Expected RetryPolicy, Hash, or nil/, err.message) + end + + def test_driver_initialize_resolves_hash_overrides_from_config + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + start_retry_policy: { initial_delay: 0.1 }, + control_plane_retry_policy: { max_delay: 5.0 }, + data_plane_retry_policy: { multiplier: 2.0 } + ) + driver = Driver.new client_stub: Object.new, config: config + + start_policy = driver.instance_variable_get :@start_retry_policy + control_policy = driver.instance_variable_get :@control_plane_retry_policy + data_policy = driver.instance_variable_get :@data_plane_retry_policy + + assert_in_delta 0.1, start_policy.initial_delay + assert_same RetryPolicies::START_PREDICATE, start_policy.retry_predicate + + assert_in_delta 5.0, control_policy.max_delay + assert_nil control_policy.retry_predicate + + assert_in_delta 2.0, data_policy.multiplier + assert_same RetryPolicies::DATA_PLANE_PREDICATE, data_policy.retry_predicate + end + + def test_start_predicate_refutes_fatal_status_codes + [401, 403, 404, 405, 410, 413, 415].each do |code| + response_double = OpenStruct.new status: code, headers: {} + refute RetryPolicies::START_PREDICATE.call(response_double), + "Expected START_PREDICATE to return false for fatal status #{code}" + + gapic_err = Gapic::Rest::Error.new "Error", code, status: "FATAL_ERROR", headers: {} + refute RetryPolicies::START_PREDICATE.call(gapic_err), + "Expected START_PREDICATE to return false for Gapic::Rest::Error with fatal status #{code}" + end + end + + def test_start_predicate_retries_missing_header_for_non_fatal_codes + [200, 400, 500, 503].each do |code| + response_double = OpenStruct.new status: code, headers: {} + assert RetryPolicies::START_PREDICATE.call(response_double), + "Expected START_PREDICATE to return true for non-fatal status #{code} with missing status header" + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb new file mode 100644 index 0000000..2fca882 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_retry_test.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver retry behavior during session start and queries. +# +# rubocop:disable Metrics/MethodLength +class DriverRetryTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + # Fake client stub recording calls and yielding scripted responses. + class FakeClientStub + attr_reader :requests + + def initialize responses + @responses = responses + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @responses.empty? + + @responses.shift + end + end + + def test_start_retries_when_response_lacks_status_header_even_on_200 + responses = [ + FakeResponse.new(status: 200, headers: {}, body: ""), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 1.0 } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 3, stub.requests.size + assert_start_request stub.requests[0] + assert_start_request stub.requests[1] + assert_chunk_request stub.requests[2], offset: "0", length: "4", body: "0123", finalize: true + end + + def test_start_exhausts_retries_when_200_responses_continually_lack_status_header + responses = Array.new(10) { FakeResponse.new status: 200, headers: {}, body: "" } + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 0.01 } + ) + + driver = Driver.new client_stub: stub, config: config + err = assert_raises RequestFailedError do + driver.run + end + + assert_match(/Missing X-Goog-Upload-Status/, err.message) + assert_equal 200, err.status_code + assert_instance_of BadResponseError, err.cause + assert stub.requests.size > 1 + end + + def test_start_exhausts_retries_when_non_200_responses_continually_lack_status_header + responses = Array.new(10) { FakeResponse.new status: 503, headers: {}, body: "Service Unavailable" } + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10, + start_retry_policy: { initial_delay: 0.001, max_delay: 0.002, timeout: 0.01 } + ) + + driver = Driver.new client_stub: stub, config: config + err = assert_raises BadResponseError do + driver.run + end + + assert_equal 503, err.status_code + assert_includes err.message, "503" + refute_match(/Missing X-Goog-Upload-Status/, err.message) + assert stub.requests.size > 1 + end + + def test_query_does_not_retry_on_missing_status_header_in_driver + responses = [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + FakeResponse.new(status: 200, headers: {}, body: ""), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123"), + upload_size: 4, + chunk_size: 10 + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 5, stub.requests.size + assert_start_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: true + assert_query_request stub.requests[2] + assert_query_request stub.requests[3] + assert_chunk_request stub.requests[4], offset: "0", length: "4", body: "0123", finalize: true + end + + private + + def assert_start_request req + assert_equal "https://example.com/upload", req[:uri] + assert_equal "start", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_query_request req + assert_equal "https://example.com/session/1", req[:uri] + assert_equal "query", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_chunk_request req, offset:, length:, body:, finalize: + expected_cmd = finalize ? "upload, finalize" : "upload" + metadata = req[:options][:metadata] + assert_equal "https://example.com/session/1", req[:uri] + assert_equal expected_cmd, metadata["X-Goog-Upload-Command"] + assert_equal offset, metadata["X-Goog-Upload-Offset"] + assert_equal length, metadata["Content-Length"] + assert_equal body, req[:body] + end +end +# rubocop:enable Metrics/MethodLength diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb new file mode 100644 index 0000000..b5d73df --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb @@ -0,0 +1,430 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Driver synchronous upload execution engine. +# +class DriverTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + # Fake client stub recording calls and yielding scripted responses. + class FakeClientStub + attr_reader :requests + + def initialize responses + @responses = responses + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options } + raise "Unexpected request: no scripted response left" if @responses.empty? + + @responses.shift + end + end + + def test_multi_chunk_upload_with_active_responses + progress_records = [] + stub = FakeClientStub.new build_scripted_responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 4, stub.requests.size + assert_start_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: false + assert_chunk_request stub.requests[2], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[3], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_upload_recovers_when_chunk_response_lacks_status_header + progress_records = [] + responses = build_recovery_responses + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 5, stub.requests.size + assert_start_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "0", length: "4", body: "0123", finalize: false + assert_query_request stub.requests[2] + assert_chunk_request stub.requests[3], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[4], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_resume_upload_success + progress_records = [] + responses = [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "4" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = ResumeUploadConfig.new( + upload_url: "https://example.com/session/1", + chunk_size: 4, + stream: StringIO.new("0123456789"), + upload_size: 10, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 3, stub.requests.size + assert_query_request stub.requests[0] + assert_chunk_request stub.requests[1], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[2], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_resume_upload_with_409_recovery_retry + progress_records = [] + responses = [ + FakeResponse.new( + status: 409, + headers: { "X-Goog-Upload-Status" => "active" }, + body: "Conflict" + ), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "4" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + stub = FakeClientStub.new responses + config = ResumeUploadConfig.new( + upload_url: "https://example.com/session/1", + chunk_size: 4, + stream: StringIO.new("0123456789"), + upload_size: 10, + on_progress: ->(p) { progress_records << p } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + assert_equal 4, stub.requests.size + assert_query_request stub.requests[0] + assert_query_request stub.requests[1] + assert_chunk_request stub.requests[2], offset: "4", length: "4", body: "4567", finalize: false + assert_chunk_request stub.requests[3], offset: "8", length: "2", body: "89", finalize: true + + assert_equal [ + Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 4, total_bytes: 10), + Progress.new(phase: :uploading, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :finalizing, bytes_uploaded: 8, total_bytes: 10), + Progress.new(phase: :completed, bytes_uploaded: 10, total_bytes: 10) + ], progress_records + end + + def test_run_returns_nil_body_when_final_response_has_none + responses = [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "X-Goog-Upload-Status" => "final" }, + body: nil + ) + ] + stub = FakeClientStub.new responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("ab"), + upload_size: 2, + chunk_size: 4 + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_nil result + assert_equal 2, stub.requests.size + end + + # Fake Core yielding a fixed Decision to test Driver#run invariant guards. + class FakeCore + attr_reader :state, :last_decision + + def initialize decision + @decision = decision + @state = decision.next_state + @last_decision = nil + end + + def dispatch _event + @last_decision = @decision + @decision.instructions + end + end + + def test_run_raises_internal_error_on_empty_batch + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_empty, + next_state: State.new(status: :starting), + instructions: [] + ) + driver = Driver.new client_stub: FakeClientStub.new([]), config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_empty " \ + "produced no continuation event and did not terminate", + err.message + end + + def test_run_raises_internal_error_on_multiple_continuation_events + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + send_start = Instruction::SendStart.new url: "https://example.com/upload", headers: {}, body: "" + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_multi, + next_state: State.new(status: :starting), + instructions: [send_start, send_start] + ) + resp = FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ) + stub = FakeClientStub.new [resp, resp] + driver = Driver.new client_stub: stub, config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_multi produced multiple continuation events", + err.message + assert_empty stub.requests + end + + def test_run_raises_internal_error_on_mixed_continuation_and_terminal + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + send_start = Instruction::SendStart.new url: "https://example.com/upload", headers: {}, body: "" + term_success = Instruction::TerminateSuccess.new response: Event::HttpResponse.new(status: 200, headers: {}, body: "") + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_mixed, + next_state: State.new(status: :starting), + instructions: [send_start, term_success] + ) + stub = FakeClientStub.new [] + driver = Driver.new client_stub: stub, config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_mixed " \ + "produced both a continuation event and a terminal instruction", + err.message + assert_empty stub.requests + end + + def test_run_raises_internal_error_on_unclassified_instruction + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4 + ) + decision = Decision.new( + from_status: :initializing, + shape: :start_upload, + recipe: :broken_unclassified, + next_state: State.new(status: :starting), + instructions: [Object.new] + ) + stub = FakeClientStub.new [] + driver = Driver.new client_stub: stub, config: config, core: FakeCore.new(decision) + + err = assert_raises InternalError do + driver.run + end + assert_equal "Resumable upload internal error: recipe :broken_unclassified emitted unclassified instruction Object", + err.message + assert_empty stub.requests + end + + def test_on_progress_return_value_does_not_leak_into_trampoline_invariant + stub = FakeClientStub.new build_scripted_responses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("0123456789"), + upload_size: 10, + chunk_size: 4, + on_progress: ->(_p) { Event::HttpResponse.new status: 200, headers: {} } + ) + + driver = Driver.new client_stub: stub, config: config + result = driver.run + + assert_equal '{"done":true}', result + end + + private + + def build_scripted_responses + [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + end + + def build_recovery_responses + [ + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-URL" => "https://example.com/session/1", + "X-Goog-Upload-Status" => "active" + }, + body: "" + ), + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + FakeResponse.new( + status: 200, + headers: { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-Size-Received" => "4" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: ""), + FakeResponse.new(status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}') + ] + end + + def assert_start_request req + assert_equal "https://example.com/upload", req[:uri] + assert_equal "start", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_query_request req + assert_equal "https://example.com/session/1", req[:uri] + assert_equal "query", req[:options][:metadata]["X-Goog-Upload-Command"] + end + + def assert_chunk_request req, offset:, length:, body:, finalize: + expected_cmd = finalize ? "upload, finalize" : "upload" + metadata = req[:options][:metadata] + assert_equal "https://example.com/session/1", req[:uri] + assert_equal expected_cmd, metadata["X-Goog-Upload-Command"] + assert_equal offset, metadata["X-Goog-Upload-Offset"] + assert_equal length, metadata["Content-Length"] + assert_equal body, req[:body] + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb new file mode 100644 index 0000000..da1a692 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "ostruct" +require "faraday" + +## +# Tests for ResumableUpload RetryPolicies and header extraction. +# +class RetryPoliciesTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + # ============================================================================ + # SUT: extract_headers + # ============================================================================ + + def test_extract_headers_from_headers_method + obj = OpenStruct.new headers: { "X-Test-Header" => "value1" } + assert_equal({ "X-Test-Header" => "value1" }, RetryPolicies.extract_headers(obj)) + end + + def test_extract_headers_from_response_headers_method + obj = OpenStruct.new response_headers: { "X-Test-Header" => "value2" } + assert_equal({ "X-Test-Header" => "value2" }, RetryPolicies.extract_headers(obj)) + end + + def test_extract_headers_from_faraday_response_hash + err = Faraday::ClientError.new "error message", { headers: { "X-Test-Header" => "value3" } } + assert_equal({ "X-Test-Header" => "value3" }, RetryPolicies.extract_headers(err)) + end + + def test_extract_headers_returns_nil_when_no_headers_present + assert_nil RetryPolicies.extract_headers(StandardError.new("error")) + assert_nil RetryPolicies.extract_headers(nil) + assert_nil RetryPolicies.extract_headers(Object.new) + assert_nil RetryPolicies.extract_headers("string") + assert_nil RetryPolicies.extract_headers({}) + end + + # ============================================================================ + # SUT: RetryPolicies.default_start + # ============================================================================ + + def test_default_start_missing_status_header_retries_unconditionally + policy = RetryPolicies.default_start + + # Retriable code (503) without status header + err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without status header (still retries because missing status header is retriable) + err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_400) + + # Status 200 OK without status header + resp_200 = OpenStruct.new response_status: 200, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(resp_200) + + # No status code without status header + err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_no_code) + + # Empty status header string + err_empty_status = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "" } + assert policy.retry_error?(err_empty_status) + end + + def test_default_start_with_status_header_falls_back_to_codes + policy = RetryPolicies.default_start + + # Retriable code (503) with status header + err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) with status header + err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_400) + + # Without status code with status header + err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_no_code) + end + + def test_default_start_no_headers_falls_back_to_codes + policy = RetryPolicies.default_start + + # Retriable code (503) without headers + err_503 = OpenStruct.new response_status: 503 + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without headers + err_400 = OpenStruct.new response_status: 400 + refute policy.retry_error?(err_400) + + # Without status code and without headers + err_no_code = RuntimeError.new "generic network error" + refute policy.retry_error?(err_no_code) + end + + # ============================================================================ + # SUT: RetryPolicies.default_control_plane + # ============================================================================ + + def test_default_control_plane_missing_status_header_falls_back_to_codes + policy = RetryPolicies.default_control_plane + + # Retriable code (503) without status header + err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without status header + err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_400) + + # Without status code without status header + err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_no_code) + end + + def test_default_control_plane_with_status_header_falls_back_to_codes + policy = RetryPolicies.default_control_plane + + # Retriable code (503) with status header + err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) with status header + err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_400) + + # Without status code with status header + err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_no_code) + end + + def test_default_control_plane_no_headers_falls_back_to_codes + policy = RetryPolicies.default_control_plane + + # Retriable code (503) without headers + err_503 = OpenStruct.new response_status: 503 + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without headers + err_400 = OpenStruct.new response_status: 400 + refute policy.retry_error?(err_400) + + # Without status code and without headers + err_no_code = RuntimeError.new "generic network error" + refute policy.retry_error?(err_no_code) + end + + # ============================================================================ + # SUT: RetryPolicies.default_data_plane + # ============================================================================ + + def test_default_data_plane_missing_status_header_unretriable + policy = RetryPolicies.default_data_plane + + # Retriable code (503) without status header (predicate returns false -> unretriable) + err_503 = OpenStruct.new response_status: 503, headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_503) + + # Non-retriable code (400) without status header + err_400 = OpenStruct.new response_status: 400, headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_400) + + # Without status code without status header + err_no_code = OpenStruct.new headers: { "Content-Type" => "text/plain" } + refute policy.retry_error?(err_no_code) + + # Empty status header string + err_empty_status = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "" } + refute policy.retry_error?(err_empty_status) + end + + def test_default_data_plane_with_status_header_falls_back_to_codes + policy = RetryPolicies.default_data_plane + + # Retriable code (503) with status header + err_503 = OpenStruct.new response_status: 503, headers: { "X-Goog-Upload-Status" => "active" } + assert policy.retry_error?(err_503) + + # Non-retriable code (400) with status header + err_400 = OpenStruct.new response_status: 400, headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_400) + + # Without status code with status header + err_no_code = OpenStruct.new headers: { "X-Goog-Upload-Status" => "active" } + refute policy.retry_error?(err_no_code) + end + + def test_default_data_plane_no_headers_falls_back_to_codes + policy = RetryPolicies.default_data_plane + + # Retriable code (503) without headers + err_503 = OpenStruct.new response_status: 503 + assert policy.retry_error?(err_503) + + # Non-retriable code (400) without headers + err_400 = OpenStruct.new response_status: 400 + refute policy.retry_error?(err_400) + + # Without status code and without headers + err_no_code = RuntimeError.new "generic network error" + refute policy.retry_error?(err_no_code) + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb new file mode 100644 index 0000000..d2b7c3d --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_classification_test.rb @@ -0,0 +1,351 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" + +## +# Tests for classification and header extraction rules in the Resumable Upload protocol. +# +class RulesClassificationTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def test_header_value_exact_match + headers = { "X-Goog-Upload-Status" => "active" } + assert_equal "active", Rules.header_value(headers, "X-Goog-Upload-Status") + end + + def test_header_value_case_insensitivity + assert_equal "active", Rules.header_value({ "x-goog-upload-status" => "active" }, "X-Goog-Upload-Status") + assert_equal "active", Rules.header_value({ "X-GOOG-UPLOAD-STATUS" => "active" }, "X-Goog-Upload-Status") + assert_equal "active", Rules.header_value({ "x-Goog-UpLoad-Status" => "active" }, "X-Goog-Upload-Status") + end + + def test_header_value_with_symbol_keys + headers = { :"x-goog-upload-status" => "active" } + assert_equal "active", Rules.header_value(headers, "X-Goog-Upload-Status") + end + + def test_header_value_missing_or_non_hash + assert_nil Rules.header_value({ "Content-Type" => "text/plain" }, "X-Goog-Upload-Status") + assert_nil Rules.header_value(nil, "X-Goog-Upload-Status") + assert_nil Rules.header_value([], "X-Goog-Upload-Status") + assert_nil Rules.header_value("string", "X-Goog-Upload-Status") + end + + def test_classify_http_response_active + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + assert_equal :response_active, Rules.classify_http_response(resp_200) + + # Value case variations + ["Active", "ACTIVE", "aCtIvE"].each do |val| + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" + assert_equal :response_active, Rules.classify_http_response(resp) + end + + # Non-200 with active maps to Category 2 + [503, 500, 400, 408].each do |code| + resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp_non_200) + end + end + + def test_classify_http_response_final + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: "" + assert_equal :response_final, Rules.classify_http_response(resp_200) + + # Value case variations + ["Final", "FINAL"].each do |val| + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" + assert_equal :response_final, Rules.classify_http_response(resp) + end + + # Non-200 with final maps to response_rejected + [400, 404, 500].each do |code| + resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "final" }, body: "" + assert_equal :response_rejected, Rules.classify_http_response(resp_non_200) + end + end + + def test_classify_http_response_cancelled + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "cancelled" }, body: "" + assert_equal :response_cancelled, Rules.classify_http_response(resp_200) + + # Value case variations + ["Cancelled", "CANCELLED"].each do |val| + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => val }, body: "" + assert_equal :response_cancelled, Rules.classify_http_response(resp) + end + + # Non-200 with cancelled maps to fatal bad response + [400, 500].each do |code| + resp_non_200 = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "cancelled" }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_non_200) + end + end + + def test_classify_http_response_missing_header_non_fatal + # HTTP 200 missing header + resp_200 = Event::HttpResponse.new status: 200, headers: {}, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp_200) + + # Recoverable 4xx missing header + Rules::CAT2_STATUS_CODES.each do |code| + resp = Event::HttpResponse.new status: code, headers: {}, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp), "Expected #{code} to classify as :response_cat2" + end + + # 5xx server/gateway errors missing header + [500, 502, 503, 504].each do |code| + resp = Event::HttpResponse.new status: code, headers: {}, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp), "Expected #{code} to classify as :response_cat2" + end + + # Empty string header + resp_empty = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "" }, body: "" + assert_equal :response_cat2, Rules.classify_http_response(resp_empty) + end + + def test_classify_http_response_missing_header_fatal_status_codes + Rules::FATAL_STATUS_CODES.each do |code| + resp = Event::HttpResponse.new status: code, headers: {}, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp), + "Expected fatal code #{code} to classify as :response_fatal_bad_response" + + resp_empty = Event::HttpResponse.new status: code, headers: { "X-Goog-Upload-Status" => "" }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_empty), + "Expected fatal code #{code} with empty header to classify as :response_fatal_bad_response" + end + end + + def test_classify_http_response_unknown_header_values + ["absconded", "pending", "in_progress", "error", "unknown"].each do |unknown_val| + resp_200 = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => unknown_val }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_200) + + resp_400 = Event::HttpResponse.new status: 400, headers: { "X-Goog-Upload-Status" => unknown_val }, body: "" + assert_equal :response_fatal_bad_response, Rules.classify_http_response(resp_400) + end + end + + def test_classify_http_response_header_key_casing + keys = ["x-goog-upload-status", "X-GOOG-UPLOAD-STATUS", "X-Goog-Upload-Status", :"x-goog-upload-status"] + keys.each do |key| + resp = Event::HttpResponse.new status: 200, headers: { key => "active" }, body: "" + assert_equal :response_active, Rules.classify_http_response(resp) + end + end + + def test_shape_of_control_events + assert_equal :start_upload, Rules.shape_of(Event::StartUpload.new) + assert_equal :start_upload, Rules.shape_of(Event::StartUpload) + assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload.new) + assert_equal :resume_upload, Rules.shape_of(Event::ResumeUpload) + assert_equal :user_cancel, Rules.shape_of(Event::Cancel.new) + assert_equal :user_cancel, Rules.shape_of(Event::Cancel) + assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded.new) + assert_equal :global_deadline_exceeded, Rules.shape_of(Event::GlobalDeadlineExceeded) + end + + def test_shape_of_chunk_read + full_chunk = Event::ChunkRead.new bytes_buffered: 4096, eof: false + assert_equal :chunk_read_full, Rules.shape_of(full_chunk) + + eof_data = Event::ChunkRead.new bytes_buffered: 1024, eof: true + assert_equal :chunk_read_eof_with_data, Rules.shape_of(eof_data) + + eof_empty = Event::ChunkRead.new bytes_buffered: 0, eof: true + assert_equal :chunk_read_eof_empty, Rules.shape_of(eof_empty) + end + + def test_shape_of_request_failed + timeout = Event::RequestFailed.new kind: :timeout, message: "read timeout" + assert_equal :request_timeout, Rules.shape_of(timeout) + + exhausted = Event::RequestFailed.new kind: :retries_exhausted, message: "exhausted" + assert_equal :request_retries_exhausted, Rules.shape_of(exhausted) + + conn_failed = Event::RequestFailed.new kind: :connection_failed, message: "dropped" + assert_equal :request_connection_failed, Rules.shape_of(conn_failed) + + other = Event::RequestFailed.new kind: :other, message: "unknown error" + assert_equal :request_failed_unknown, Rules.shape_of(other) + end + + def test_shape_of_http_response_delegates_to_classify + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "active" }, body: "" + assert_equal :response_active, Rules.shape_of(resp) + end + + def test_shape_of_unknown_event + assert_equal :unknown, Rules.shape_of(Object.new) + assert_equal :unknown, Rules.shape_of(nil) + assert_equal :unknown, Rules.shape_of("unrecognized_event") + end + + ## + # One event per shape Rules can produce. Used to check SHAPES in both directions, so that a new shape must + # be added to the constant and a retired shape must be removed from it. + # + def shape_corpus + active = { "X-Goog-Upload-Status" => "active" } + final = { "X-Goog-Upload-Status" => "final" } + cancelled = { "X-Goog-Upload-Status" => "cancelled" } + { + start_upload: Event::StartUpload.new, + resume_upload: Event::ResumeUpload.new, + user_cancel: Event::Cancel.new, + global_deadline_exceeded: Event::GlobalDeadlineExceeded.new, + chunk_read_full: Event::ChunkRead.new(bytes_buffered: 4096, eof: false), + chunk_read_eof_with_data: Event::ChunkRead.new(bytes_buffered: 1024, eof: true), + chunk_read_eof_empty: Event::ChunkRead.new(bytes_buffered: 0, eof: true), + request_timeout: Event::RequestFailed.new(kind: :timeout), + request_retries_exhausted: Event::RequestFailed.new(kind: :retries_exhausted), + request_connection_failed: Event::RequestFailed.new(kind: :connection_failed), + request_failed_unknown: Event::RequestFailed.new(kind: :something_else), + response_active: Event::HttpResponse.new(status: 200, headers: active), + response_final: Event::HttpResponse.new(status: 200, headers: final), + response_cancelled: Event::HttpResponse.new(status: 200, headers: cancelled), + response_rejected: Event::HttpResponse.new(status: 400, headers: final), + response_cat2: Event::HttpResponse.new(status: 200, headers: {}), + response_fatal_bad_response: Event::HttpResponse.new(status: 401, headers: {}), + unknown: Object.new + } + end + + def test_shapes_constant_is_exhaustive_and_minimal + corpus = shape_corpus + + corpus.each do |expected_shape, event| + assert_equal expected_shape, Rules.shape_of(event), + "Corpus event for #{expected_shape} no longer classifies as that shape" + end + + assert_empty Rules::SHAPES - corpus.keys, + "SHAPES members that no corpus event produces (phantom or untested shapes)" + assert_empty corpus.keys - Rules::SHAPES, + "shape_of produces shapes that are missing from SHAPES" + assert_predicate Rules::SHAPES, :frozen? + end + + def test_statuses_tracks_state_descriptions + assert_empty Rules::STATUSES - Rules::STATE_DESCRIPTIONS.keys, + "status missing a description" + assert_empty Rules::STATE_DESCRIPTIONS.keys - Rules::STATUSES, + "description for unknown status" + assert_equal Rules::STATUSES.uniq, Rules::STATUSES + assert_predicate Rules::STATUSES, :frozen? + + assert_empty Rules::TERMINAL_STATUSES - Rules::STATUSES, + "TERMINAL_STATUSES contains statuses outside STATUSES" + assert_predicate Rules::TERMINAL_STATUSES, :frozen? + end + + def test_resume_handle_from_returns_nil_for_all_terminal_statuses_except_error + base_state = State.new upload_url: "https://example.com/session/123", chunk_size: 262_144 + + Rules::TERMINAL_STATUSES.each do |terminal_status| + state = base_state.with status: terminal_status + handle = Rules.resume_handle_from state + if terminal_status == :error + refute_nil handle, "Expected resume_handle_from to return a ResumeHandle for :error status" + else + assert_nil handle, "Expected resume_handle_from to return nil for terminal status #{terminal_status.inspect}" + end + end + end + + def test_decide_rejects_a_shape_outside_the_vocabulary + state = State.new + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("data") + + error = Rules.stub :shape_of, :not_a_real_shape do + assert_raises InternalError do + Rules.decide state, Event::StartUpload.new, config + end + end + + assert_match(/Resumable upload internal error: shape_of returned unknown shape :not_a_real_shape/, error.message) + end + + def test_decide_rejects_a_recipe_outside_the_vocabulary + state = State.new + config = StartUploadConfig.new initial_url: "https://example.com/upload", stream: StringIO.new("data") + original_recipes = Rules::RECIPES + + error = begin + Rules.send :remove_const, :RECIPES + Rules.const_set :RECIPES, [].freeze + assert_raises InternalError do + Rules.decide state, Event::StartUpload.new, config + end + ensure + Rules.send :remove_const, :RECIPES + Rules.const_set :RECIPES, original_recipes + end + + assert_match(/Resumable upload internal error: decide selected unknown recipe :start_session/, error.message) + end + + def test_resolve_chunk_size_with_nil_or_non_positive_granularity + # nil granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, nil) + assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, nil) + + # 0 granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, 0) + assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, 0) + + # Negative granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, -256) + assert_equal Rules::DEFAULT_CHUNK_SIZE, Rules.resolve_chunk_size(nil, -1) + end + + def test_resolve_chunk_size_when_divisible + # User-specified evenly divides granularity + assert_equal 1024, Rules.resolve_chunk_size(1024, 256) + assert_equal 1_048_576, Rules.resolve_chunk_size(1_048_576, 262_144) + + # Default chunk size (8_388_608) evenly divides the standard 256 KB backend granularity + assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 262_144) + assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 524_288) + end + + def test_resolve_chunk_size_when_not_divisible_rounds_down + # User-specified rounds down to nearest multiple + assert_equal 768, Rules.resolve_chunk_size(1000, 256) + assert_equal 9_961_472, Rules.resolve_chunk_size(10_000_000, 262_144) + + # Default chunk size (8_388_608) rounds down with non-divisor granularity (500_000 * 16) + assert_equal 8_000_000, Rules.resolve_chunk_size(nil, 500_000) + assert_equal 8_192_000, Rules.resolve_chunk_size(nil, 1_024_000) + end + + def test_resolve_chunk_size_when_granularity_equal_to_chunk_size + assert_equal 256, Rules.resolve_chunk_size(256, 256) + assert_equal 262_144, Rules.resolve_chunk_size(262_144, 262_144) + assert_equal 8_388_608, Rules.resolve_chunk_size(nil, 8_388_608) + end + + def test_resolve_chunk_size_when_granularity_greater_than_chunk_size + # User chunk size strictly less than granularity promotes to granularity (avoids 0) + assert_equal 256, Rules.resolve_chunk_size(100, 256) + assert_equal 262_144, Rules.resolve_chunk_size(1, 262_144) + + # Default chunk size strictly less than large server granularity promotes to granularity + assert_equal 16_777_216, Rules.resolve_chunk_size(nil, 16_777_216) + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb new file mode 100644 index 0000000..9b3edf1 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_decide_test.rb @@ -0,0 +1,334 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules.decide transitions per router row. +# +class RulesDecideTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_recipe_phases_partition + notifying = Rules::RECIPE_PHASES.keys + non_notifying = Rules::NON_NOTIFYING_RECIPES + all_classified = notifying + non_notifying + + assert_empty Rules::RECIPES - all_classified, + "Recipes missing from RECIPE_PHASES or NON_NOTIFYING_RECIPES" + assert_empty all_classified - Rules::RECIPES, + "Phantom recipes in RECIPE_PHASES or NON_NOTIFYING_RECIPES" + assert_empty notifying & non_notifying, + "Recipes present in both RECIPE_PHASES and NON_NOTIFYING_RECIPES" + end + + def test_row_initializing_start_upload + decision = Rules.decide State.new(status: :initializing), Event::StartUpload.new, @config + assert_equal :initializing, decision.from_status + assert_equal :start_upload, decision.shape + assert_equal :start_session, decision.recipe + assert_equal :starting, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendStart, decision.instructions[1] + end + + def test_row_initializing_resume_upload + resume_config = ResumeUploadConfig.new( + upload_url: "https://example.com/upload/session1", + chunk_size: 512, + stream: StringIO.new("data"), + upload_size: 1024 + ) + decision = Rules.decide State.new(status: :initializing), Event::ResumeUpload.new, resume_config + assert_equal :initializing, decision.from_status + assert_equal :resume_upload, decision.shape + assert_equal :resume_session, decision.recipe + assert_equal :recovery, decision.next_state.status + assert_equal "https://example.com/upload/session1", decision.next_state.upload_url + assert_equal 512, decision.next_state.chunk_size + assert_equal 0, decision.next_state.offset + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendQuery, decision.instructions[1] + assert_equal "https://example.com/upload/session1", decision.instructions[1].url + end + + def test_row_starting_response_active + active_resp = Event::HttpResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-url" => "https://example.com/session" } + ) + decision = Rules.decide State.new(status: :starting), active_resp, @config + assert_equal :starting, decision.from_status + assert_equal :response_active, decision.shape + assert_equal :begin_transmission, decision.recipe + assert_equal :transmission_reading, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::FillBuffer, decision.instructions[1] + end + + def test_row_transmission_reading_chunk_read_full + decision = Rules.decide( + State.new(status: :transmission_reading, upload_url: "https://example.com/session"), + Event::ChunkRead.new(bytes_buffered: 512, eof: false), + @config + ) + assert_equal :transmission_reading, decision.from_status + assert_equal :chunk_read_full, decision.shape + assert_equal :send_chunk, decision.recipe + assert_equal :transmission_sending, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendChunk, decision.instructions.first + refute decision.instructions.first.finalize + end + + def test_row_transmission_reading_chunk_read_eof_with_data + decision = Rules.decide( + State.new(status: :transmission_reading, upload_url: "https://example.com/session"), + Event::ChunkRead.new(bytes_buffered: 256, eof: true), + @config + ) + assert_equal :transmission_reading, decision.from_status + assert_equal :chunk_read_eof_with_data, decision.shape + assert_equal :send_upload_finalize, decision.recipe + assert_equal :finalizing_sending_upload, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendChunk, decision.instructions[1] + assert decision.instructions[1].finalize + end + + def test_row_transmission_reading_chunk_read_eof_empty + decision = Rules.decide( + State.new(status: :transmission_reading, upload_url: "https://example.com/session"), + Event::ChunkRead.new(bytes_buffered: 0, eof: true), + @config + ) + assert_equal :transmission_reading, decision.from_status + assert_equal :chunk_read_eof_empty, decision.shape + assert_equal :send_finalize, decision.recipe + assert_equal :finalizing_sending_finalize, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendFinalize, decision.instructions[1] + end + + def test_row_transmission_sending_response_active + active_resp = Event::HttpResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-url" => "https://example.com/session" } + ) + decision = Rules.decide( + State.new(status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, in_flight_length: 512), + active_resp, + @config + ) + assert_equal :transmission_sending, decision.from_status + assert_equal :response_active, decision.shape + assert_equal :ack_chunk, decision.recipe + assert_equal :transmission_reading, decision.next_state.status + assert_recipe_progress_notification decision + assert_equal 3, decision.instructions.size + end + + def test_row_transmission_sending_enter_recovery + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + decision = Rules.decide( + State.new(status: :transmission_sending, upload_url: "https://example.com/session"), + cat2_resp, + @config + ) + assert_equal :transmission_sending, decision.from_status + assert_equal :response_cat2, decision.shape + assert_equal :enter_recovery, decision.recipe + assert_equal :recovery, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendQuery, decision.instructions[1] + end + + def test_row_finalizing_sending_upload_response_final + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + decision = Rules.decide( + State.new(status: :finalizing_sending_upload, offset: 512, in_flight_length: 512), + final_resp, + @config + ) + assert_equal :finalizing_sending_upload, decision.from_status + assert_equal :response_final, decision.shape + assert_equal :complete_upload_with_data, decision.recipe + assert_equal :success, decision.next_state.status + assert_recipe_progress_notification decision + assert_equal 2, decision.instructions.size + end + + def test_row_finalizing_sending_finalize_response_final + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + decision = Rules.decide State.new(status: :finalizing_sending_finalize), final_resp, @config + assert_equal :finalizing_sending_finalize, decision.from_status + assert_equal :response_final, decision.shape + assert_equal :complete_upload_finalized, decision.recipe + assert_equal :success, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::TerminateSuccess, decision.instructions[1] + end + + def test_row_recovery_response_active + recovery_active_resp = Event::HttpResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-size-received" => "256" } + ) + decision = Rules.decide State.new(status: :recovery), recovery_active_resp, @config + assert_equal :recovery, decision.from_status + assert_equal :response_active, decision.shape + assert_equal :realign_from_recovery, decision.recipe + assert_equal :transmission_reading, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::RealignBuffer, decision.instructions[1] + end + + def test_row_recovery_response_cat2 + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + decision = Rules.decide State.new(status: :recovery), cat2_resp, @config + assert_equal :recovery, decision.from_status + assert_equal :response_cat2, decision.shape + assert_equal :retry_recovery, decision.recipe + assert_equal :recovery, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendQuery, decision.instructions.first + end + + def test_row_cancelling_response_cancelled + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + decision = Rules.decide State.new(status: :cancelling), cancelled_resp, @config + assert_equal :cancelling, decision.from_status + assert_equal :response_cancelled, decision.shape + assert_equal :complete_cancellation, decision.recipe + assert_equal :cancelled, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::TerminateFailure, decision.instructions.first + end + + def test_row_cancelling_user_cancel_raises_invalid_transition + err = assert_raises InvalidTransitionError do + Rules.decide State.new(status: :cancelling), Event::Cancel.new, @config + end + assert_equal :cancelling, err.state + end + + def test_row_global_deadline_exceeded + decision = Rules.decide State.new(status: :transmission_sending), Event::GlobalDeadlineExceeded.new, @config + assert_equal :transmission_sending, decision.from_status + assert_equal :global_deadline_exceeded, decision.shape + assert_equal :fail_with_deadline_exceeded, decision.recipe + assert_equal :error, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of DeadlineExceededError, decision.next_state.last_error + end + + def test_row_user_cancel + decision = Rules.decide State.new(status: :transmission_sending), Event::Cancel.new, @config + assert_equal :transmission_sending, decision.from_status + assert_equal :user_cancel, decision.shape + assert_equal :cancel_session, decision.recipe + assert_equal :cancelling, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::SendCancel, decision.instructions[1] + end + + def test_user_cancel_across_all_statuses + cancellable = [ + :transmission_reading, + :transmission_sending, + :finalizing_sending_upload, + :finalizing_sending_finalize, + :recovery + ] + + Rules::STATUSES.each do |status| + state = State.new status: status, upload_url: "https://example.com/upload/session-1" + if cancellable.include? status + decision = Rules.decide state, Event::Cancel.new, @config + assert_equal :cancel_session, decision.recipe, "Expected :cancel_session for status #{status.inspect}" + assert_equal :cancelling, decision.next_state.status + else + err = assert_raises InvalidTransitionError, "Expected InvalidTransitionError for status #{status.inspect}" do + Rules.decide state, Event::Cancel.new, @config + end + assert_equal status, err.state + end + end + end + + def test_row_response_rejected + rejected_resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Rejected" + decision = Rules.decide State.new(status: :starting), rejected_resp, @config + assert_equal :starting, decision.from_status + assert_equal :response_rejected, decision.shape + assert_equal :fail_with_rejected, decision.recipe + assert_equal :rejected, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of UploadRejectedError, decision.next_state.last_error + end + + def test_row_fail_with_bad_response + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + decision = Rules.decide State.new(status: :starting), cat2_resp, @config + assert_equal :starting, decision.from_status + assert_equal :response_cat2, decision.shape + assert_equal :fail_with_bad_response, decision.recipe + assert_equal :error, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of BadResponseError, decision.next_state.last_error + end + + def test_row_fail_with_request_error + req_failed = Event::RequestFailed.new kind: :retries_exhausted, message: "Exhausted" + decision = Rules.decide State.new(status: :starting), req_failed, @config + assert_equal :starting, decision.from_status + assert_equal :request_retries_exhausted, decision.shape + assert_equal :fail_with_request_error, decision.recipe + assert_equal :error, decision.next_state.status + assert_recipe_progress_notification decision + assert_instance_of Instruction::TerminateFailure, decision.instructions.first + end + + private + + def assert_recipe_progress_notification decision + if Rules::RECIPE_PHASES.key? decision.recipe + expected_phase = Rules::RECIPE_PHASES[decision.recipe] + first_inst = decision.instructions.first + assert_instance_of Instruction::NotifyProgress, first_inst, + "Expected #{decision.recipe} to emit NotifyProgress as first instruction" + assert_equal expected_phase, first_inst.progress.phase, + "Expected #{decision.recipe} to emit phase #{expected_phase}" + else + assert_includes Rules::NON_NOTIFYING_RECIPES, decision.recipe + refute decision.instructions.any? { |i| i.is_a? Instruction::NotifyProgress }, + "Expected non-notifying recipe #{decision.recipe} to emit no NotifyProgress" + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb new file mode 100644 index 0000000..212deec --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_error_test.rb @@ -0,0 +1,474 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules terminal error transitions and actionable error formatting. +# +class RulesErrorTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_transition_starting_rejected + state = State.new status: :starting + resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Forbidden" + next_state, instructions = Rules.step state, resp, @config + + assert_equal :rejected, next_state.status + assert_instance_of UploadRejectedError, next_state.last_error + assert_equal "Forbidden", next_state.last_error.response_body + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_starting_fatal_error + state = State.new status: :starting + resp = Event::HttpResponse.new status: 400, headers: {}, body: "Bad Request" + next_state, instructions = Rules.step state, resp, @config + + assert_equal :error, next_state.status + assert_instance_of BadResponseError, next_state.last_error + assert_equal 400, next_state.last_error.status_code + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + end + + def test_transition_starting_request_failed + state = State.new status: :starting + err = StandardError.new "DNS resolution failed" + failed = Event::RequestFailed.new kind: :connection_failed, message: "DNS resolution failed", source_error: err + next_state, instructions = Rules.step state, failed, @config + + assert_equal :error, next_state.status + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_starting_timeout_terminates_failure + state = State.new status: :starting + err = StandardError.new "Read timeout" + req_failed = Event::RequestFailed.new kind: :timeout, message: "Read timeout", source_error: err + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :error, next_state.status + assert_equal 0, next_state.in_flight_length + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_transmission_sending_retries_exhausted_terminates_failure + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + err = StandardError.new "Retries exhausted" + req_failed = Event::RequestFailed.new kind: :retries_exhausted, message: "Retries exhausted", source_error: err + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :error, next_state.status + assert_equal 0, next_state.in_flight_length + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_recovery_timeout_terminates_failure + state = State.new status: :recovery, upload_url: "https://example.com/session" + err = StandardError.new "Query read timeout" + req_failed = Event::RequestFailed.new kind: :timeout, message: "Query read timeout", source_error: err + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :error, next_state.status + assert_equal 0, next_state.in_flight_length + assert_instance_of RequestFailedError, next_state.last_error + assert_equal err, next_state.last_error.cause + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + assert_equal next_state.last_error, instructions.first.error + end + + def test_transition_global_deadline_exceeded + state = State.new status: :transmission_sending, upload_url: "https://example.com/session" + next_state, instructions = Rules.step state, Event::GlobalDeadlineExceeded.new, @config + + assert_equal :error, next_state.status + assert_instance_of DeadlineExceededError, next_state.last_error + assert_equal 1, instructions.size + assert_instance_of Instruction::TerminateFailure, instructions.first + end + + def test_invalid_transition_raises_actionable_error_with_response_details_and_header + state = State.new status: :transmission_sending + resp = Event::HttpResponse.new status: 200, headers: { "X-Goog-Upload-Status" => "final" }, body: '{"done":true}' + + err = assert_raises InvalidTransitionError do + Rules.step state, resp, @config + end + + expected_msg = "Resumable upload failed while sending a chunk of data: " \ + "received an unexpected HTTP 200 response (X-Goog-Upload-Status: 'final')." + assert_equal expected_msg, err.message + assert_equal :transmission_sending, err.state + assert_equal resp, err.event + assert_equal resp, err.response + end + + def test_invalid_transition_shows_missing_when_upload_status_header_absent + state = State.new status: :transmission_reading + resp = Event::HttpResponse.new status: 200, headers: {}, body: "" + + err = assert_raises InvalidTransitionError do + Rules.step state, resp, @config + end + + expected_msg = "Resumable upload failed while reading chunk from stream: " \ + "received an unexpected HTTP 200 response (X-Goog-Upload-Status: missing)." + assert_equal expected_msg, err.message + assert_equal :transmission_reading, err.state + assert_equal resp, err.response + end + + def test_invalid_transition_raises_error_for_non_http_event + state = State.new status: :starting + event = Event::ChunkRead.new bytes_buffered: 512, eof: false + + err = assert_raises InvalidTransitionError do + Rules.step state, event, @config + end + + expected_msg = "Resumable upload failed while initiating upload session: " \ + "received unexpected stream chunk read (512 bytes, eof: false)." + assert_equal expected_msg, err.message + assert_equal :starting, err.state + assert_equal event, err.event + assert_nil err.response + end + + def test_rejected_with_wrapped_error_deprefixes_message_and_preserves_metadata + details = [{ "reason" => "ACCESS_DENIED" }] + headers = { "x-goog-upload-status" => "final", "content-type" => "application/json" } + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: The caller does not have permission", + 403, + status: "PERMISSION_DENIED", + details: details, + headers: headers + ) + resp = Event::HttpResponse.new( + status: 403, + headers: headers, + body: '{"error":{"message":"The caller does not have permission"}}', + error: wrapped_err + ) + + state = State.new status: :starting + next_state, instructions = Rules.step state, resp, @config + + assert_equal :rejected, next_state.status + err = next_state.last_error + assert_instance_of UploadRejectedError, err + assert_equal "Upload rejected by server with HTTP 403 PERMISSION_DENIED: The caller does not have permission", + err.message + assert_equal 403, err.status_code + assert_equal "PERMISSION_DENIED", err.status + assert_equal details, err.details + assert_equal details, err.status_details + assert_equal headers, err.headers + assert_equal headers, err.header + assert_equal '{"error":{"message":"The caller does not have permission"}}', err.response_body + assert_equal err, instructions.first.error + end + + def test_rejected_fallback_without_wrapped_error + resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, body: "Forbidden" + state = State.new status: :starting + next_state, _instructions = Rules.step state, resp, @config + + assert_equal :rejected, next_state.status + err = next_state.last_error + assert_instance_of UploadRejectedError, err + assert_equal "Upload rejected by server with HTTP 403 Forbidden (X-Goog-Upload-Status: 'final')", err.message + assert_equal 403, err.status_code + assert_equal "Forbidden", err.response_body + end + + def test_bad_response_with_wrapped_error_deprefixes_message_and_preserves_metadata + details = ["Quota limit details"] + headers = { "x-goog-upload-status" => "active" } + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Quota limit reached", + 429, + status: "RESOURCE_EXHAUSTED", + details: details, + headers: headers + ) + resp = Event::HttpResponse.new status: 429, headers: headers, body: "Too many requests", error: wrapped_err + + state = State.new status: :starting + next_state, _instructions = Rules.step state, resp, @config + + assert_equal :error, next_state.status + err = next_state.last_error + assert_instance_of BadResponseError, err + assert_equal "Resumable upload failed with HTTP 429 RESOURCE_EXHAUSTED: Quota limit reached", err.message + assert_equal 429, err.status_code + assert_equal "RESOURCE_EXHAUSTED", err.status + assert_equal details, err.status_details + assert_equal headers, err.headers + assert_equal "Too many requests", err.response_body + end + + def test_bad_response_fallback_without_wrapped_error + resp = Event::HttpResponse.new status: 503, headers: {}, body: "Service unavailable" + state = State.new status: :starting + next_state, _instructions = Rules.step state, resp, @config + + assert_equal :error, next_state.status + err = next_state.last_error + assert_instance_of BadResponseError, err + assert_equal "Resumable upload failed with HTTP 503 Service Unavailable (X-Goog-Upload-Status: missing)", err.message + assert_equal 503, err.status_code + assert_equal "Service unavailable", err.response_body + end + + def test_format_status_preserves_canonical_status_token + wrapped_err = Gapic::Rest::Error.new( + "#{Gapic::Rest::Error::REST_ERROR_PREFIX}: Object not found", + 404, + status: "NOT_FOUND", + headers: { "x-goog-upload-status" => "final" } + ) + resp = Event::HttpResponse.new status: 404, headers: { "x-goog-upload-status" => "final" }, + body: "Not found", error: wrapped_err + err = UploadRejectedError.from resp + assert_equal "Upload rejected by server with HTTP 404 NOT_FOUND: Object not found", err.message + assert_equal "NOT_FOUND", err.status + end + + def test_error_class_inheritance_hierarchy + assert_operator UploadCancelledError, :<, Gapic::Common::Error + refute_operator UploadCancelledError, :<, Gapic::Rest::Error + + assert_operator DeadlineExceededError, :<, Gapic::Common::Error + refute_operator DeadlineExceededError, :<, Gapic::Rest::Error + + assert_operator UploadRejectedError, :<, Gapic::Rest::Error + assert_operator BadResponseError, :<, Gapic::Rest::Error + + assert_operator StreamMismatchError, :<, Gapic::Common::Error + assert_operator RequestFailedError, :<, Gapic::Common::Error + assert_operator HasResumeHandle, :===, BadResponseError.new + assert_operator HasResumeHandle, :===, DeadlineExceededError.new + assert_operator HasResumeHandle, :===, UnseekableStreamError.new + assert_operator HasResumeHandle, :===, InvalidTransitionError.new("invalid") + assert_operator HasResumeHandle, :===, StreamMismatchError.new + assert_operator HasResumeHandle, :===, RequestFailedError.new("failed") + refute_operator HasResumeHandle, :===, UploadRejectedError.new + refute_operator HasResumeHandle, :===, UploadCancelledError.new + end + + def test_rules_resume_handle_from + assert_nil Rules.resume_handle_from(nil) + assert_nil Rules.resume_handle_from(State.new(status: :starting, upload_url: nil)) + assert_nil Rules.resume_handle_from(State.new(status: :rejected, upload_url: "https://upload.example.com/id123")) + assert_nil Rules.resume_handle_from(State.new(status: :cancelled, upload_url: "https://upload.example.com/id123")) + assert_nil Rules.resume_handle_from(State.new(status: :success, upload_url: "https://upload.example.com/id123")) + + state = State.new status: :transmission_sending, upload_url: "https://upload.example.com/id123", chunk_size: 1024 + handle = Rules.resume_handle_from state + refute_nil handle + assert_equal "https://upload.example.com/id123", handle.upload_url + assert_equal 1024, handle.chunk_size + end + + def test_resume_handle_present_on_errors_when_upload_url_set + state = State.new( + status: :transmission_sending, + upload_url: "https://upload.example.com/session_abc", + chunk_size: 512 + ) + + # 1. Deadline exceeded + next_state, = Rules.step state, Event::GlobalDeadlineExceeded.new, @config + assert_equal :error, next_state.status + deadline_err = next_state.last_error + assert_instance_of DeadlineExceededError, deadline_err + refute_nil deadline_err.resume_handle + assert_equal "https://upload.example.com/session_abc", deadline_err.resume_handle.upload_url + assert_equal 512, deadline_err.resume_handle.chunk_size + assert_includes deadline_err.message, "(upload session is resumable: see #resume_handle)" + + # 2. Bad response + resp = Event::HttpResponse.new status: 401, headers: {}, body: "Fatal 401" + next_state, = Rules.step state, resp, @config + assert_equal :error, next_state.status + bad_resp_err = next_state.last_error + assert_instance_of BadResponseError, bad_resp_err + refute_nil bad_resp_err.resume_handle + assert_equal "https://upload.example.com/session_abc", bad_resp_err.resume_handle.upload_url + assert_equal 512, bad_resp_err.resume_handle.chunk_size + assert_includes bad_resp_err.message, "(upload session is resumable: see #resume_handle)" + + # 3. Unmatched transition + unmatched_err = assert_raises InvalidTransitionError do + Rules.step state, Object.new, @config + end + refute_nil unmatched_err.resume_handle + assert_equal "https://upload.example.com/session_abc", unmatched_err.resume_handle.upload_url + assert_equal 512, unmatched_err.resume_handle.chunk_size + assert_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" + + # 4. Request failed (retries exhausted) + req_failed = Event::RequestFailed.new( + kind: :retries_exhausted, + message: "Connection reset", + source_error: StandardError.new("reset") + ) + next_state, = Rules.step state, req_failed, @config + assert_equal :error, next_state.status + req_err = next_state.last_error + assert_instance_of RequestFailedError, req_err + refute_nil req_err.resume_handle + assert_equal "https://upload.example.com/session_abc", req_err.resume_handle.upload_url + assert_equal 512, req_err.resume_handle.chunk_size + assert_equal "reset", req_err.cause.message + assert_includes req_err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_resume_handle_nil_on_errors_before_session_created + state = State.new status: :starting, upload_url: nil + + # 1. Deadline exceeded before session creation + next_state, = Rules.step state, Event::GlobalDeadlineExceeded.new, @config + assert_equal :error, next_state.status + deadline_err = next_state.last_error + assert_nil deadline_err.resume_handle + refute_includes deadline_err.message, "(upload session is resumable: see #resume_handle)" + + # 2. Bad response before session creation + resp = Event::HttpResponse.new status: 503, headers: {}, body: "Init failed" + next_state, = Rules.step state, resp, @config + assert_equal :error, next_state.status + bad_resp_err = next_state.last_error + assert_nil bad_resp_err.resume_handle + refute_includes bad_resp_err.message, "(upload session is resumable: see #resume_handle)" + + # 3. Unmatched transition before session creation + unmatched_err = assert_raises InvalidTransitionError do + Rules.step state, Object.new, @config + end + assert_nil unmatched_err.resume_handle + refute_includes unmatched_err.message, "(upload session is resumable: see #resume_handle)" + + # 4. Request failed before session creation + req_failed = Event::RequestFailed.new( + kind: :connection_failed, + message: "Connection reset", + source_error: StandardError.new("reset") + ) + next_state, = Rules.step state, req_failed, @config + assert_equal :error, next_state.status + req_err = next_state.last_error + assert_instance_of RequestFailedError, req_err + assert_nil req_err.resume_handle + refute_includes req_err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_resume_handle_absent_on_rejected_and_cancelled + state = State.new( + status: :transmission_sending, + upload_url: "https://upload.example.com/session_abc", + chunk_size: 512 + ) + + # 1. Rejected error + rejected_resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" }, + body: "Access Denied" + next_state, = Rules.step state, rejected_resp, @config + assert_equal :rejected, next_state.status + rejected_err = next_state.last_error + assert_instance_of UploadRejectedError, rejected_err + refute_respond_to rejected_err, :resume_handle + refute_includes rejected_err.message, "(upload session is resumable: see #resume_handle)" + + # 2. Cancelled error + cancelling_state = state.with status: :cancelling + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" }, + body: "" + next_state, = Rules.step cancelling_state, cancelled_resp, @config + assert_equal :cancelled, next_state.status + cancelled_err = next_state.last_error + assert_instance_of UploadCancelledError, cancelled_err + refute_respond_to cancelled_err, :resume_handle + refute_includes cancelled_err.message, "(upload session is resumable: see #resume_handle)" + end + + def test_stream_mismatch_error_behavior + handle = ResumeHandle.new upload_url: "https://upload.example.com/resume", chunk_size: 256 + err_with_handle = StreamMismatchError.new "Stream too short", resume_handle: handle + + assert_instance_of StreamMismatchError, err_with_handle + assert_equal handle, err_with_handle.resume_handle + assert_equal "Stream too short (upload session is resumable: see #resume_handle)", err_with_handle.message + + err_from = StreamMismatchError.from "Stream corrupted", resume_handle: handle + assert_equal handle, err_from.resume_handle + assert_equal "Stream corrupted (upload session is resumable: see #resume_handle)", err_from.message + + err_without_handle = StreamMismatchError.new "No handle" + assert_nil err_without_handle.resume_handle + assert_equal "No handle", err_without_handle.message + end + + def test_request_failed_error_behavior + handle = ResumeHandle.new upload_url: "https://upload.example.com/resume", chunk_size: 256 + cause = Gapic::Rest::Error.new "Underlying Faraday error", 500, status: "INTERNAL", details: ["foo"], headers: { "k" => "v" } + + err_with_handle = RequestFailedError.from cause, resume_handle: handle + assert_instance_of RequestFailedError, err_with_handle + assert_equal cause, err_with_handle.cause + assert_equal handle, err_with_handle.resume_handle + assert_equal 500, err_with_handle.status_code + assert_equal "INTERNAL", err_with_handle.status + assert_equal ["foo"], err_with_handle.details + assert_equal({ "k" => "v" }, err_with_handle.headers) + assert_equal "Underlying Faraday error (upload session is resumable: see #resume_handle)", err_with_handle.message + + err_without_handle = RequestFailedError.from cause + assert_nil err_without_handle.resume_handle + assert_equal "Underlying Faraday error", err_without_handle.message + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb new file mode 100644 index 0000000..9c67f0d --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_recovery_test.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules protocol recovery state transitions. +# +class RulesRecoveryTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_transition_transmission_sending_cat2_triggers_recovery + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + resp = Event::HttpResponse.new status: 503, headers: {} + next_state, instructions = Rules.step state, resp, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_transmission_sending_connection_failed_triggers_recovery + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + req_failed = Event::RequestFailed.new kind: :connection_failed, message: "Network unreachable" + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_transmission_sending_timeout_triggers_recovery + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512 + req_failed = Event::RequestFailed.new kind: :timeout, message: "Read timeout" + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_finalizing_sending_upload_timeout_triggers_recovery + state = State.new status: :finalizing_sending_upload, upload_url: "https://example.com/session", offset: 512, + in_flight_length: 512 + req_failed = Event::RequestFailed.new kind: :timeout, message: "Read timeout" + next_state, instructions = Rules.step state, req_failed, @config + + assert_equal :recovery, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :recovering, bytes_uploaded: 512, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendQuery, instructions[1] + end + + def test_transition_recovery_active_realigns_buffer + state = State.new status: :recovery, upload_url: "https://example.com/session", offset: 0, chunk_size: 512 + headers = { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "768" + } + resp = Event::HttpResponse.new status: 200, headers: headers + next_state, instructions = Rules.step state, resp, @config + + assert_equal :transmission_reading, next_state.status + assert_equal 768, next_state.offset + assert_equal 3, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 768, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::RealignBuffer, instructions[1] + assert_equal 768, instructions[1].server_offset + assert_instance_of Instruction::FillBuffer, instructions[2] + end + + def test_transition_recovery_final_completes_upload + state = State.new status: :recovery, upload_url: "https://example.com/session", offset: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + next_state, instructions = Rules.step state, resp, @config + + assert_equal :success, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :completed, bytes_uploaded: 512, total_bytes: 512), instructions[0].progress + assert_instance_of Instruction::TerminateSuccess, instructions[1] + end + + def test_transition_recovery_cat2_retries_query + state = State.new status: :recovery, upload_url: "https://example.com/session" + resp = Event::HttpResponse.new status: 416, headers: {} + next_state, instructions = Rules.step state, resp, @config + + assert_equal :recovery, next_state.status + assert_equal 1, instructions.size + assert_instance_of Instruction::SendQuery, instructions.first + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb b/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb new file mode 100644 index 0000000..0c8031f --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/rules_test.rb @@ -0,0 +1,279 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +## +# Tests for ResumableUpload Rules normal progression and session lifecycle state transitions. +# +class RulesTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + def setup + @config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + initial_headers: { "X-Custom" => "value" }, + initial_body: '{"name":"obj"}', + stream: StringIO.new("data"), + upload_size: 1024, + chunk_size: 512 + ) + end + + def test_transition_initializing_to_starting + state = State.new status: :initializing + next_state, instructions = Rules.step state, Event::StartUpload.new, @config + + assert_equal :starting, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :initiating, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendStart, instructions[1] + assert_equal "https://example.com/upload", instructions[1].url + assert_equal({ "X-Custom" => "value" }, instructions[1].headers) + assert_equal '{"name":"obj"}', instructions[1].body + end + + def test_transition_starting_to_transmission_reading + state = State.new status: :starting + headers = { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://example.com/session", + "x-goog-upload-chunk-granularity" => "256" + } + resp = Event::HttpResponse.new status: 200, headers: headers + next_state, instructions = Rules.step state, resp, @config + + assert_equal :transmission_reading, next_state.status + assert_equal "https://example.com/session", next_state.upload_url + assert_equal 256, next_state.chunk_granularity + assert_equal 512, next_state.chunk_size + assert_equal 0, next_state.offset + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::FillBuffer, instructions[1] + assert_equal 512, instructions[1].target_bytesize + end + + def test_transition_transmission_reading_full_chunk + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 0, + chunk_size: 512 + event = Event::ChunkRead.new bytes_buffered: 512, eof: false + next_state, instructions = Rules.step state, event, @config + + assert_equal :transmission_sending, next_state.status + assert_equal 512, next_state.in_flight_length + assert_equal 1, instructions.size + assert_instance_of Instruction::SendChunk, instructions.first + assert_equal 0, instructions.first.offset + assert_equal 512, instructions.first.length + refute instructions.first.finalize + end + + def test_transition_transmission_reading_eof_with_data + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 512, + chunk_size: 512 + event = Event::ChunkRead.new bytes_buffered: 200, eof: true + next_state, instructions = Rules.step state, event, @config + + assert_equal :finalizing_sending_upload, next_state.status + assert_equal 200, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :finalizing, bytes_uploaded: 512, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendChunk, instructions[1] + assert_equal 512, instructions[1].offset + assert_equal 200, instructions[1].length + assert instructions[1].finalize + end + + def test_transition_transmission_reading_eof_empty + state = State.new status: :transmission_reading, upload_url: "https://example.com/session", offset: 1024, + chunk_size: 512 + event = Event::ChunkRead.new bytes_buffered: 0, eof: true + next_state, instructions = Rules.step state, event, @config + + assert_equal :finalizing_sending_finalize, next_state.status + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :finalizing, bytes_uploaded: 1024, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendFinalize, instructions[1] + assert_equal "https://example.com/session", instructions[1].url + end + + def test_transition_transmission_sending_ack_chunk + state = State.new status: :transmission_sending, upload_url: "https://example.com/session", offset: 0, + in_flight_length: 512, chunk_size: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "active" } + next_state, instructions = Rules.step state, resp, @config + + assert_equal :transmission_reading, next_state.status + assert_equal 512, next_state.offset + assert_equal 0, next_state.in_flight_length + assert_equal 3, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :uploading, bytes_uploaded: 512, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::RealignBuffer, instructions[1] + assert_equal 512, instructions[1].server_offset + assert_instance_of Instruction::FillBuffer, instructions[2] + assert_equal 512, instructions[2].target_bytesize + end + + def test_transition_finalizing_sending_upload_success + state = State.new status: :finalizing_sending_upload, offset: 512, in_flight_length: 512 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"done":true}' + next_state, instructions = Rules.step state, resp, @config + + assert_equal :success, next_state.status + assert_equal 1024, next_state.offset + assert_equal 0, next_state.in_flight_length + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :completed, bytes_uploaded: 1024, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::TerminateSuccess, instructions[1] + end + + def test_transition_finalizing_sending_finalize_success + state = State.new status: :finalizing_sending_finalize, offset: 1024, in_flight_length: 0 + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"done":true}' + next_state, instructions = Rules.step state, resp, @config + + assert_equal :success, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :completed, bytes_uploaded: 1024, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::TerminateSuccess, instructions[1] + end + + def test_transition_cancellation_flow + state = State.new status: :transmission_sending, upload_url: "https://example.com/session" + next_state, instructions = Rules.step state, Event::Cancel.new, @config + + assert_equal :cancelling, next_state.status + assert_equal 2, instructions.size + assert_instance_of Instruction::NotifyProgress, instructions[0] + assert_equal Progress.new(phase: :cancelling, bytes_uploaded: 0, total_bytes: 1024), instructions[0].progress + assert_instance_of Instruction::SendCancel, instructions[1] + + # A duplicate cancel in :cancelling raises InvalidTransitionError + assert_raises InvalidTransitionError do + Rules.step next_state, Event::Cancel.new, @config + end + + # Cancellation confirmed + resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + final_state, final_instructions = Rules.step next_state, resp, @config + assert_equal :cancelled, final_state.status + assert_instance_of UploadCancelledError, final_state.last_error + assert_equal 1, final_instructions.size + assert_instance_of Instruction::TerminateFailure, final_instructions.first + end + + def test_all_recipes_respond_to_rules_method + Rules::RECIPES.each do |recipe| + assert_respond_to Rules, recipe + end + end + + def test_all_recipes_satisfy_trampoline_invariant + resume_config = ResumeUploadConfig.new( + upload_url: "https://example.com/session", + chunk_size: 256, + stream: StringIO.new("abcd") + ) + active_resp = Event::HttpResponse.new status: 200, headers: { + "x-goog-upload-url" => "https://example.com/session", + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "256" + } + final_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "final" } + cancelled_resp = Event::HttpResponse.new status: 200, headers: { "x-goog-upload-status" => "cancelled" } + cat2_resp = Event::HttpResponse.new status: 503, headers: {} + rejected_resp = Event::HttpResponse.new status: 403, headers: { "x-goog-upload-status" => "final" } + bad_resp = Event::HttpResponse.new status: 401, headers: {} + req_err = Event::RequestFailed.new kind: :connection_failed, message: "connection lost" + chunk_full = Event::ChunkRead.new bytes_buffered: 256, eof: false + chunk_eof = Event::ChunkRead.new bytes_buffered: 256, eof: true + base_state = State.new( + status: :transmission_sending, + upload_url: "https://example.com/session", + chunk_size: 256, + in_flight_length: 256 + ) + + fixtures = { + start_session: [State.new(status: :initializing), Event::StartUpload.new, @config], + resume_session: [State.new(status: :initializing), Event::ResumeUpload.new, resume_config], + begin_transmission: [State.new(status: :starting), active_resp, @config], + send_chunk: [base_state.with(status: :transmission_reading), chunk_full, @config], + send_upload_finalize: [base_state.with(status: :transmission_reading), chunk_eof, @config], + send_finalize: [base_state.with(status: :transmission_reading), Event::ChunkRead.new(bytes_buffered: 0, eof: true), @config], + ack_chunk: [base_state, active_resp, @config], + enter_recovery: [base_state, cat2_resp, @config], + retry_recovery: [base_state.with(status: :recovery), cat2_resp, @config], + realign_from_recovery: [base_state.with(status: :recovery), active_resp, @config], + complete_upload_with_data: [base_state.with(status: :finalizing_sending_upload), final_resp, @config], + complete_upload_finalized: [base_state.with(status: :finalizing_sending_finalize), final_resp, @config], + cancel_session: [base_state, Event::Cancel.new, @config], + complete_cancellation: [base_state.with(status: :cancelling), cancelled_resp, @config], + fail_with_deadline_exceeded: [base_state, Event::GlobalDeadlineExceeded.new, @config], + fail_with_rejected: [base_state, rejected_resp, @config], + fail_with_bad_response: [base_state, bad_resp, @config], + fail_with_request_error: [base_state, req_err, @config], + fail_with_unmatched_transition: [State.new(status: :success), Event::StartUpload.new, @config] + } + + assert_equal Rules::RECIPES.sort, fixtures.keys.sort + + fixtures.each do |recipe, (state, event, cfg)| + if recipe == :fail_with_unmatched_transition + assert_raises InvalidTransitionError do + Rules.public_send recipe, state, event, cfg + end + next + end + + _next_state, instructions = Rules.public_send recipe, state, event, cfg + event_producing_count = instructions.count { |inst| Instruction::CONTINUATION.include? inst.class } + terminal_count = instructions.count { |inst| Instruction::TERMINAL.include? inst.class } + + valid = (event_producing_count == 1 && terminal_count.zero?) || + (event_producing_count.zero? && terminal_count == 1) + assert valid, "Recipe :#{recipe} produced #{event_producing_count} event-producing and #{terminal_count} terminal instructions" + end + end + + def test_instruction_constants_partition_all_instruction_classes + all_classes = Instruction.constants(false).map { |name| Instruction.const_get name }.grep(Class) + partition_union = Instruction::CONTINUATION + Instruction::TERMINAL + Instruction::SIDE_EFFECT + + assert_empty all_classes - partition_union, + "Instruction classes missing from CONTINUATION/TERMINAL/SIDE_EFFECT partition" + assert_empty partition_union - all_classes, + "Partition contains classes that are not Instruction classes" + assert_empty Instruction::CONTINUATION & Instruction::TERMINAL + assert_empty Instruction::CONTINUATION & Instruction::SIDE_EFFECT + assert_empty Instruction::TERMINAL & Instruction::SIDE_EFFECT + assert_predicate Instruction::CONTINUATION, :frozen? + assert_predicate Instruction::TERMINAL, :frozen? + assert_predicate Instruction::SIDE_EFFECT, :frozen? + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/session_test.rb b/gapic-common/test/gapic/rest/resumable_upload/session_test.rb new file mode 100644 index 0000000..fe6e782 --- /dev/null +++ b/gapic-common/test/gapic/rest/resumable_upload/session_test.rb @@ -0,0 +1,704 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest/resumable_upload" +require "stringio" + +class SessionTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + class ScriptedClientStub + attr_reader :requests + + def initialize responses = [] + @responses = responses.dup + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @responses.empty? + + res = @responses.shift + if res.is_a? Proc + res.call + elsif res.is_a? Exception + raise res + else + res + end + end + end + + class UnseekableStream + attr_reader :pos + + def initialize string + @io = StringIO.new string + @pos = 0 + end + + def read length = nil + chunk = @io.read length + @pos += chunk.bytesize if chunk + chunk + end + end + + class StreamWithoutPos + def initialize string + @io = StringIO.new string + end + + def read length = nil + @io.read length + end + end + + START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze + + # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, + # so tests can keep calling `start_session session`. + def build_session stub: nil, stream: nil, upload_size: 10, chunk_size: 4, **kwargs + stream ||= StringIO.new "0123456789" + stub ||= ScriptedClientStub.new + @start_args = { + initial_url: "https://example.com/initiate", + initial_body: '{"name":"test.txt"}', + chunk_size: chunk_size + }.merge(kwargs.slice(*START_ONLY_KEYS)) + + Session.new( + client_stub: stub, + stream: stream, + upload_size: upload_size, + **kwargs.except(*START_ONLY_KEYS) + ) + end + + def start_session session, **overrides + session.start(**@start_args, **overrides) + end + + # ============================================================================ + # 1. Initialization and argument validation + # ============================================================================ + + def test_initialize_mandatory_arguments + assert_raises ArgumentError do + Session.new stream: StringIO.new + end + + assert_raises ArgumentError do + Session.new client_stub: ScriptedClientStub.new + end + end + + def test_initialize_rejects_per_run_arguments + assert_raises ArgumentError do + Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, initial_url: "http://x" + end + + assert_raises ArgumentError do + Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, chunk_size: 4 + end + end + + def test_initialize_defaults + session = Session.new( + client_stub: ScriptedClientStub.new, + stream: StringIO.new("abc"), + upload_size: 300 + ) + + assert_equal 300, session.upload_size + assert_nil session.content_type + assert_nil session.timeout + assert_nil session.control_plane_retry_policy + assert_nil session.data_plane_retry_policy + assert_nil session.on_progress + assert_nil session.logger + end + + def test_start_without_initial_url_raises_argument_error + session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 + + error = assert_raises ArgumentError do + session.start + end + assert_match(/initial_url/, error.message) + end + + def test_start_with_blank_initial_url_raises_argument_error + session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 + + error = assert_raises ArgumentError do + session.start initial_url: " " + end + assert_match(/initial_url is required/, error.message) + end + + def test_start_with_malformed_retry_policy_raises_argument_error + session = build_session + + error = assert_raises ArgumentError do + start_session session, start_retry_policy: "nonsense" + end + assert_match(/Expected RetryPolicy, Hash, or nil/, error.message) + end + + def test_start_with_reserved_initial_header_raises_before_any_request + stub = ScriptedClientStub.new + session = build_session stub: stub + + error = assert_raises ArgumentError do + start_session session, initial_headers: { "X-Goog-Upload-Header-Content-Type" => "image/png" } + end + assert_match(/must not set protocol header/, error.message) + assert_empty stub.requests + refute session.bound? + end + + def test_failed_start_leaves_session_reusable + session = build_session + + assert_raises ArgumentError do + start_session session, start_retry_policy: "nonsense" + end + + refute session.running? + refute session.bound? + end + + def test_resume_needs_no_initiation_arguments + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"resumed":true}' + ) + ] + session = Session.new( + client_stub: ScriptedClientStub.new(responses), + stream: StringIO.new("01"), + upload_size: 2 + ) + handle = ResumeHandle.new upload_url: "https://upload.example.com/persisted", chunk_size: 4 + + result = session.resume resume_handle: handle + + assert_equal '{"resumed":true}', result + assert_equal "https://upload.example.com/persisted", session.upload_url + end + + # ============================================================================ + # 2. Observable States: Unbound & Bound + # ============================================================================ + + def test_initial_unbound_state + session = build_session + refute session.bound? + assert_nil session.upload_url + assert_nil session.resume_handle + refute session.resumable? + refute session.running? + end + + # ============================================================================ + # 3. Start Lifecycle & Single-Run Contract + # ============================================================================ + + def test_start_successful_upload_transitions_to_bound + responses = [ + # Initiation response + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_1", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + # Chunk 1 (0-3) + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + # Chunk 2 (4-7) + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active" }, + body: "" + ), + # Final Chunk (8-9) + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"status":"completed"}' + ) + ] + + stub = ScriptedClientStub.new responses + session = build_session stub: stub, upload_size: 10, chunk_size: 4 + + result = start_session session + + assert_equal '{"status":"completed"}', result + assert session.bound? + assert_equal "https://upload.example.com/session_1", session.upload_url + refute session.running? + refute session.resumable? + assert_nil session.resume_handle + end + + def test_second_start_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_1", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"done":true}' + ) + ] + session = build_session( + stub: ScriptedClientStub.new(responses), + stream: StringIO.new("01"), + upload_size: 2, + chunk_size: 4 + ) + start_session session + + assert session.bound? + + err = assert_raises SessionStateError do + start_session session + end + assert_includes err.message, "Session has already executed a run" + end + + def test_resume_after_start_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_1", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"done":true}' + ) + ] + session = build_session( + stub: ScriptedClientStub.new(responses), + stream: StringIO.new("01"), + upload_size: 2, + chunk_size: 4 + ) + start_session session + + assert_nil session.resume_handle + refute session.resumable? + + handle = ResumeHandle.new upload_url: "https://upload.example.com/session_1", chunk_size: 4 + err = assert_raises SessionStateError do + session.resume resume_handle: handle + end + assert_includes err.message, "Session has already executed a run" + end + + # ============================================================================ + # 4. Resume Forms: Explicit URL or ResumeHandle + # ============================================================================ + + def test_resume_explicit_url_and_chunk_size_binds_and_executes + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"from_url":true}' + ) + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + + refute session.bound? + result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + + assert_equal '{"from_url":true}', result + assert session.bound? + assert_equal "https://upload.example.com/direct", session.upload_url + end + + def test_resume_resume_handle_binds_and_executes + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"from_handle":true}' + ) + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + + handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 + + refute session.bound? + result = session.resume resume_handle: handle + + assert_equal '{"from_handle":true}', result + assert session.bound? + assert_equal "https://upload.example.com/from_handle", session.upload_url + end + + def test_start_after_resume_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + + assert session.bound? + err = assert_raises SessionStateError do + start_session session + end + assert_includes err.message, "Session has already executed a run" + end + + def test_second_resume_raises_session_state_error + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 + session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + + assert session.bound? + err = assert_raises SessionStateError do + session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + end + assert_includes err.message, "Session has already executed a run" + end + + # ============================================================================ + # 5. Argument Shape & Preconditions + # ============================================================================ + + def test_resume_without_arguments_raises_argument_error + session = build_session + err = assert_raises ArgumentError do + session.resume + end + assert_includes err.message, "Must provide either resume_handle or upload_url and chunk_size" + end + + def test_resume_mixing_arguments_raises_argument_error + session = build_session + handle = ResumeHandle.new upload_url: "https://example.com", chunk_size: 4 + + assert_raises ArgumentError do + session.resume resume_handle: handle, upload_url: "https://example.com" + end + + assert_raises ArgumentError do + session.resume resume_handle: handle, chunk_size: 4 + end + + assert_raises ArgumentError do + session.resume upload_url: "https://example.com" + end + + assert_raises ArgumentError do + session.resume chunk_size: 4 + end + + assert_raises ArgumentError do + session.resume handle + end + end + + def test_resume_with_non_zero_stream_pos_raises_argument_error + stream = StringIO.new "0123456789" + stream.seek 4 + + session = build_session stream: stream + handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 + + err = assert_raises ArgumentError do + session.resume resume_handle: handle + end + assert_includes err.message, "Stream must be positioned at byte 0 to resume an upload (got pos 4)" + end + + def test_resume_with_stream_without_pos_is_trusted + stream = StreamWithoutPos.new "01" + responses = [ + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') + ] + stub = ScriptedClientStub.new responses + session = build_session stub: stub, stream: stream, upload_size: 2, chunk_size: 4 + + result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 + assert_equal '{"ok":true}', result + end + + # ============================================================================ + # 6. Cross-Session Resumption + # ============================================================================ + + def test_cross_session_resumption_from_failed_run + stream = StringIO.new "0123456789" + stub1 = ScriptedClientStub.new [ + # Initiation succeeds + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_cross", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ), + # Chunk 1 returns 503 + FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), + # Recovery query fails + Faraday::ConnectionFailed.new("network connection failed") + ] + + session1 = build_session stub: stub1, stream: stream, upload_size: 10, chunk_size: 4 + raised = assert_raises RequestFailedError do + start_session session1 + end + + assert session1.bound? + assert session1.resumable? + handle = session1.resume_handle + refute_nil handle + assert_equal handle, raised.resume_handle + assert_equal "https://upload.example.com/session_cross", handle.upload_url + assert_equal 4, handle.chunk_size + + # Prepare for session 2: rewind the stream to byte 0 + stream.rewind + assert_equal 0, stream.pos + + stub2 = ScriptedClientStub.new [ + # Recovery query on resume: server acknowledges 0 bytes received + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-size-received" => "0" + }, + body: "" + ), + # Chunk 1 + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), + # Chunk 2 + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), + # Chunk 3 (final) + FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"resumed":true}') + ] + + session2 = build_session stub: stub2, stream: stream, upload_size: 10, chunk_size: 4 + refute session2.bound? + + result = session2.resume resume_handle: handle + assert_equal '{"resumed":true}', result + assert session2.bound? + refute session2.resumable? + assert_nil session2.resume_handle + end + + # ============================================================================ + # 7. Concurrency & Running Guard + # ============================================================================ + + def test_running_guard_prevents_concurrent_runs + started_q = Queue.new + unblock_q = Queue.new + + blocking_proc = proc do + started_q.push :started + unblock_q.pop # wait until test signals to proceed + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => "https://upload.example.com/session_block", + "x-goog-upload-chunk-granularity" => "4" + }, + body: "" + ) + end + + stub = ScriptedClientStub.new [ + blocking_proc, + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "final" }, + body: '{"done":true}' + ) + ] + session = build_session( + stub: stub, + stream: StringIO.new("01"), + upload_size: 2, + chunk_size: 4 + ) + + worker = Thread.new do + start_session session + end + + started_q.pop # wait for worker thread to enter driver.run + assert session.running? + + # Concurrent call from another thread raises SessionStateError + handle = ResumeHandle.new upload_url: "https://upload.example.com/session_block", chunk_size: 4 + err = assert_raises SessionStateError do + session.resume resume_handle: handle + end + assert_includes err.message, "A run is already in progress for this session" + + err_start = assert_raises SessionStateError do + start_session session + end + assert_includes err_start.message, "A run is already in progress for this session" + + # Unblock worker thread + unblock_q.push :continue + result = worker.value + + assert_equal '{"done":true}', result + refute session.running? + end + + # ============================================================================ + # 8. Driver#upload_url Direct Verification + # ============================================================================ + + def test_driver_upload_url_across_statuses + dummy_client = ScriptedClientStub.new + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 4 + ) + driver = Driver.new client_stub: dummy_client, config: config + + assert_nil driver.upload_url + + # Active + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :transmission_sending, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url + + # Rejected (resume_handle is nil, but upload_url remains readable) + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + + # Cancelled (resume_handle is nil, but upload_url remains readable) + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + + # Success (resume_handle is nil, but upload_url remains readable) + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: :success, upload_url: "https://upload.example.com/sess1") + ) + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + end +end diff --git a/gapic-common/test/test_helper.rb b/gapic-common/test/test_helper.rb index ea4cab9..fcf5283 100644 --- a/gapic-common/test/test_helper.rb +++ b/gapic-common/test/test_helper.rb @@ -15,6 +15,7 @@ gem "minitest" require "minitest/autorun" require "minitest/focus" +require "minitest/mock" require "minitest/rg" require "pp" @@ -127,3 +128,27 @@ def spoof_logging_env enabled: nil, cloud_run: false ensure ENV["GOOGLE_SDK_RUBY_LOGGING_GEMS"] = old_enabled end + +class RecordingLogger < Logger + Entry = Data.define :severity, :message + + attr_reader :entries + + def initialize + super nil + @entries = [] + end + + def add severity, message = nil, progname = nil + msg = block_given? ? yield : (message || progname) + @entries << Entry.new(severity: severity, message: msg) + end +end + +def log_corpus recording_logger + formatter = Google::Logging::StructuredFormatter.new + recording_logger.entries.map do |entry| + sev = Logger::SEV_LABEL[entry.severity] || "INFO" + formatter.call sev, Time.now, nil, entry.message + end.join +end From 450a2dac234f9e8823195bb7326a5286f224b9c6 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Tue, 15 Sep 2026 06:50:50 +0000 Subject: [PATCH 2/4] chore: fix yardoc invocation --- gapic-common/.toys.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gapic-common/.toys.rb b/gapic-common/.toys.rb index 0b16a3f..f16b2b0 100644 --- a/gapic-common/.toys.rb +++ b/gapic-common/.toys.rb @@ -28,7 +28,7 @@ t.fail_on_undocumented_objects = false # TODO: Fix so this can be enabled t.bundler = true end -alias_tool :yard, :yardoc +tool "yard", delegate_to: "yardoc" expand :gem_build From 9f50adcac910c8a0090f43625d622193338dde12 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Tue, 15 Sep 2026 07:12:28 +0000 Subject: [PATCH 3/4] chore: fix toys (again) --- gapic-common/.toys.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gapic-common/.toys.rb b/gapic-common/.toys.rb index f16b2b0..4bb826c 100644 --- a/gapic-common/.toys.rb +++ b/gapic-common/.toys.rb @@ -30,10 +30,6 @@ end tool "yard", delegate_to: "yardoc" -expand :gem_build - -expand :gem_build, name: "install", install_gem: true - tool "ci" do include :exec, e: true include :terminal From ce62f2d4a47c3d150d4ea8af55478a4a00d4b682 Mon Sep 17 00:00:00 2001 From: Viacheslav Rostovtsev Date: Thu, 17 Sep 2026 19:37:27 +0000 Subject: [PATCH 4/4] feat: remove Session and replace with ::Gapic::ResumableUpload, and trim public surface --- .../resumable_upload/implementation-guide.md | 122 +-- .../resumable_upload/integration-test-plan.md | 16 +- .../integration/integration_helper.rb | 54 +- .../resumable_upload/resume_test.rb | 116 +-- gapic-common/lib/gapic/rest.rb | 1 + .../lib/gapic/rest/resumable_upload.rb | 109 ++- .../gapic/rest/resumable_upload/data_types.rb | 18 +- .../lib/gapic/rest/resumable_upload/driver.rb | 16 +- .../lib/gapic/rest/resumable_upload/errors.rb | 4 +- .../rest/resumable_upload/retry_policies.rb | 6 +- .../gapic/rest/resumable_upload/session.rb | 546 -------------- gapic-common/lib/gapic/resumable_upload.rb | 507 +++++++++++++ .../resumable_upload/driver_logging_test.rb | 28 + .../rest/resumable_upload/driver_test.rb | 31 + .../resumable_upload/retry_policies_test.rb | 64 ++ .../rest/resumable_upload/session_test.rb | 704 ------------------ .../test/gapic/resumable_upload_test.rb | 625 ++++++++++++++++ 17 files changed, 1528 insertions(+), 1439 deletions(-) delete mode 100644 gapic-common/lib/gapic/rest/resumable_upload/session.rb create mode 100644 gapic-common/lib/gapic/resumable_upload.rb delete mode 100644 gapic-common/test/gapic/rest/resumable_upload/session_test.rb create mode 100644 gapic-common/test/gapic/resumable_upload_test.rb diff --git a/gapic-common/design/resumable_upload/implementation-guide.md b/gapic-common/design/resumable_upload/implementation-guide.md index 3a561a9..8ca3f1a 100644 --- a/gapic-common/design/resumable_upload/implementation-guide.md +++ b/gapic-common/design/resumable_upload/implementation-guide.md @@ -20,9 +20,8 @@ graph TD ### 1.0 Domain Vocabulary * **Upload**: Server-side entity created by a successful session initiation (`start`), identified by `upload_url`. * **Resume Handle (`ResumeHandle`)**: An immutable snapshot (`upload_url`, `chunk_size`) identifying an upload for resumption. -* **Session (`Session`)**: Client-side transfer coordinator; performs exactly one run (`start` or `resume`), never both, never twice. It owns the input stream and configuration options. -* **Run**: One invocation of `Driver#run` (either a start or resume execution). -* **Bound**: The property that a session has executed a run or is bound to an upload (`session.bound?`). A session becomes bound when `start` or `resume` begins execution. A bound session never runs again. +* **Coordinator (`::Gapic::ResumableUpload`)**: Client-side transfer coordinator and the only public entry point. Reusable: it builds one `Driver` per run and retains the last one. Per-run arguments — the stream, sizes, the upload budget, the progress callback — are passed to the run, not to the coordinator. +* **Run**: One invocation of `Driver#run` (either a start or resume execution). At most one run per coordinator at a time. ### 1.1 Driver (Synchronous I/O Adapter) The `Driver` executes all operations with side-effects. It interacts with HTTP transport via `Gapic::Rest::ClientStub`, reads binary data from local input streams, tracks monotonic execution deadlines, and dispatches progress callbacks. @@ -42,60 +41,71 @@ The `Rules` module encapsulates the Resumable Upload Protocol state transitions ### 1.4 Stream Buffering Because arbitrary Ruby `IO` objects (network sockets, pipes, `STDIN`) do not support seeking (`#seek`), the Driver buffers the current in-flight chunk in memory (bounded by chunk size, default: 8MB). When `RetryPolicy` executes transport retries, or when `Core` triggers Category 2 recovery realignments within the buffered range, the Driver retransmits directly from memory. The buffer is discarded only after receiving a `200 OK` durably confirming receipt of the chunk. -### 1.5 Session (Transfer Coordinator) -The `Session` (`Gapic::Rest::ResumableUpload::Session`) provides a client-facing coordinator that encapsulates shared transfer configuration, owns the input stream, and manages upload execution across a strict single-run lifecycle. - -#### Single-Run Contract & Two-State Model -A session adheres to a two-state model with a strict single-run contract: a session performs exactly one run (`start` or `resume`), never both, never twice. - -1. **Unbound (`!session.bound?`)**: - * Initial state upon construction (`Session.new`). The session has not yet executed a run. - * Permitted operations: `start(...)` or `resume(...)`. -2. **Bound (`session.bound?`)**: - * Transitions to bound as soon as `start` or `resume` begins execution. - * The session has executed its run and cannot be reused. - * Both `start` and `resume` raise `SessionStateError` ("Session has already executed a run"). - -#### Constructor & Initiation Signatures -Configuration is split between transfer-wide options passed to `Session.new` (`COMMON_MEMBERS` plus `client_stub` and `logger`) and initiation-only arguments passed to `Session#start`: -* **Constructor (`Session#initialize`)**: - `Session.new(client_stub:, stream:, upload_size: nil, content_type: nil, timeout: nil, control_plane_retry_policy: nil, data_plane_retry_policy: nil, on_progress: nil, logger: nil)` -* **Initiation (`Session#start`)**: - `session.start(initial_url:, initial_body: nil, initial_headers: {}, chunk_size: nil, start_retry_policy: nil)` - * `initial_url` is required (`ArgumentError` if missing or blank). - * `initial_headers` accepts caller-supplied HTTP headers for the initiation request, merged over the driver's headers. Any key in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`, in any casing) raises an `ArgumentError`; callers influence `X-Goog-Upload-Header-Content-Type` and `X-Goog-Upload-Header-Content-Length` through `content_type:` and `upload_size:` on the constructor. Pass-through headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. - -#### Resumability (`session.resumable?`) -* Reports whether a *new* session can resume the transfer (`!session.resume_handle.nil?`). -* Completed uploads are finalized: once a transfer succeeds, `resume_handle` returns `nil` and `resumable?` returns `false`, so there is no handle to resume from. Calling `#resume` on the session that completed the run raises `SessionStateError`. Resuming a *fresh* session against a finalized `upload_url` is undefined behavior: it queries the server and might return the response body or raise an error, depending on the server response. -* When a run fails with a recoverable error, `resume_handle` captures the upload parameters (`upload_url`, `chunk_size`) and `resumable?` returns `true`. +### 1.5 Coordinator (`::Gapic::ResumableUpload`) +The coordinator (`lib/gapic/resumable_upload.rb`) is the only public entry point above `Driver`, and the object a generated client method returns instead of a response. It lives outside the `Gapic::Rest::ResumableUpload` namespace on purpose: it is not part of the protocol implementation, it is the layer built on top of it. It builds the per-run configuration, constructs one `Driver` per run, retains it, and serves its readers from it. + +#### Construction +`::Gapic::ResumableUpload.new(client_stub_proc:, initial_request_proc:, response_type:, initial_headers: {}, start_retry_policy: nil, control_plane_retry_policy: nil, data_plane_retry_policy: nil, error_handler: nil, method_name: nil)` + +* **Procs only; there is no value form.** `client_stub_proc` returns the `Gapic::Rest::ClientStub` and is called at the top of every run, so a client that cannot perform REST calls can still hand back a working coordinator and fail only when an upload is attempted. `initial_request_proc` returns the `[url, body]` pair for initiation and is called by `#start` only, so a coordinator built without a request message is still fully functional for resuming. +* `response_type` is the protobuf message class the final body is decoded into. `nil` returns the raw body. +* `initial_headers` is stringified (keys and values) before reaching `StartUploadConfig`, which rejects any key in `RESERVED_INITIAL_HEADERS` (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`, in any casing). Callers shape the content descriptors with `content_type:` and `upload_size:` on the run; pass-through headers such as `X-Goog-Upload-Header-Content-Disposition` remain permitted. +* `method_name` is forwarded to `Driver`, which prefixes it onto the per-request logging names. +* **No `logger` argument.** `Driver#initialize` already falls back to `client_stub.logger`. + +#### Run Signatures +* `#start(stream:, content_type: nil, upload_size: nil, chunk_size: nil, upload_timeout: nil, on_progress: nil)` +* `#resume(stream:, resume_handle: nil, content_type: nil, upload_size: nil, upload_timeout: nil, on_progress: nil)` + +Everything a run owns is a per-run keyword, because the coordinator outlives the run. Two arguments are deliberately not: the control- and data-plane retry policies sit on the constructor (they describe the upload path, not one run), and `chunk_size` is a `#start` argument only — a resumed run takes its chunk size from the `ResumeHandle`, because the server reports its granularity during initiation and a resumed run skips initiation. + +The budget keyword is named `upload_timeout` even though the config member behind it is `timeout`: a generated client already has a per-call `timeout` in scope, which in an upload bounds the initiation request alone and reaches the protocol through `start_retry_policy`. The two are three orders of magnitude apart and must not be confusable. + +Order of operations in each run, before any byte is read from the stream: +1. Claim the run slot: raise `SessionStateError` if a run is already in flight. +2. `#resume` only: reject a stream that is not positioned at byte 0, then resolve the `ResumeHandle`. +3. `client_stub_proc.call` — raises here if REST is unavailable. +4. `#start` only: `initial_request_proc.call` -> `[url, body]`. +5. Build `StartUploadConfig` or `ResumeUploadConfig`, then `Driver.new`, and retain it. +6. Run it; decode and return, or wrap and raise. + +#### Reusability & Lifecycle +* One coordinator, many runs, one `Driver` per run. A run started while another is in flight raises `SessionStateError`; a coordinator whose run has finished may legitimately start another one. +* The run slot is released in an `ensure`, so every exit — a clean return, a protocol error, an `on_progress` callback raising, a `Thread#kill` — leaves the coordinator usable and the driver retained. +* A failure while building the configuration (a reserved header, a non-positive `chunk_size`, a malformed retry policy) retains no driver at all, so it cannot disturb the resume handle an earlier run left behind. +* Network execution occurs outside the mutex; the mutex guards only the lifecycle flag and the retained driver reference. + +#### Readers +All read from the retained driver under the coordinator's mutex, are safe to call from another thread mid-run, and return a best-effort snapshot. All are `nil`/`false` before the first run. +* `#resume_handle`: `Driver#resume_handle` — the upload URL and the resolved chunk size, or `nil` for a finalized upload. +* `#resumable?`: `!resume_handle.nil?`. +* `#running?`: the coordinator's own lifecycle flag. + +`#upload_url` and `#chunk_size` are **not** exposed: both are fields of the `ResumeHandle` this set already returns. + +#### Resume Forms +1. **Bare**: `upload.resume(stream: io)` takes the retained driver's `resume_handle`, and raises `ArgumentError` when there is none. That one rule covers a coordinator that has never run, a run that finished successfully, and a run that failed in a way the protocol considers unresumable. +2. **Explicit handle**: `upload.resume(stream: io, resume_handle: handle)` — what the protocol's own errors carry, and what a caller persists between processes. + +There is no explicit `upload_url:`/`chunk_size:` form: those are exactly the two fields of a `ResumeHandle`, so a caller holding them in a database row constructs one. Resuming against a finalized upload URL is undefined behavior: it queries the server and might return the response body or raise an error, depending on the server response. #### Precondition on Stream Position for Resume -* Before executing `resume`, the caller must ensure the input stream is positioned at byte 0. -* If `stream.respond_to?(:pos) && !stream.pos.zero?`, `Session#resume` raises `ArgumentError` ("Input stream must be at byte 0 to resume; rewind the stream before resuming"). -* For unseekable streams without `:pos` (or streams at `pos == 0`), `Session` trusts the stream is at byte 0 and delegates to `Driver`, which fast-forwards to the server-confirmed offset by seeking or reading and discarding bytes. - -#### Resume Invocations & Forms -The `Session#resume` method accepts strictly keyword-only arguments: `upload_url: nil, chunk_size: nil, resume_handle: nil`. -Resumption always requires an unbound session with one of two mutually exclusive parameter forms: -1. **Explicit URL & Chunk Size**: `session.resume(upload_url: url, chunk_size: size)` -2. **Resume Handle**: `session.resume(resume_handle: handle)` - -Calling `resume` without arguments (bare resume), calling `resume` with `upload_url` but omitting `chunk_size`, or mixing `resume_handle` with other parameters raises `ArgumentError`. - -#### Cross-Session Resumption Flow -Because a session performs only a single run, resuming an interrupted upload requires instantiating a fresh session: -1. Session 1 encounters a recoverable error. -2. Caller extracts `resume_handle = session1.resume_handle` (or from the error's `#resume_handle`). -3. Caller rewinds the stream to byte 0 (if seekable, or provides an equivalent stream starting at byte 0). -4. Caller instantiates Session 2 and invokes `session2.resume(resume_handle: resume_handle)`. - -#### Concurrency & Execution Model -* At most one run (`Driver#run`) may execute at any time. -* `@running` is checked and toggled exclusively inside a `Mutex`. -* Network execution (`driver.run`) occurs outside the mutex to prevent blocking reader threads. -* Invoking `start` or `resume` while `@running` is `true` raises `SessionStateError`. -* Errors propagate unchanged. The failed `Driver` remains referenced so `upload_url`, `resume_handle`, and `bound?` remain inspectable after an exception. +* The stream must be positioned at byte 0 of the whole object, not at the server's acknowledged offset. +* If `stream.respond_to?(:pos) && !stream.pos.zero?`, `#resume` raises `ArgumentError`. +* A stream that reports no position is trusted, and `Driver` fast-forwards to the server-confirmed offset by seeking or by reading and discarding bytes. + +#### Response Decoding +`response_type.decode_json body.to_s, ignore_unknown_fields: true`, identical to what a generated REST service stub does with a unary response. An empty or absent final body decodes to an empty message; malformed JSON raises `Google::Protobuf::ParseError`. A `nil` `response_type` returns the raw body — a `String`, or `nil` when the final response carried none. That is `@private` behavior, for this gem's own tests. + +#### Error Handling +`error_handler` is a lambda that **returns** the exception to raise; it must not raise. A `nil` return, or a return of the original error, re-raises the original. When the original carries `HasResumeHandle` and the replacement does not, the replacement is extended with the mixin and given the original's handle, so a library-specific error type cannot erase the fact that the upload is resumable. + +Only the run is wrapped. Argument and configuration errors are raised while the driver is still being built, and reach the caller as themselves. + +#### Retry Policy Placement +All three planes reach the coordinator on the **constructor**; none is per-run. `start_retry_policy` is the documented one and comes from the generated method's `CallOptions`. `control_plane_retry_policy` and `data_plane_retry_policy` are `@private`: no generated client passes them, but `Gapic::Common`'s own integration harness does, which keeps recovery and retry-exhaustion tests on the production code path rather than on a hand-built `Driver`. + +`Gapic::Rest::ResumableUpload.start_retry_policy_for(options)` (`@private`, beside `RetryPolicies::START_DEFAULTS`) converts per-call options into the initiation policy. It returns a **Hash**, never a policy object, because the protocol treats an object as a wholesale replacement — which would silently drop the initiation predicate that makes a missing `X-Goog-Upload-Status` retriable. It always sets `timeout:` from `options.timeout`, copies backoff settings and retry codes only where the caller set them (an empty `retry_codes` list counts as unset), and raises `ArgumentError` for a Proc retry policy. --- @@ -526,7 +536,7 @@ Terminal errors provide actionable context so downstream SDK callers can inspect * `InvalidTransitionError < Gapic::Common::Error`: Unexpected event dispatched for state; includes `HasResumeHandle`. * `StreamMismatchError < Gapic::Common::Error`: Stream content or length does not match resumed upload specifications; includes `HasResumeHandle`. * `RequestFailedError < Gapic::Common::Error`: Terminal HTTP request failure (e.g. transport connection failure, request timeout, or retries exhausted). Retains `attr_reader :cause` returning the underlying error, preserves REST error attributes (`status_code`, `status`, `details`, `headers`) when available, and includes `HasResumeHandle`. - * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates Session lifecycle rules (e.g. attempting to start an already-bound session, resuming an unbound session without a target upload, re-binding to a different upload, resuming a finalized/dead session, or concurrent run invocations). Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. + * `SessionStateError < Gapic::Common::Error`: Raised when an operation violates the upload session lifecycle rules, e.g. starting a second run on a coordinator while one is still in flight. Distinguished from `ArgumentError`, which is raised strictly for invalid argument shapes. * **Resume Handle Propagation (`HasResumeHandle`)**: * The `HasResumeHandle` mixin exposes `attr_reader :resume_handle` returning a `ResumeHandle` (or `nil` if session initiation was incomplete or if the session was `:rejected` or `:cancelled`). * Whenever `resume_handle` is non-nil, the uniform suffix `" (upload session is resumable: see #resume_handle)"` is automatically appended to the error message. diff --git a/gapic-common/design/resumable_upload/integration-test-plan.md b/gapic-common/design/resumable_upload/integration-test-plan.md index 46a5750..b3c5115 100644 --- a/gapic-common/design/resumable_upload/integration-test-plan.md +++ b/gapic-common/design/resumable_upload/integration-test-plan.md @@ -47,7 +47,9 @@ flowchart TD * **`ShowcaseIntegrationTest`**: Base class providing helper methods for test configuration: * `showcase_client_stub`: Instantiates a real `Gapic::Rest::ClientStub` targeting `SHOWCASE_ENDPOINT` with `raise_faraday_errors: false` and an attached `DEBUG` logger. * `build_config(scenario: nil, scenario_config: {}, **overrides)`: Creates a `StartUploadConfig` targeting `/resumable/upload/v1beta1/files:upload`. When `scenario` is provided, injects `X-Goog-Test-Scenario` and `X-Goog-Test-Scenario-Config` (with a generated `client_uuid` merged with `scenario_config`) into `initial_headers` (these test headers are unaffected by `RESERVED_INITIAL_HEADERS` since they are not in the five reserved protocol headers). Configures fast retry policies (`FAST_RETRY = { initial_delay: 0.01, max_delay: 0.05, multiplier: 1, timeout: 2 }`), a default 10-second session timeout, a default payload of `786_432` bytes (`3 * 262_144`), default chunk size of `262_144` bytes, and an `on_progress` callback appending each `Progress` struct to `@progress_records`. - * `build_session(scenario: nil, scenario_config: {}, **overrides)` & `start_session(session, **overrides)`: Partitions overrides using `START_ONLY_KEYS` (`[:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy]`). `build_session` instantiates a `Gapic::Rest::ResumableUpload::Session` with the common members (`client_stub`, `stream`, `upload_size`, `content_type`, `timeout`, `control_plane_retry_policy`, `data_plane_retry_policy`, `on_progress`, `logger`) and stores initiation arguments in `@start_args`, while `start_session` invokes `session.start(**@start_args, **overrides)`. + * `build_upload(scenario: nil, scenario_config: {}, initial_headers: {}, **overrides)`: Instantiates a `::Gapic::ResumableUpload` whose `client_stub_proc` returns a Showcase client stub and whose `initial_request_proc` returns `[UPLOAD_PATH, nil]`. Injects the same scenario headers as `build_config`, sets all three retry policies to `FAST_RETRY`, and passes `response_type: nil` so runs return the raw response body for the tests to parse. + * `start_args(**overrides)` & `resume_args(**overrides)`: Default per-run arguments — a `786_432`-byte payload and its `upload_size`, a 10-second `upload_timeout`, and an `on_progress` callback appending each `Progress` struct to `@progress_records`. `start_args` adds the `262_144`-byte `chunk_size`, which a resumed run takes from its `ResumeHandle` instead. + * `resume_handle_for(upload_url, chunk_size: DEFAULT_CHUNK_SIZE)`: Builds a `ResumeHandle` for an upload created out-of-band by `raw_start`. * `phases` & `offsets`: Convenience accessors returning `@progress_records.map(&:phase)` and `@progress_records.map(&:bytes_uploaded)`. * `payload(size)`: Generates deterministic binary strings of arbitrary byte length for stream uploads. * `UnseekableStream`: Stream wrapper around `StringIO` that exposes `#read` and `#pos` while omitting `#seek` (`respond_to?(:seek)` is `false`). @@ -256,10 +258,10 @@ Tests non-fatal transient retries, missing status headers, retry exhaustion, fat ### 2.5 Resumption Suite (`integration/resumable_upload/resume_test.rb`) -Tests `Gapic::Rest::ResumableUpload::Session` resumption capabilities against Showcase. +Tests `Gapic::ResumableUpload` resumption capabilities against Showcase. #### Case 1. Resume in-progress upload on seekable stream (`test_resume_in_progress_upload`) -* Uploads chunk 1 via `raw_upload`, then resumes with a fresh session and full stream. +* Uploads chunk 1 via `raw_upload`, then resumes a fresh upload handle with an explicit `resume_handle:` and the full stream. * Asserts `phases == [:initiating, :uploading, :uploading, :uploading, :finalizing, :completed]` and offsets align correctly. #### Case 2. Resume already finalized upload (`test_resume_finalized_upload`) @@ -279,8 +281,8 @@ Tests `Gapic::Rest::ResumableUpload::Session` resumption capabilities against Sh * **Case 5b (`test_resume_wrong_stream_seekable_size_guard`)**: Seekable `StringIO` with `server_offset > stream.size` (and `upload_size: nil`) raises `StreamMismatchError` via stream size guard. #### Case 6. Golden user-style resume (`test_golden_user_style_resume_seekable`, `test_golden_user_style_resume_unseekable`) -* User raises exception in `on_progress` carrying `session.resume_handle` on first upload ack. -* Fresh session resumes via `resume_handle: handle` and completes the transfer. Tested on both seekable streams and fresh unseekable streams starting at byte 0. +* User raises exception in `on_progress` carrying `upload.resume_handle` on first upload ack. +* The same handle object then resumes — bare for the seekable case, with an explicit `resume_handle:` for the unseekable one — and completes the transfer. The unseekable variant uses a fresh stream starting at byte 0. -#### Case 7. Lifecycle and contract violations (`test_lifecycle_violations`) -* Verifies second `start` and `resume` on bound session raise `SessionStateError`, and bare `resume` raises `ArgumentError`. +#### Case 7. Lifecycle of a reused handle (`test_lifecycle_of_a_reused_handle`) +* Verifies a completed run leaves `resumable?` and `running?` false, a bare `resume` afterwards raises `ArgumentError`, a second `start` on the same handle is legal and initiates an unrelated upload, and a bare `resume` on a handle that has never run raises `ArgumentError`. diff --git a/gapic-common/integration/integration_helper.rb b/gapic-common/integration/integration_helper.rb index 0af5c0b..c33324f 100644 --- a/gapic-common/integration/integration_helper.rb +++ b/gapic-common/integration/integration_helper.rb @@ -123,13 +123,15 @@ def build_config scenario: nil, scenario_config: {}, **overrides Gapic::Rest::ResumableUpload::StartUploadConfig.new(**defaults, **overrides, initial_headers: headers) end - START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze - - # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, - # so callers can run it with `start_session session`. - def build_session scenario: nil, scenario_config: {}, **overrides + ## + # Builds a coordinator pointed at Showcase, with the short retry policies these tests rely on. + # + # The handle decodes nothing (`response_type: nil`), so runs return the raw response body and the tests + # can assert on what Showcase actually sent. + # + def build_upload scenario: nil, scenario_config: {}, initial_headers: {}, **overrides @progress_records = [] - headers = (overrides.delete(:initial_headers) || {}).dup + headers = initial_headers.dup if scenario headers["X-Goog-Test-Scenario"] = scenario headers["X-Goog-Test-Scenario-Config"] = JSON.generate( @@ -137,31 +139,39 @@ def build_session scenario: nil, scenario_config: {}, **overrides ) end - @start_args = { - initial_url: UPLOAD_PATH, - initial_headers: headers, - start_retry_policy: FAST_RETRY, - chunk_size: DEFAULT_CHUNK_SIZE - }.merge(overrides.slice(*START_ONLY_KEYS)) - - defaults = { - client_stub: showcase_client_stub, + Gapic::ResumableUpload.new( + client_stub_proc: -> { showcase_client_stub }, + initial_request_proc: -> { [UPLOAD_PATH, nil] }, + response_type: nil, + initial_headers: headers, + start_retry_policy: FAST_RETRY, control_plane_retry_policy: FAST_RETRY, data_plane_retry_policy: FAST_RETRY, - timeout: 10, - on_progress: ->(progress) { @progress_records << progress }, - logger: @logger + **overrides + ) + end + + # Default run arguments for `#start`. + def start_args **overrides + { chunk_size: DEFAULT_CHUNK_SIZE }.merge(resume_args(**overrides)) + end + + # Default run arguments for `#resume`, which takes no chunk size: a resumed run carries it on the + # resume handle. + def resume_args **overrides + defaults = { + upload_timeout: 10, + on_progress: ->(progress) { @progress_records << progress } } unless overrides.key? :stream defaults[:stream] = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) defaults[:upload_size] = DEFAULT_PAYLOAD_SIZE end - - Gapic::Rest::ResumableUpload::Session.new(**defaults, **overrides.except(*START_ONLY_KEYS)) + defaults.merge overrides end - def start_session session, **overrides - session.start(**@start_args, **overrides) + def resume_handle_for upload_url, chunk_size: DEFAULT_CHUNK_SIZE + Gapic::Rest::ResumableUpload::ResumeHandle.new upload_url: upload_url, chunk_size: chunk_size end def raw_start scenario: nil, scenario_config: {}, upload_size: nil, headers: {} diff --git a/gapic-common/integration/resumable_upload/resume_test.rb b/gapic-common/integration/resumable_upload/resume_test.rb index 5c4dc5f..f751e14 100644 --- a/gapic-common/integration/resumable_upload/resume_test.rb +++ b/gapic-common/integration/resumable_upload/resume_test.rb @@ -19,7 +19,7 @@ require "stringio" ## -# Suite D: Integration tests for Resumable Upload Session resumption against Showcase. +# Suite D: Integration tests for resumption against Showcase, driven through ::Gapic::ResumableUpload. # class ResumeTest < ShowcaseIntegrationTest ## @@ -40,8 +40,8 @@ def test_resume_in_progress_upload chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - session = build_session - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -54,8 +54,10 @@ def test_resume_finalized_upload upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: true - session = build_session stream: StringIO.new(payload(DEFAULT_CHUNK_SIZE)), upload_size: DEFAULT_CHUNK_SIZE - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(stream: StringIO.new(payload(DEFAULT_CHUNK_SIZE)), + upload_size: DEFAULT_CHUNK_SIZE, + resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_CHUNK_SIZE, parsed["size"] @@ -72,8 +74,8 @@ def test_resume_query_503_absorbed_by_retry chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - session = build_session - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -91,8 +93,8 @@ def test_resume_query_409_triggers_retry_recovery chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - session = build_session - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -107,9 +109,9 @@ def test_resume_unseekable_stream_fast_forwards chunk1 = payload(DEFAULT_PAYLOAD_SIZE).byteslice 0, DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: chunk1, finalize: false - stream = UnseekableStream.new payload(DEFAULT_PAYLOAD_SIZE) - session = build_session stream: stream - result = session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload = build_upload + result = upload.resume(**resume_args(stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), + resume_handle: resume_handle_for(upload_url))) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] @@ -122,11 +124,12 @@ def test_resume_wrong_stream_unseekable_mismatch upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false - stream = UnseekableStream.new payload(100) - session = build_session stream: stream, upload_size: nil + upload = build_upload assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do - session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload.resume(**resume_args(stream: UnseekableStream.new(payload(100)), + upload_size: nil, + resume_handle: resume_handle_for(upload_url))) end refute_includes phases, :finalizing end @@ -136,94 +139,93 @@ def test_resume_wrong_stream_seekable_size_guard upload_url = raw_start upload_size: DEFAULT_CHUNK_SIZE raw_upload upload_url: upload_url, offset: 0, bytes: payload(DEFAULT_CHUNK_SIZE), finalize: false - stream = StringIO.new payload(100) - session = build_session stream: stream, upload_size: nil + upload = build_upload assert_raises Gapic::Rest::ResumableUpload::StreamMismatchError do - session.resume upload_url: upload_url, chunk_size: DEFAULT_CHUNK_SIZE + upload.resume(**resume_args(stream: StringIO.new(payload(100)), + upload_size: nil, + resume_handle: resume_handle_for(upload_url))) end refute_includes phases, :finalizing end - # D6a. Golden user-style resume on a seekable stream after user abort in on_progress. + # D6a. Golden user-style resume on a seekable stream after user abort in on_progress, reusing the handle + # and its retained resume handle rather than passing one back in. def test_golden_user_style_resume_seekable stream = StringIO.new payload(DEFAULT_PAYLOAD_SIZE) - session1 = nil + upload = nil on_progress = lambda do |progress| if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE - raise UserPauseError.new("user paused", session1.resume_handle) + raise UserPauseError.new("user paused", upload.resume_handle) end end - session1 = build_session stream: stream, on_progress: on_progress + upload = build_upload err = assert_raises UserPauseError do - start_session session1 + upload.start(**start_args(stream: stream, on_progress: on_progress)) end - assert session1.bound? - assert session1.resumable? - handle = err.resume_handle - refute_nil handle + assert upload.resumable? + refute_nil err.resume_handle + assert_equal err.resume_handle, upload.resume_handle stream.rewind - session2 = build_session stream: stream - result = session2.resume resume_handle: handle + result = upload.resume(**resume_args(stream: stream)) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] - assert session2.bound? - refute session2.resumable? + refute upload.resumable? end - # D6b. Golden user-style resume with a fresh unseekable stream starting at byte 0. + # D6b. Golden user-style resume with a fresh unseekable stream starting at byte 0, resuming from the + # handle the error carried. def test_golden_user_style_resume_unseekable - session1 = nil + upload = nil on_progress = lambda do |progress| if progress.phase == :uploading && progress.bytes_uploaded == DEFAULT_CHUNK_SIZE - raise UserPauseError.new("user paused", session1.resume_handle) + raise UserPauseError.new("user paused", upload.resume_handle) end end - session1 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), on_progress: on_progress + upload = build_upload err = assert_raises UserPauseError do - start_session session1 + upload.start(**start_args(stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), + on_progress: on_progress)) end - assert session1.bound? - assert session1.resumable? + assert upload.resumable? handle = err.resume_handle refute_nil handle - session2 = build_session stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)) - result = session2.resume resume_handle: handle + result = upload.resume(**resume_args(stream: UnseekableStream.new(payload(DEFAULT_PAYLOAD_SIZE)), + resume_handle: handle)) parsed = JSON.parse result assert_equal DEFAULT_PAYLOAD_SIZE, parsed["size"] - assert session2.bound? - refute session2.resumable? + refute upload.resumable? end - # D7. Lifecycle and contract violations on session runs. - def test_lifecycle_violations - session = build_session stream: StringIO.new(payload(100)), upload_size: 100 - start_session session + # D7. Lifecycle of a reusable handle: a finished run leaves nothing to resume, but the handle itself + # stays usable. + def test_lifecycle_of_a_reused_handle + upload = build_upload + upload.start(**start_args(stream: StringIO.new(payload(100)), upload_size: 100)) - assert session.bound? + refute upload.resumable? + refute upload.running? - # Second start on executed session raises SessionStateError - assert_raises Gapic::Rest::ResumableUpload::SessionStateError do - start_session session + # A bare resume after a completed run has no handle to work from. + assert_raises ArgumentError do + upload.resume(**resume_args(stream: StringIO.new(payload(100)), upload_size: 100)) end - # Resume on already bound/executed session raises SessionStateError - assert_raises Gapic::Rest::ResumableUpload::SessionStateError do - session.resume upload_url: "https://example.com/test", chunk_size: DEFAULT_CHUNK_SIZE - end + # Starting again is legal, and initiates a second, unrelated upload. + result = upload.start(**start_args(stream: StringIO.new(payload(100)), upload_size: 100)) + assert_equal 100, JSON.parse(result)["size"] - # Resume without parameters on fresh session raises ArgumentError - fresh_session = build_session + # A bare resume on a handle that has never run has nothing to work from either. assert_raises ArgumentError do - fresh_session.resume + build_upload.resume(**resume_args) end end end diff --git a/gapic-common/lib/gapic/rest.rb b/gapic-common/lib/gapic/rest.rb index 693aba2..59be632 100644 --- a/gapic-common/lib/gapic/rest.rb +++ b/gapic-common/lib/gapic/rest.rb @@ -29,6 +29,7 @@ require "gapic/rest/operation" require "gapic/rest/paged_enumerable" require "gapic/rest/resumable_upload" +require "gapic/resumable_upload" require "gapic/rest/server_stream" require "gapic/rest/threaded_enumerator" require "gapic/rest/transport_operation" diff --git a/gapic-common/lib/gapic/rest/resumable_upload.rb b/gapic-common/lib/gapic/rest/resumable_upload.rb index a7530e1..2dfd37d 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload.rb @@ -24,50 +24,101 @@ require "gapic/rest/resumable_upload/rules" require "gapic/rest/resumable_upload/core" require "gapic/rest/resumable_upload/driver" -require "gapic/rest/resumable_upload/session" module Gapic module Rest ## - # Resumable Upload Protocol implementation for REST transport. + # Resumable Upload Protocol implementation for REST transport: session initiation, chunked + # streaming, automatic retries, progress reporting via {Progress}, and resumption via + # {ResumeHandle}. # - # {Session} is the primary public entry point for initiating and resuming uploads. - # It manages session initiation, chunked streaming, automatic retries, progress - # callbacks via {Progress}, and cross-session resumption via {ResumeHandle}. + # **This namespace has no callable surface.** Every method and class in it is internal machinery, + # documented as `@private` and excluded from these docs; what remains visible is the data a caller + # receives — {Progress}, {ResumeHandle}, {HasResumeHandle} — and the error classes listed below. + # Uploads are driven from {Gapic::ResumableUpload}, which sits above this namespace and coordinates + # runs against it. + # + # Errors raised from here carry a {ResumeHandle} where the upload can still be continued, so the + # usual shape of handling one is to rescue {HasResumeHandle} and hand the handle back to the + # coordinator: + # + # @example Uploading, then resuming after a recoverable failure + # upload = client.upload_media ... # returns a Gapic::ResumableUpload + # begin + # upload.start stream: File.open("movie.mp4", "rb"), upload_size: File.size("movie.mp4") + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # handle = e.resume_handle + # upload.resume stream: File.open("movie.mp4", "rb"), resume_handle: handle + # end # # ### Error Types # * {RequestFailedError} - Transport connection failure, timeout, or retries exhausted (includes {HasResumeHandle}). - # * {DeadlineExceededError} - Global upload timeout exceeded (includes {HasResumeHandle}). + # * {DeadlineExceededError} - Whole-upload timeout exceeded (includes {HasResumeHandle}). # * {BadResponseError} - Unexpected or malformed HTTP response (includes {HasResumeHandle}). # * {UnseekableStreamError} - Stream rewinding required on an unseekable stream (includes {HasResumeHandle}). # * {StreamMismatchError} - Stream content or length does not match resumed upload (includes {HasResumeHandle}). # * {InvalidTransitionError} - Unmatched event for the current protocol state (includes {HasResumeHandle}). # * {UploadRejectedError} - Server explicitly rejected the upload session (final). - # * {SessionStateError} - Session lifecycle rule violation, e.g., calling `#start` twice (final). - # - # @example Initiating an upload, rescuing an error, and resuming from a fresh session - # session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: stream - # ) - # - # begin - # response = session.start( - # initial_url: "https://example.googleapis.com/resumable/upload/v1/example/upload:new" - # ) - # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e - # handle = e.resume_handle - # raise unless handle - # - # stream.rewind - # resumed_session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: stream - # ) - # response = resumed_session.resume resume_handle: handle - # end + # * {SessionStateError} - Upload session lifecycle rule violation, e.g. starting a second run while one + # is in flight (final). # module ResumableUpload + ## + # @private + # Backoff settings carried from a caller's retry policy into the initiation policy, mapped to the + # {Gapic::Common::RetryPolicy} value a reader returns when the setting was never set. + # + # `jitter`'s default is a private constant, so it is read off a default policy rather than named. + # + # @return [Hash{Symbol=>Numeric}] + BACKOFF_DEFAULTS = { + initial_delay: Gapic::Common::RetryPolicy::DEFAULT_INITIAL_DELAY, + max_delay: Gapic::Common::RetryPolicy::DEFAULT_MAX_DELAY, + multiplier: Gapic::Common::RetryPolicy::DEFAULT_MULTIPLIER, + jitter: Gapic::Common::RetryPolicy.new.jitter + }.freeze + + ## + # @private + # Converts the per-call options a generated client assembles into overrides for the initiation + # retry policy. + # + # Returns a **Hash**, never a policy object: the protocol treats a {Gapic::Common::RetryPolicy} as a + # wholesale replacement and a Hash as a per-key override. Initiation's default policy carries a + # predicate that treats a response missing `X-Goog-Upload-Status` as retriable gateway noise, and + # handing over an object would silently drop it. + # + # `timeout` is always set, and becomes the local deadline of the initiation request alone — the + # whole-upload budget is separate and is not derived here. Without it, initiation would inherit + # {Gapic::Common::RetryPolicy::DEFAULT_TIMEOUT} (one hour), because `Gapic::CallOptions::RetryPolicy` + # never populates `@timeout` even though it subclasses {Gapic::Common::RetryPolicy}. + # + # Backoff settings and retry codes are copied only where the caller set them, which is why each is + # compared against the corresponding default: a reader on an unset policy returns that default, and + # the initiation defaults are the values that should survive. An empty `retry_codes` list counts as + # unset. A `retry_predicate` is deliberately not copied; the initiation predicate stays in place. + # + # @param options [Gapic::CallOptions, nil] Per-call options from a generated client method + # @return [Hash] Overrides for the initiation retry policy + # @raise [ArgumentError] If the call options carry a Proc (or any other non-{Gapic::Common::RetryPolicy}) + # retry policy, which has no coherent meaning across the three retry planes of an upload + def self.start_retry_policy_for options + overrides = { timeout: options&.timeout } + policy = options&.retry_policy + return overrides if policy.nil? + unless policy.is_a? Gapic::Common::RetryPolicy + raise ArgumentError, + "Resumable upload cannot derive an initiation retry policy from a #{policy.class}; " \ + "use a Gapic::Common::RetryPolicy or a Hash of retry settings" + end + + overrides[:retry_codes] = policy.retry_codes unless policy.retry_codes.empty? + BACKOFF_DEFAULTS.each do |setting, default| + value = policy.public_send setting + overrides[setting] = value unless value == default + end + overrides + end end end end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb index d08c00d..5d90520 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/data_types.rb @@ -14,9 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# rubocop:disable Metrics/ModuleLength + module Gapic module Rest - # rubocop:disable Metrics/ModuleLength module ResumableUpload ## # @private @@ -75,7 +76,8 @@ module ResumableUpload ## # @private - # Immutable configuration for a run that initiates a new upload session, i.e. {Session#start}. + # Immutable configuration for a run that initiates a new upload session, i.e. + # {Gapic::ResumableUpload#start}. # # Carries {COMMON_MEMBERS} plus the members only an initiating run uses. # @@ -163,7 +165,8 @@ def initialize initial_url:, ## # @private - # Immutable configuration for a run that resumes an existing upload session, i.e. {Session#resume}. + # Immutable configuration for a run that resumes an existing upload session, i.e. + # {Gapic::ResumableUpload#resume}. # # Carries {COMMON_MEMBERS} plus the upload URL and chunk size the earlier run established. There is # no `start_retry_policy` here: a resumed run issues no initiation request, so the member would @@ -230,7 +233,7 @@ def initialize upload_url:, # # The `on_progress` callback runs synchronously on the same thread as the upload protocol # and must not block. Any exception raised inside the callback aborts the upload session - # and propagates out of {Session#start} or {Session#resume}. + # and propagates out of {Gapic::ResumableUpload#start} or {Gapic::ResumableUpload#resume}. # # @!attribute [r] phase # @return [Symbol] Current upload phase, one of {Progress::PHASES} @@ -274,8 +277,8 @@ def initialize phase:, bytes_uploaded:, total_bytes: nil # Allowed lifecycle phases for an upload session. # # A callback observes `:initiating`, `:uploading`, `:recovering`, `:finalizing` and `:completed`. - # `:cancelling` is reserved: cancellation is not exposed on {Session}, so no phase with that value is - # currently emitted. + # `:cancelling` is reserved: cancellation is not exposed on {Gapic::ResumableUpload}, so no phase + # with that value is currently emitted. # # @return [Array] Progress::PHASES = [:initiating, :uploading, :recovering, :finalizing, :cancelling, :completed].freeze @@ -410,6 +413,7 @@ def initialize from_status:, shape:, recipe:, next_state:, instructions: [] end end end - # rubocop:enable Metrics/ModuleLength end end + +# rubocop:enable Metrics/ModuleLength diff --git a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb index 563171d..8b0d7b7 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/driver.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/driver.rb @@ -89,12 +89,16 @@ def upload_url # @param config [StartUploadConfig, ResumeUploadConfig] Configuration for this upload session # @param core [Core, nil] Optional Core state machine (defaults to new Core with config) # @param logger [Logger, nil] Optional logger override - def initialize client_stub:, config:, core: nil, logger: nil + # @param method_name [String, nil] RPC name this upload was started from, prefixed onto the + # per-request logging names (`"create_media_upload.start"`, `"create_media_upload.upload"`, and + # so on). Defaults to `"ResumableUpload"`. + def initialize client_stub:, config:, core: nil, logger: nil, method_name: nil @client_stub = client_stub @config = config @core = core || Core.new(config) @buffer = "".b @buffer_start_offset = 0 + @method_name_prefix = method_name || "ResumableUpload" endpoint = client_stub.respond_to?(:endpoint) ? client_stub.endpoint : nil setup_logging logger: logger || (client_stub.respond_to?(:logger) ? client_stub.logger : nil), @@ -567,7 +571,7 @@ def execute_send_start instruction return Event::GlobalDeadlineExceeded.new if deadline_exceeded? event = make_post_request instruction.url, headers: headers, body: instruction.body, - retry_policy: policy, method_name: "ResumableUpload.start", + retry_policy: policy, method_name: "#{@method_name_prefix}.start", start_attempt: attempt return event unless event.is_a? Event::HttpResponse @@ -631,7 +635,7 @@ def execute_send_chunk instruction make_post_request instruction.url, headers: headers, body: body, retry_policy: @data_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.upload" + method_name: "#{@method_name_prefix}.upload" end ## @@ -649,7 +653,7 @@ def execute_send_finalize instruction } make_post_request instruction.url, headers: headers, body: "", retry_policy: @data_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.finalize" + method_name: "#{@method_name_prefix}.finalize" end ## @@ -663,7 +667,7 @@ def execute_send_query instruction headers = { "X-Goog-Upload-Command" => "query", "Content-Length" => "0" } make_post_request instruction.url, headers: headers, body: "", retry_policy: @control_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.query" + method_name: "#{@method_name_prefix}.query" end ## @@ -677,7 +681,7 @@ def execute_send_cancel instruction headers = { "X-Goog-Upload-Command" => "cancel", "Content-Length" => "0" } make_post_request instruction.url, headers: headers, body: "", retry_policy: @control_plane_retry_policy.dup.start!, - method_name: "ResumableUpload.cancel" + method_name: "#{@method_name_prefix}.cancel" end ## diff --git a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb index b54e114..17141be 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/errors.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/errors.rb @@ -546,8 +546,8 @@ def self.from event_or_error, message: nil, resume_handle: nil end ## - # Raised when an operation violates the Session lifecycle rules - # (e.g. calling a `start` method more than once). + # Raised when an operation violates the upload session lifecycle rules, e.g. starting a second run + # on a coordinator while one is still in flight. # class SessionStateError < Gapic::Common::Error end diff --git a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb index deb92bf..29d8341 100644 --- a/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb +++ b/gapic-common/lib/gapic/rest/resumable_upload/retry_policies.rb @@ -59,7 +59,7 @@ module RetryPolicies ## # @private # Default options for start command retry policy. - # Keep in sync with the "Retry Policies" section of Session's class doc. + # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] START_DEFAULTS = { retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, @@ -72,7 +72,7 @@ module RetryPolicies ## # @private # Default options for query and cancel commands retry policy. - # Keep in sync with the "Retry Policies" section of Session's class doc. + # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] CONTROL_PLANE_DEFAULTS = { retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, @@ -84,7 +84,7 @@ module RetryPolicies ## # @private # Default options for upload and finalize commands retry policy. - # Keep in sync with the "Retry Policies" section of Session's class doc. + # Keep in sync with the "Retry Policies" section of the Gapic::ResumableUpload class doc. # @return [Hash] DATA_PLANE_DEFAULTS = { retry_codes: ["UNAVAILABLE", "DEADLINE_EXCEEDED", "RESOURCE_EXHAUSTED", "INTERNAL"].freeze, diff --git a/gapic-common/lib/gapic/rest/resumable_upload/session.rb b/gapic-common/lib/gapic/rest/resumable_upload/session.rb deleted file mode 100644 index ac0e431..0000000 --- a/gapic-common/lib/gapic/rest/resumable_upload/session.rb +++ /dev/null @@ -1,546 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "gapic/rest/resumable_upload/data_types" -require "gapic/rest/resumable_upload/driver" -require "gapic/rest/resumable_upload/errors" - -module Gapic - module Rest - module ResumableUpload - ## - # Coordinates a resumable upload across its lifecycle. - # - # A Session performs exactly one run (`start` or `resume`), never both, never twice. - # - # ### Two-State Model - # 1. **Unbound** (`!bound?`): Fresh session prior to execution. Permitted operations: `start` - # or `resume(...)`. - # 2. **Bound** (`bound?`): Session has executed or bound to an upload URL. Permitted operations: - # none (`start` and `resume` both raise {SessionStateError}). - # - # Calling {#resumable?} reports whether a new session can resume the upload (`!resume_handle.nil?`). - # Completed uploads (`:success`) and rejected uploads are finalized and not resumable - # (`resumable?` returns `false`, `resume_handle` returns `nil`). - # - # ### Execution Model - # - # {#start} and {#resume} are synchronous: they block the calling thread for the entire duration of the - # upload and return only on completion or failure. The `on_progress` callback runs on that same thread. - # - # The remaining readers ({#upload_url}, {#bound?}, {#resume_handle}, {#resumable?}, {#running?}) are - # guarded by an internal mutex and may be called from another thread while a run is in progress. Values - # read mid-run are a best-effort snapshot of a state the upload thread is still advancing. - # - # ### Where Arguments Live - # - # The constructor takes what both run types share: the client stub, the stream, `upload_size`, - # `content_type`, `timeout`, the control- and data-plane retry policies, `on_progress` and `logger`. - # Arguments that belong to one run live on the method performing it — `initial_url`, `initial_body`, - # `initial_headers`, `chunk_size` and `start_retry_policy` on {#start}; `upload_url` and `chunk_size`, - # or a {ResumeHandle}, on {#resume}. - # - # ### Recovering From a Failure - # - # A bound session never runs again, so recovery means constructing a new Session. Errors that carry a - # resume handle include the {HasResumeHandle} mixin, which can be rescued directly to catch all of them: - # - # @example Resuming after a recoverable failure - # begin - # session.start initial_url: url - # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e - # raise unless e.resume_handle - # Session.new(client_stub: client_stub, stream: File.open(path, "rb")) - # .resume(resume_handle: e.resume_handle) - # end - # - # The replacement session needs a stream positioned at byte 0 of the whole object, not at the server's - # acknowledged offset; {#resume} fast-forwards on its own. For an unseekable stream that means opening a - # fresh one, since it cannot be rewound. - # - # ### Defaults - # - # * `chunk_size` defaults to 8 MB, then rounds down to a multiple of any chunk granularity the server - # requires. - # * `timeout` defaults to `upload_size / 1 MB per second` when `upload_size` is known, floored at one - # hour, and to one hour flat when it is not. - # - # ### Retry Policies - # - # Retry behavior is partitioned across three policies: `start_retry_policy` on {#start}, and - # `control_plane_retry_policy` and `data_plane_retry_policy` on {#initialize}. - # - # Passing a {Gapic::Common::RetryPolicy} replaces the corresponding default policy outright. Passing a - # Hash overrides only the keys it names and leaves the remaining defaults — including `retry_codes` and - # any status-header predicates — in place. - # - # All three policies share the same default retry codes and exponential backoff settings: - # - # | Setting | Default | - # |---|---| - # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | - # | `initial_delay` | `1.0` s | - # | `max_delay` | `15.0` s | - # | `multiplier` | `1.3` | - # - # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response - # header is treated: - # - # | Policy | Governs | Missing status header | - # |---|---|---| - # | `start_retry_policy` | session initiation | **Retriable** on any status (incl. `200`), unless fatal | - # | `control_plane_retry_policy` | `query` and `cancel` | No predicate; decided on `retry_codes` alone | - # | `data_plane_retry_policy` | `upload` and `finalize` | **Not** retriable | - # - # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data - # plane treats it as a response it cannot interpret and refuses to replay bytes against it. - # - class Session - # @return [Gapic::Rest::ClientStub] Underlying REST client stub - attr_reader :client_stub - - ## - # Binary input stream to upload. The stream is assumed to be positioned at byte 0 - # (it is not rewound prior to reading) and is not closed after use. - # - # @return [IO] - attr_reader :stream - - # @return [Integer, nil] Total upload bytes if known upfront - attr_reader :upload_size - - # @return [String, nil] MIME type of uploaded media - attr_reader :content_type - - ## - # Total upload timeout in seconds, covering the whole run rather than any single request. When `nil`, - # it resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to - # one hour flat otherwise. Zero and negative values are treated as `nil`. - # - # @return [Numeric, nil] - attr_reader :timeout - - ## - # Retry policy for control commands (query, cancel). A {Gapic::Common::RetryPolicy} replaces the - # default policy outright; a Hash overrides only the settings it names. - # - # @return [Gapic::Common::RetryPolicy, Hash, nil] - attr_reader :control_plane_retry_policy - - ## - # Retry policy for data commands (upload, finalize). A {Gapic::Common::RetryPolicy} replaces the - # default policy outright; a Hash overrides only the settings it names. - # - # @return [Gapic::Common::RetryPolicy, Hash, nil] - attr_reader :data_plane_retry_policy - - ## - # Callback invoked with {Progress} snapshots during upload execution. - # Executed synchronously on the thread running the upload protocol; it must not block. - # Exceptions raised inside the callback immediately abort the upload session and - # propagate out of {#start} or {#resume}. - # - # @return [Proc, nil] - attr_reader :on_progress - - # @return [Logger, nil] Logger instance - attr_reader :logger - - ## - # Initializes a new Resumable Upload Session. - # - # The constructor takes only what both run types share. Arguments specific to a single run live on - # the method that performs it: initiation details on {#start}, the upload URL and chunk size on - # {#resume}. - # - # @param client_stub [Gapic::Rest::ClientStub] Underlying REST client stub - # @param stream [IO] Binary input stream to upload. Precondition: assumed to be positioned at byte 0 - # (not rewound prior to reading) and not closed after use. - # @param upload_size [Integer, nil] Total upload bytes if known upfront - # @param content_type [String, nil] MIME type of uploaded media - # @param timeout [Numeric, nil] Total upload timeout in seconds covering the whole run. When `nil`, - # resolves to `upload_size / 1 MB per second` floored at one hour if `upload_size` is known, and to - # one hour flat otherwise. - # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for control - # commands (`query` and `cancel`). See the "Retry Policies" section in the class documentation. - # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for data - # commands (`upload` and `finalize`). See the "Retry Policies" section in the class documentation. - # @param on_progress [Proc, nil] Progress callback invoked as `->(progress)` with a {Progress} instance. - # Executed synchronously on the upload protocol thread; it must not block. - # Exceptions raised inside the callback abort the session and propagate out of {#start} or {#resume}. - # @param logger [Logger, nil] Logger instance - # - def initialize client_stub:, - stream:, - upload_size: nil, - content_type: nil, - timeout: nil, - control_plane_retry_policy: nil, - data_plane_retry_policy: nil, - on_progress: nil, - logger: nil - @client_stub = client_stub - @stream = stream - @upload_size = upload_size - @content_type = content_type - @timeout = timeout - @control_plane_retry_policy = control_plane_retry_policy - @data_plane_retry_policy = data_plane_retry_policy - @on_progress = on_progress - @logger = logger - - @mutex = Mutex.new - @running = false - @executed = false - @upload_url = nil - @last_driver = nil - end - - ## - # Returns the raw upload session URL if established. - # - # @return [String, nil] - def upload_url - @mutex.synchronize { upload_url_internal } - end - - ## - # Returns whether the session is bound to a server-side upload. - # - # @return [Boolean] - def bound? - @mutex.synchronize { bound_internal? } - end - - ## - # Returns the current {ResumeHandle} if the session is alive and resumable. - # Completed uploads are not resumable (returns nil). Rejected uploads and - # cancelled uploads are also finalized and not resumable, returning nil. - # - # @return [ResumeHandle, nil] - def resume_handle - @mutex.synchronize { resume_handle_internal } - end - - ## - # Returns whether a new session can resume the upload. - # Completed uploads are not resumable (returns false). Rejected uploads and - # cancelled uploads are also finalized and not resumable (returns false). - # - # @return [Boolean] - def resumable? - @mutex.synchronize { !resume_handle_internal.nil? } - end - - ## - # Returns whether a run is currently executing. - # - # @return [Boolean] - def running? - @mutex.synchronize { @running } - end - - ## - # Starts a new upload session on the server. - # - # A session performs exactly one run (`start` or `resume`). Calling `start` on an already-bound - # or executed session raises {SessionStateError}. Precondition: the stream is assumed to be - # positioned at byte 0 (the session does not rewind it before reading) and is not closed after use. - # - # Blocks the calling thread until the upload completes or fails. - # - # @example Uploading a file with progress reporting - # session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: File.open("movie.mp4", "rb"), - # upload_size: File.size("movie.mp4"), - # content_type: "video/mp4", - # on_progress: ->(progress) { puts "#{progress.phase}: #{progress.bytes_uploaded} bytes" } - # ) - # response = session.start initial_url: "https://example.googleapis.com/upload/v1/media" - # - # @param initial_url [String] Initial endpoint URI for session initiation - # @param initial_body [String, nil] Request payload for session initiation - # @param initial_headers [Hash] Additional headers for the initiation request. - # The five reserved protocol headers (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, - # `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, - # `X-Goog-Upload-Header-Content-Length`) are rejected with an `ArgumentError` in any casing — - # they carry protocol mechanics the session owns. Use the constructor's `content_type` and - # `upload_size` to shape the media descriptors. Pass-through headers such as - # `X-Goog-Upload-Header-Content-Disposition` are permitted. - # @param chunk_size [Integer, nil] Requested chunk size in bytes, defaulting to 8 MB. The effective - # size is rounded down to a multiple of any chunk granularity the server requires, or raised to that - # granularity if it exceeds the requested size. - # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for the initiation - # request (`start`). See the "Retry Policies" section in the class documentation. - # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the - # response carried no body), typically the JSON resource the backend created that the caller - # parses. A client stub carrying response-decoding middleware is outside the contract. - # @raise [ArgumentError] If `initial_url` is missing or blank, if `initial_headers` sets a - # reserved protocol header, or if a retry policy argument is neither a - # {Gapic::Common::RetryPolicy}, a Hash, nor `nil` - # @raise [SessionStateError] If already bound/executed or if a run is currently in progress - # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs - # @raise [DeadlineExceededError] If the global upload timeout is exceeded - # @raise [BadResponseError] If an unexpected or malformed HTTP response is received - # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream - # @raise [StreamMismatchError] If stream content or length does not match protocol expectations - # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state - # @raise [UploadRejectedError] If the server explicitly rejects the upload session - # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug - # in this library rather than a caller or server error - def start initial_url:, - initial_body: nil, - initial_headers: {}, - chunk_size: nil, - start_retry_policy: nil - config = build_start_config initial_url: initial_url, - initial_body: initial_body, - initial_headers: initial_headers, - chunk_size: chunk_size, - start_retry_policy: start_retry_policy - - driver = nil - @mutex.synchronize do - raise SessionStateError, "A run is already in progress for this session" if @running - raise SessionStateError, "Session has already executed a run" if bound_internal? - - driver = Driver.new client_stub: @client_stub, config: config, logger: @logger - @executed = true - @running = true - end - - execute_run driver - end - - ## - # Resumes an upload session using one of two explicit keyword forms: - # 1. `resume(upload_url:, chunk_size:)`: Resumes with explicit URL and chunk size. - # 2. `resume(resume_handle:)`: Resumes via {ResumeHandle}. - # - # A session performs exactly one run (`start` or `resume`). Resuming must be executed on a - # fresh, unexecuted session. Blocks the calling thread until the upload completes or fails. - # - # A resumed run targets an upload the server has already created, so it takes no initiation - # arguments; everything it needs beyond the constructor is on this method. - # - # ### Chunk size - # - # A chunk size must be given explicitly because the server reports chunk granularity during - # initiation, which a resumed run skips. {ResumeHandle} carries the effective value from the original - # run for exactly this reason. - # - # ### Stream position - # - # The stream must be positioned at byte 0 of the whole object, not at the server's acknowledged - # offset, and is not closed after use. The Driver fast-forwards on its own, by seeking on seekable - # streams or by reading and discarding on unseekable ones. An unseekable stream therefore has to be - # freshly opened rather than rewound. - # - # A completed upload is finalized: {#resume_handle} returns `nil` and {#resumable?} returns - # `false`, so there is no handle to resume from. Calling `#resume` on the session that completed - # the run raises {SessionStateError}, as it would after any run. Resuming a *fresh* session - # against a finalized `upload_url` is undefined behavior: it queries the server and might return - # the response body or raise an error, depending on the server response. - # - # @example Resuming from a handle persisted by an earlier process - # handle = Gapic::Rest::ResumableUpload::ResumeHandle.new( - # upload_url: row[:upload_url], - # chunk_size: row[:chunk_size] - # ) - # session = Gapic::Rest::ResumableUpload::Session.new( - # client_stub: client_stub, - # stream: File.open("movie.mp4", "rb"), - # upload_size: File.size("movie.mp4") - # ) - # response = session.resume resume_handle: handle - # - # @param upload_url [String, nil] Explicit upload URL - # @param chunk_size [Integer, nil] Explicit chunk size - # @param resume_handle [ResumeHandle, nil] Explicit resume handle - # @return [String, nil] Raw, undecoded body of the finalizing HTTP response (or `nil` if the - # response carried no body), typically the JSON resource the backend created that the caller - # parses. A client stub carrying response-decoding middleware is outside the contract. - # @raise [ArgumentError] If argument shape is invalid, target upload is missing, or stream.pos != 0 - # @raise [SessionStateError] If already bound/executed or if a run is currently in progress - # @raise [RequestFailedError] If a transport error, timeout, or retry exhaustion occurs - # @raise [DeadlineExceededError] If the global upload timeout is exceeded - # @raise [BadResponseError] If an unexpected or malformed HTTP response is received - # @raise [UnseekableStreamError] If stream rewinding is required during recovery on an unseekable stream - # @raise [StreamMismatchError] If stream content or length does not match the resumed upload - # @raise [InvalidTransitionError] If an unmatched event occurs for the current protocol state - # @raise [UploadRejectedError] If the server explicitly rejects the upload session - # @raise [InternalError] If the library detects an internal invariant breach; this signals a bug - # in this library rather than a caller or server error - def resume upload_url: nil, - chunk_size: nil, - resume_handle: nil - target_url, target_chunk_size = resolve_resume_args( - upload_url: upload_url, - chunk_size: chunk_size, - resume_handle: resume_handle - ) - - driver = nil - @mutex.synchronize do - raise SessionStateError, "A run is already in progress for this session" if @running - raise SessionStateError, "Session has already executed a run" if bound_internal? - - if @stream.respond_to?(:pos) && !@stream.pos.zero? - raise ArgumentError, "Stream must be positioned at byte 0 to resume an upload (got pos #{@stream.pos})" - end - - config = build_resume_config target_url, target_chunk_size - driver = Driver.new client_stub: @client_stub, config: config, logger: @logger - @executed = true - @running = true - @upload_url = target_url - end - - execute_run driver - end - - private - - ## - # @private - # Returns the established upload URL without locking. - # - # @return [String, nil] - def upload_url_internal - @upload_url || @last_driver&.upload_url - end - - ## - # @private - # Returns whether the session is bound without locking. - # - # @return [Boolean] - def bound_internal? - @executed || !upload_url_internal.nil? - end - - ## - # @private - # Returns the current resume handle from the driver without locking. - # - # @return [ResumeHandle, nil] - def resume_handle_internal - @last_driver&.resume_handle - end - - ## - # @private - # Returns the configuration members shared by both run types, mirroring - # {ResumableUpload::COMMON_MEMBERS}. - # - # @return [Hash{Symbol=>Object}] - def common_config_args - { - stream: @stream, - upload_size: @upload_size, - content_type: @content_type, - timeout: @timeout, - control_plane_retry_policy: @control_plane_retry_policy, - data_plane_retry_policy: @data_plane_retry_policy, - on_progress: @on_progress - } - end - - ## - # @private - # Builds configuration for a new upload session. - # - # @param initial_url [String] Initial endpoint URI for session initiation - # @param initial_body [String, nil] Request payload for session initiation - # @param initial_headers [Hash, nil] Additional headers for initiation - # @param chunk_size [Integer, nil] Requested chunk size in bytes - # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Initiation retry policy - # @return [StartUploadConfig] - def build_start_config initial_url:, initial_body:, initial_headers:, chunk_size:, start_retry_policy: - StartUploadConfig.new( - initial_url: initial_url, - initial_body: initial_body, - initial_headers: initial_headers || {}, - chunk_size: chunk_size, - start_retry_policy: start_retry_policy, - **common_config_args - ) - end - - ## - # @private - # Builds configuration for resuming an upload session. - # - # @param target_url [String] Target upload session URL - # @param target_chunk_size [Integer] Effective chunk size in bytes - # @return [ResumeUploadConfig] - def build_resume_config target_url, target_chunk_size - ResumeUploadConfig.new( - upload_url: target_url, - chunk_size: target_chunk_size, - **common_config_args - ) - end - - ## - # @private - # Validates and extracts target upload URL and chunk size from resume keyword arguments. - # - # @param upload_url [String, nil] Explicit upload URL - # @param chunk_size [Integer, nil] Explicit chunk size - # @param resume_handle [ResumeHandle, nil] Explicit resume handle - # @return [Array] Tuple of [upload_url, chunk_size] - # @raise [ArgumentError] If arguments are missing or mutually exclusive - def resolve_resume_args upload_url:, chunk_size:, resume_handle: - if resume_handle - raise ArgumentError, "Cannot pass both resume_handle and upload_url/chunk_size" if upload_url || chunk_size - [resume_handle.upload_url, resume_handle.chunk_size] - elsif upload_url - raise ArgumentError, "Must provide chunk_size with upload_url" if chunk_size.nil? - [upload_url, chunk_size] - elsif chunk_size - raise ArgumentError, "Cannot pass chunk_size without upload_url" - else - raise ArgumentError, "Must provide either resume_handle or upload_url and chunk_size" - end - end - - ## - # @private - # Executes the driver run and records the final upload URL and state. - # - # @param driver [Driver] Driver instance to run - # @return [String, nil] Final response body upon completion - def execute_run driver - @mutex.synchronize { @last_driver = driver } - result = driver.run - @mutex.synchronize do - @upload_url ||= driver.upload_url - @running = false - end - result - rescue StandardError - @mutex.synchronize do - @upload_url ||= driver.upload_url - @running = false - end - raise - end - end - end - end -end diff --git a/gapic-common/lib/gapic/resumable_upload.rb b/gapic-common/lib/gapic/resumable_upload.rb new file mode 100644 index 0000000..c65b717 --- /dev/null +++ b/gapic-common/lib/gapic/resumable_upload.rb @@ -0,0 +1,507 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "gapic/rest/resumable_upload" + +module Gapic + ## + # Coordinates resumable uploads for a client method that performs them. + # + # A client method that uploads media returns one of these handles instead of a response. No request is + # sent and no byte is read from the stream until {#start} or {#resume} is called on it. Both are + # synchronous: they block the calling thread for the whole upload and return the decoded response + # message. + # + # ### Reusable + # + # A handle is reusable, and a failed run is resumed on the same object: + # + # @example Uploading, then resuming after a recoverable failure + # upload = client.create_media_upload request + # begin + # upload.start stream: File.open("movie.mp4", "rb"), content_type: "video/mp4" + # rescue Gapic::Rest::ResumableUpload::HasResumeHandle => e + # raise unless upload.resumable? + # upload.resume stream: File.open("movie.mp4", "rb") + # end + # + # The stream handed to {#resume} must be positioned at byte 0 of the whole object, not at the server's + # acknowledged offset; the upload fast-forwards on its own, by seeking on a seekable stream or by + # reading and discarding on an unseekable one. An unseekable stream therefore has to be freshly opened + # rather than rewound. + # + # A run that failed in a way the protocol can recover from leaves a {Gapic::Rest::ResumableUpload::ResumeHandle} + # behind, readable from {#resume_handle} and also carried on the error. Persisting that handle lets a + # later process resume the same upload: + # + # @example Resuming an upload started by an earlier process + # upload = client.create_media_upload + # upload.resume stream: File.open("movie.mp4", "rb"), + # resume_handle: Gapic::Rest::ResumableUpload::ResumeHandle.new( + # upload_url: row[:upload_url], chunk_size: row[:chunk_size] + # ) + # + # A completed upload is finalized: {#resume_handle} returns `nil` and {#resumable?} returns `false`, so + # there is no handle to resume from. Calling {#start} again is permitted and begins a second, unrelated + # upload. + # + # ### The Two Timeouts + # + # An upload is bounded by two independent budgets, and they are three orders of magnitude apart: + # + # | Budget | Set by | Covers | + # |---|---|---| + # | whole upload | `upload_timeout:` on {#start} and {#resume} | every request, retry and byte of the run | + # | initiation request | per-call `timeout`, or `timeout:` in `start_retry_policy` | creating the session | + # + # The per-call `timeout` a client method takes reaches only the initiation request. An upload still + # transferring bytes an hour later has long outlived it, and that is expected. To bound the run as a + # whole, pass `upload_timeout:`. + # + # ### Threading + # + # {#start} and {#resume} block the calling thread, and the `on_progress` callback runs on that same + # thread. The readers ({#resume_handle}, {#resumable?}, {#running?}) are guarded by an internal mutex and + # may be called from another thread mid-run; values read that way are a best-effort snapshot of a state + # the upload thread is still advancing. + # + # ### Defaults + # + # * `chunk_size` defaults to 8 MB, then rounds down to a multiple of any chunk granularity the server + # requires. + # * `upload_timeout` defaults to `upload_size / 1 MB per second` when `upload_size` is known, floored at + # one hour, and to one hour flat when it is not. + # + # ### Retry Policies + # + # Retry behavior is partitioned across three policies. Only the initiation policy is caller-supplied; + # the other two are the protocol's own and govern the requests no call option describes. + # + # All three share the same default retry codes and exponential backoff settings: + # + # | Setting | Default | + # |---|---| + # | `retry_codes` | `UNAVAILABLE`, `DEADLINE_EXCEEDED`, `RESOURCE_EXHAUSTED`, `INTERNAL` | + # | `initial_delay` | `1.0` s | + # | `max_delay` | `15.0` s | + # | `multiplier` | `1.3` | + # + # They differ in which requests they govern and how a missing or empty `X-Goog-Upload-Status` response + # header is treated: + # + # | Policy | Governs | Missing status header | + # |---|---|---| + # | initiation | session initiation | **Retriable** on any status (incl. `200`), unless fatal | + # | control plane | `query` and `cancel` | No predicate; decided on `retry_codes` alone | + # | data plane | `upload` and `finalize` | **Not** retriable | + # + # Initiation treats a response missing `X-Goog-Upload-Status` as gateway noise worth retrying; the data + # plane treats it as a response it cannot interpret and refuses to replay bytes against it. + # + class ResumableUpload + ## + # @private + # Builds a coordinator for one client method call. + # + # Instances come from generated client methods; the arguments below are what such a method has to + # hand over, not a surface a caller assembles. + # + # Both procs are deferred deliberately. `client_stub_proc` lets a client that cannot perform REST + # calls hand back a working handle and fail only when an upload is actually attempted. + # `initial_request_proc` means the initiation URL and body are computed on {#start} and never on + # {#resume}, so a handle built without a request message is still fully functional for resuming. + # + # @param client_stub_proc [Proc] Returns the {Gapic::Rest::ClientStub} to upload through. Called at + # the top of every run, and may raise if the client cannot perform REST calls. + # @param initial_request_proc [Proc] Returns the `[url, body]` pair for session initiation. Called by + # {#start} only. + # @param response_type [Class, nil] Protobuf message class the final response body is decoded into. + # `nil` returns the raw body; see {#start}. + # @param initial_headers [Hash] Headers for the initiation request. Keys and values are stringified. + # The five reserved protocol headers (`X-Goog-Upload-Protocol`, `X-Goog-Upload-Command`, + # `X-Goog-Upload-Offset`, `X-Goog-Upload-Header-Content-Type`, `X-Goog-Upload-Header-Content-Length`) + # are rejected in any casing; use `content_type` and `upload_size` on the run methods instead. + # @param start_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for the initiation + # request. A {Gapic::Common::RetryPolicy} replaces the default policy outright; a Hash overrides only + # the settings it names. See the "Retry Policies" section in the class documentation. + # @param control_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for `query` + # and `cancel`. `nil` uses the protocol default. + # @param data_plane_retry_policy [Gapic::Common::RetryPolicy, Hash, nil] Retry policy for `upload` and + # `finalize`. `nil` uses the protocol default. + # @param error_handler [Proc, nil] Called with a run failure and **returns** the exception to raise in + # its place. It must not raise. + # @param method_name [String, nil] RPC name used in log entries. + # + def initialize client_stub_proc:, + initial_request_proc:, + response_type:, + initial_headers: {}, + start_retry_policy: nil, + control_plane_retry_policy: nil, + data_plane_retry_policy: nil, + error_handler: nil, + method_name: nil + @client_stub_proc = client_stub_proc + @initial_request_proc = initial_request_proc + @response_type = response_type + @initial_headers = stringify_headers initial_headers + @start_retry_policy = start_retry_policy + @control_plane_retry_policy = control_plane_retry_policy + @data_plane_retry_policy = data_plane_retry_policy + @error_handler = error_handler + @method_name = method_name + + @mutex = Mutex.new + @running = false + @driver = nil + end + + ## + # Creates an upload session on the server and transfers the stream into it. + # + # Blocks the calling thread until the upload completes or fails. The stream is assumed to be + # positioned at byte 0 (it is not rewound before reading) and is not closed after use. + # + # @example + # response = upload.start stream: File.open("movie.mp4", "rb"), + # content_type: "video/mp4", + # upload_size: File.size("movie.mp4"), + # upload_timeout: 4 * 3600, + # on_progress: ->(p) { puts "#{p.phase}: #{p.bytes_uploaded}" } + # + # @param stream [IO] Binary input stream to upload, positioned at byte 0. + # @param content_type [String, nil] MIME type of the uploaded media. + # @param upload_size [Integer, nil] Total upload bytes, if known upfront. + # @param chunk_size [Integer, nil] Requested chunk size in bytes, defaulting to 8 MB. The effective + # size is rounded down to a multiple of any chunk granularity the server requires, or raised to that + # granularity if it exceeds the requested size. A resumed run has no such argument: it takes its + # chunk size from the {Gapic::Rest::ResumableUpload::ResumeHandle}. + # @param upload_timeout [Numeric, nil] Budget in seconds for the **whole run** — every request, every + # retry, every byte — not for any single request. The per-call `timeout` a client method takes bounds + # the initiation request alone. When `nil`, resolves to `upload_size / 1 MB per second` floored at one + # hour if `upload_size` is known, and to one hour flat otherwise. + # @param on_progress [Proc, nil] Called as `->(progress)` with a {Gapic::Rest::ResumableUpload::Progress} + # instance. Runs synchronously on the upload thread and must not block; an exception raised inside it + # aborts the run and propagates out of this method. + # @return [Object] The final response decoded into the handle's response type, or the raw response body + # (a String, or `nil` when the final response carried none) when the handle has no response type. + # @raise [ArgumentError] If the initiation headers set a reserved protocol header, if a retry policy is + # neither a {Gapic::Common::RetryPolicy}, a Hash, nor `nil`, or if the client cannot perform REST calls + # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress + # @raise [Gapic::Rest::ResumableUpload::RequestFailedError] If a transport error, timeout, or retry + # exhaustion occurs + # @raise [Gapic::Rest::ResumableUpload::DeadlineExceededError] If `upload_timeout` is exceeded + # @raise [Gapic::Rest::ResumableUpload::BadResponseError] If an unexpected or malformed HTTP response + # is received + # @raise [Gapic::Rest::ResumableUpload::UnseekableStreamError] If stream rewinding is required during + # recovery on an unseekable stream + # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match + # protocol expectations + # @raise [Gapic::Rest::ResumableUpload::InvalidTransitionError] If an unmatched event occurs for the + # current protocol state + # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload + # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant + # breach; this signals a bug in this library rather than a caller or server error + # + def start stream:, + content_type: nil, + upload_size: nil, + chunk_size: nil, + upload_timeout: nil, + on_progress: nil + execute_run do + client_stub = @client_stub_proc.call + initial_url, initial_body = @initial_request_proc.call + config = ::Gapic::Rest::ResumableUpload::StartUploadConfig.new( + initial_url: initial_url, + initial_body: initial_body, + initial_headers: @initial_headers, + chunk_size: chunk_size, + start_retry_policy: @start_retry_policy, + **run_config_args(stream: stream, content_type: content_type, upload_size: upload_size, + upload_timeout: upload_timeout, on_progress: on_progress) + ) + build_driver client_stub, config + end + end + + ## + # Resumes an upload session the server has already created, transferring whatever it has not yet + # acknowledged. + # + # Blocks the calling thread until the upload completes or fails. A resumed run sends no initiation + # request, so it takes no initiation arguments and needs no request message. + # + # The target is a {Gapic::Rest::ResumableUpload::ResumeHandle}: either the one passed in, or — when + # `resume_handle` is omitted — the one left behind by this handle's last run. The bare form raises + # `ArgumentError` when there is none, which covers a handle that has never run, a run that finished + # successfully, and a run that failed in a way the protocol considers unresumable. + # + # Resuming against a finalized upload URL is undefined behavior: it queries the server and might + # return the response body or raise an error, depending on the server response. + # + # @param stream [IO] Binary input stream to upload, positioned at byte 0 of the **whole object**, not + # at the server's acknowledged offset. The upload fast-forwards on its own, by seeking or by reading + # and discarding. Not closed after use. + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Upload to resume. Defaults to + # the one left behind by this handle's last run. + # @param content_type [String, nil] MIME type of the uploaded media. + # @param upload_size [Integer, nil] Total upload bytes, if known upfront. + # @param upload_timeout [Numeric, nil] Budget in seconds for the **whole run**. See {#start}. + # @param on_progress [Proc, nil] Called as `->(progress)` with a {Gapic::Rest::ResumableUpload::Progress} + # instance. Runs synchronously on the upload thread and must not block. + # @return [Object] The final response decoded into the handle's response type, or the raw response body + # (a String, or `nil` when the final response carried none) when the handle has no response type. + # @raise [ArgumentError] If there is no upload to resume, if the stream is not positioned at byte 0, if + # a retry policy is neither a {Gapic::Common::RetryPolicy}, a Hash, nor `nil`, or if the client cannot + # perform REST calls + # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress + # @raise [Gapic::Rest::ResumableUpload::RequestFailedError] If a transport error, timeout, or retry + # exhaustion occurs + # @raise [Gapic::Rest::ResumableUpload::DeadlineExceededError] If `upload_timeout` is exceeded + # @raise [Gapic::Rest::ResumableUpload::BadResponseError] If an unexpected or malformed HTTP response + # is received + # @raise [Gapic::Rest::ResumableUpload::UnseekableStreamError] If stream rewinding is required during + # recovery on an unseekable stream + # @raise [Gapic::Rest::ResumableUpload::StreamMismatchError] If stream content or length does not match + # the resumed upload + # @raise [Gapic::Rest::ResumableUpload::InvalidTransitionError] If an unmatched event occurs for the + # current protocol state + # @raise [Gapic::Rest::ResumableUpload::UploadRejectedError] If the server explicitly rejects the upload + # @raise [Gapic::Rest::ResumableUpload::InternalError] If the library detects an internal invariant + # breach; this signals a bug in this library rather than a caller or server error + # + def resume stream:, + resume_handle: nil, + content_type: nil, + upload_size: nil, + upload_timeout: nil, + on_progress: nil + execute_run do + verify_stream_at_origin stream + handle = resolve_resume_handle resume_handle + client_stub = @client_stub_proc.call + config = ::Gapic::Rest::ResumableUpload::ResumeUploadConfig.new( + upload_url: handle.upload_url, + chunk_size: handle.chunk_size, + **run_config_args(stream: stream, content_type: content_type, upload_size: upload_size, + upload_timeout: upload_timeout, on_progress: on_progress) + ) + build_driver client_stub, config + end + end + + ## + # Returns the handle needed to resume the last run, carrying its upload URL and resolved chunk size. + # + # `nil` before the first run, and after any run that left nothing to resume: a completed upload is + # finalized, and rejected and cancelled uploads are too. + # + # @return [Gapic::Rest::ResumableUpload::ResumeHandle, nil] + def resume_handle + @mutex.synchronize { @driver&.resume_handle } + end + + ## + # Returns whether the last run left an upload that can be resumed. + # + # @return [Boolean] + def resumable? + !resume_handle.nil? + end + + ## + # Returns whether a run is currently executing. + # + # @return [Boolean] + def running? + @mutex.synchronize { @running } + end + + private + + ## + # @private + # Claims the single run slot, runs the driver built by the block, and releases the slot. + # + # The slot is claimed before the block runs, so a second concurrent run is rejected while this one is + # still building its configuration. It is released in an `ensure`, so every exit — a clean return, a + # protocol error, an `on_progress` callback raising, a `Thread#kill` — leaves the handle usable and the + # driver retained. A failure inside the block leaves no driver retained at all, so a configuration + # `ArgumentError` cannot disturb the resume handle of an earlier run. + # + # @yieldreturn [Gapic::Rest::ResumableUpload::Driver] Driver to run + # @return [Object] Decoded final response + def execute_run + claim_run_slot + begin + driver = yield + @mutex.synchronize { @driver = driver } + run_driver driver + ensure + @mutex.synchronize { @running = false } + end + end + + ## + # @private + # Runs a driver and decodes its result, applying the caller's error handler to a failure. + # + # Only the run is wrapped. Argument and configuration errors are raised while the driver is still + # being built, and reach the caller as themselves: they describe a call that was never made. + # + # @param driver [Gapic::Rest::ResumableUpload::Driver] Driver to run + # @return [Object] Decoded final response + def run_driver driver + decode_response driver.run + rescue ::StandardError => e + raise wrap_error(e) + end + + ## + # @private + # Marks a run as in flight, rejecting a second concurrent one. + # + # @return [void] + # @raise [Gapic::Rest::ResumableUpload::SessionStateError] If a run is already in progress + def claim_run_slot + @mutex.synchronize do + if @running + raise ::Gapic::Rest::ResumableUpload::SessionStateError, + "A run is already in progress for this upload" + end + @running = true + end + end + + ## + # @private + # Builds the driver for a run. + # + # @param client_stub [Gapic::Rest::ClientStub] Stub returned by `client_stub_proc` + # @param config [Gapic::Rest::ResumableUpload::StartUploadConfig, + # Gapic::Rest::ResumableUpload::ResumeUploadConfig] Configuration for this run + # @return [Gapic::Rest::ResumableUpload::Driver] + def build_driver client_stub, config + ::Gapic::Rest::ResumableUpload::Driver.new client_stub: client_stub, + config: config, + method_name: @method_name + end + + ## + # @private + # Returns the configuration members both run types share, mirroring + # {Gapic::Rest::ResumableUpload::COMMON_MEMBERS}. + # + # @return [Hash{Symbol=>Object}] + def run_config_args stream:, content_type:, upload_size:, upload_timeout:, on_progress: + { + stream: stream, + upload_size: upload_size, + content_type: content_type, + timeout: upload_timeout, + control_plane_retry_policy: @control_plane_retry_policy, + data_plane_retry_policy: @data_plane_retry_policy, + on_progress: on_progress + } + end + + ## + # @private + # Resolves the upload a resumed run targets. + # + # @param resume_handle [Gapic::Rest::ResumableUpload::ResumeHandle, nil] Explicit handle, if given + # @return [Gapic::Rest::ResumableUpload::ResumeHandle] + # @raise [ArgumentError] If no handle was given and the last run left none + def resolve_resume_handle resume_handle + return resume_handle if resume_handle + + handle = @mutex.synchronize { @driver&.resume_handle } + if handle.nil? + raise ArgumentError, + "No upload to resume: this handle has not run, or its last run left nothing resumable. " \ + "Pass resume_handle: to resume an upload started elsewhere." + end + handle + end + + ## + # @private + # Rejects a stream that is not positioned at byte 0. Streams that do not report a position are + # trusted. + # + # @param stream [IO] Stream a resumed run will read + # @return [void] + # @raise [ArgumentError] If the stream reports a non-zero position + def verify_stream_at_origin stream + return unless stream.respond_to? :pos + return if stream.pos.zero? + + raise ArgumentError, "Stream must be positioned at byte 0 to resume an upload (got pos #{stream.pos})" + end + + ## + # @private + # Decodes the final response body into the handle's response type. + # + # A `nil` response type returns the raw body, exactly as the driver produced it. Generated call sites + # always pass a message class; the raw form exists so this gem's own tests can assert on what the + # server sent without decoding through a message type they do not have. + # + # @param body [String, nil] Raw body of the finalizing HTTP response + # @return [Object] Decoded message, or the raw body when there is no response type + def decode_response body + return body if @response_type.nil? + + @response_type.decode_json body.to_s, ignore_unknown_fields: true + end + + ## + # @private + # Applies the caller's error handler to a run failure. + # + # The handler returns the exception to raise. A replacement that loses the + # {Gapic::Rest::ResumableUpload::HasResumeHandle} mixin is re-extended with it and given the original's + # handle, so a library-specific error type cannot erase the fact that the upload is resumable. + # + # @param error [StandardError] Failure raised by the run + # @return [Exception] Exception to raise in its place + def wrap_error error + return error unless @error_handler + + wrapped = @error_handler.call error + return error if wrapped.nil? || wrapped.equal?(error) + + if error.is_a?(::Gapic::Rest::ResumableUpload::HasResumeHandle) && + !wrapped.is_a?(::Gapic::Rest::ResumableUpload::HasResumeHandle) + wrapped.extend ::Gapic::Rest::ResumableUpload::HasResumeHandle + wrapped.instance_variable_set :@resume_handle, error.resume_handle + end + wrapped + end + + ## + # @private + # Stringifies the keys and values of the initiation headers, which generated clients carry as a + # symbol-keyed metadata hash. + # + # @param headers [Hash, nil] Initiation headers + # @return [Hash{String=>String}] + def stringify_headers headers + (headers || {}).to_h { |key, value| [key.to_s, value.to_s] } + end + end +end diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb index 4cf93d2..9161f72 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_logging_test.rb @@ -92,6 +92,34 @@ def test_all_entries_share_upload_id_and_pass_method_names assert_includes ["complete_upload_with_data", "complete_upload_finalized"], info_recipes.last end + def test_method_name_prefixes_per_request_log_names + responses = [ + FakeResponse.new( + 200, + { + "X-Goog-Upload-Status" => "active", + "X-Goog-Upload-URL" => "https://storage.googleapis.com/session?id=123" + }, + "" + ), + FakeResponse.new(200, { "X-Goog-Upload-Status" => "final" }, "done") + ] + + stub = FakeStub.new responses + config = StartUploadConfig.new( + initial_url: "https://storage.googleapis.com/upload", + stream: StringIO.new("hello world"), + upload_size: 11, + chunk_size: 256 + ) + + driver = Driver.new client_stub: stub, config: config, logger: RecordingLogger.new, + method_name: "create_media_upload" + driver.run + + assert_equal ["create_media_upload.start", "create_media_upload.upload"], stub.method_names + end + def test_multi_chunk_upload_logs_lifecycle_entries recording = RecordingLogger.new run_two_chunk_upload_with_secret recording diff --git a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb index b5d73df..2a99c6a 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/driver_test.rb @@ -427,4 +427,35 @@ def assert_chunk_request req, offset:, length:, body:, finalize: assert_equal length, metadata["Content-Length"] assert_equal body, req[:body] end + + # `upload_url` reports the session URL whatever the lifecycle status, while `resume_handle` reports one + # only while the upload is still resumable. + def test_upload_url_and_resume_handle_across_statuses + config = StartUploadConfig.new( + initial_url: "https://example.com/upload", + stream: StringIO.new("data"), + upload_size: 4, + chunk_size: 4 + ) + driver = Driver.new client_stub: FakeClientStub.new([]), config: config + + assert_nil driver.upload_url + + set_driver_status driver, :transmission_sending + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url + + [:rejected, :cancelled, :success].each do |status| + set_driver_status driver, status + assert_equal "https://upload.example.com/sess1", driver.upload_url + assert_nil driver.resume_handle + end + end + + def set_driver_status driver, status + driver.core.instance_variable_set( + :@state, + driver.core.state.with(status: status, upload_url: "https://upload.example.com/sess1") + ) + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb index da1a692..84b723a 100644 --- a/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb +++ b/gapic-common/test/gapic/rest/resumable_upload/retry_policies_test.rb @@ -219,4 +219,68 @@ def test_default_data_plane_no_headers_falls_back_to_codes err_no_code = RuntimeError.new "generic network error" refute policy.retry_error?(err_no_code) end + + # ============================================================================ + # SUT: start_retry_policy_for + # ============================================================================ + + def test_start_retry_policy_for_always_carries_the_call_timeout + options = Gapic::CallOptions.new timeout: 17 + + assert_equal({ timeout: 17 }, Gapic::Rest::ResumableUpload.start_retry_policy_for(options)) + end + + def test_start_retry_policy_for_carries_a_nil_timeout + assert_equal({ timeout: nil }, Gapic::Rest::ResumableUpload.start_retry_policy_for(Gapic::CallOptions.new)) + end + + def test_start_retry_policy_for_tolerates_nil_options + assert_equal({ timeout: nil }, Gapic::Rest::ResumableUpload.start_retry_policy_for(nil)) + end + + def test_start_retry_policy_for_copies_only_customized_backoff_settings + options = Gapic::CallOptions.new timeout: 5, retry_policy: { initial_delay: 0.5 } + + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + assert_equal 0.5, overrides[:initial_delay] + refute overrides.key?(:max_delay) + refute overrides.key?(:multiplier) + end + + def test_start_retry_policy_for_treats_empty_retry_codes_as_unset + options = Gapic::CallOptions.new retry_policy: { initial_delay: 0.5 } + + refute Gapic::Rest::ResumableUpload.start_retry_policy_for(options).key?(:retry_codes) + end + + def test_start_retry_policy_for_copies_retry_codes_when_given + options = Gapic::CallOptions.new retry_policy: { retry_codes: ["UNAVAILABLE"] } + + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + assert_equal [Gapic::Common::ErrorCodes::ERROR_STRING_MAPPING["UNAVAILABLE"]], overrides[:retry_codes] + end + + # Applying the overrides to the initiation defaults must leave the missing-status-header predicate in + # place: that is the whole reason the conversion returns a Hash rather than a policy object. + def test_start_retry_policy_for_overrides_leave_the_start_predicate_in_place + options = Gapic::CallOptions.new timeout: 5, retry_policy: { initial_delay: 0.5 } + overrides = Gapic::Rest::ResumableUpload.start_retry_policy_for options + + policy = Gapic::Common::RetryPolicy.new(**overrides).apply_defaults RetryPolicies::START_DEFAULTS + + assert_equal 0.5, policy.initial_delay + assert_equal 5, policy.timeout + assert_same RetryPolicies::START_PREDICATE, policy.retry_predicate + end + + def test_start_retry_policy_for_rejects_a_proc_retry_policy + options = Gapic::CallOptions.new retry_policy: ->(_error) { true } + + error = assert_raises ArgumentError do + Gapic::Rest::ResumableUpload.start_retry_policy_for options + end + assert_match(/cannot derive an initiation retry policy/, error.message) + end end diff --git a/gapic-common/test/gapic/rest/resumable_upload/session_test.rb b/gapic-common/test/gapic/rest/resumable_upload/session_test.rb deleted file mode 100644 index fe6e782..0000000 --- a/gapic-common/test/gapic/rest/resumable_upload/session_test.rb +++ /dev/null @@ -1,704 +0,0 @@ -# frozen_string_literal: true - -# Copyright 2026 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require "test_helper" -require "gapic/rest/resumable_upload" -require "stringio" - -class SessionTest < Minitest::Test - include Gapic::Rest::ResumableUpload - - FakeResponse = Struct.new :status, :headers, :body, keyword_init: true - - class ScriptedClientStub - attr_reader :requests - - def initialize responses = [] - @responses = responses.dup - @requests = [] - end - - def make_post_request uri:, body:, params:, options:, method_name: nil - @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } - raise "Unexpected request: no scripted response left" if @responses.empty? - - res = @responses.shift - if res.is_a? Proc - res.call - elsif res.is_a? Exception - raise res - else - res - end - end - end - - class UnseekableStream - attr_reader :pos - - def initialize string - @io = StringIO.new string - @pos = 0 - end - - def read length = nil - chunk = @io.read length - @pos += chunk.bytesize if chunk - chunk - end - end - - class StreamWithoutPos - def initialize string - @io = StringIO.new string - end - - def read length = nil - @io.read length - end - end - - START_ONLY_KEYS = [:initial_url, :initial_body, :initial_headers, :chunk_size, :start_retry_policy].freeze - - # Builds a session from the shared arguments and remembers the per-run arguments that #start needs, - # so tests can keep calling `start_session session`. - def build_session stub: nil, stream: nil, upload_size: 10, chunk_size: 4, **kwargs - stream ||= StringIO.new "0123456789" - stub ||= ScriptedClientStub.new - @start_args = { - initial_url: "https://example.com/initiate", - initial_body: '{"name":"test.txt"}', - chunk_size: chunk_size - }.merge(kwargs.slice(*START_ONLY_KEYS)) - - Session.new( - client_stub: stub, - stream: stream, - upload_size: upload_size, - **kwargs.except(*START_ONLY_KEYS) - ) - end - - def start_session session, **overrides - session.start(**@start_args, **overrides) - end - - # ============================================================================ - # 1. Initialization and argument validation - # ============================================================================ - - def test_initialize_mandatory_arguments - assert_raises ArgumentError do - Session.new stream: StringIO.new - end - - assert_raises ArgumentError do - Session.new client_stub: ScriptedClientStub.new - end - end - - def test_initialize_rejects_per_run_arguments - assert_raises ArgumentError do - Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, initial_url: "http://x" - end - - assert_raises ArgumentError do - Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new, chunk_size: 4 - end - end - - def test_initialize_defaults - session = Session.new( - client_stub: ScriptedClientStub.new, - stream: StringIO.new("abc"), - upload_size: 300 - ) - - assert_equal 300, session.upload_size - assert_nil session.content_type - assert_nil session.timeout - assert_nil session.control_plane_retry_policy - assert_nil session.data_plane_retry_policy - assert_nil session.on_progress - assert_nil session.logger - end - - def test_start_without_initial_url_raises_argument_error - session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 - - error = assert_raises ArgumentError do - session.start - end - assert_match(/initial_url/, error.message) - end - - def test_start_with_blank_initial_url_raises_argument_error - session = Session.new client_stub: ScriptedClientStub.new, stream: StringIO.new("abc"), upload_size: 3 - - error = assert_raises ArgumentError do - session.start initial_url: " " - end - assert_match(/initial_url is required/, error.message) - end - - def test_start_with_malformed_retry_policy_raises_argument_error - session = build_session - - error = assert_raises ArgumentError do - start_session session, start_retry_policy: "nonsense" - end - assert_match(/Expected RetryPolicy, Hash, or nil/, error.message) - end - - def test_start_with_reserved_initial_header_raises_before_any_request - stub = ScriptedClientStub.new - session = build_session stub: stub - - error = assert_raises ArgumentError do - start_session session, initial_headers: { "X-Goog-Upload-Header-Content-Type" => "image/png" } - end - assert_match(/must not set protocol header/, error.message) - assert_empty stub.requests - refute session.bound? - end - - def test_failed_start_leaves_session_reusable - session = build_session - - assert_raises ArgumentError do - start_session session, start_retry_policy: "nonsense" - end - - refute session.running? - refute session.bound? - end - - def test_resume_needs_no_initiation_arguments - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"resumed":true}' - ) - ] - session = Session.new( - client_stub: ScriptedClientStub.new(responses), - stream: StringIO.new("01"), - upload_size: 2 - ) - handle = ResumeHandle.new upload_url: "https://upload.example.com/persisted", chunk_size: 4 - - result = session.resume resume_handle: handle - - assert_equal '{"resumed":true}', result - assert_equal "https://upload.example.com/persisted", session.upload_url - end - - # ============================================================================ - # 2. Observable States: Unbound & Bound - # ============================================================================ - - def test_initial_unbound_state - session = build_session - refute session.bound? - assert_nil session.upload_url - assert_nil session.resume_handle - refute session.resumable? - refute session.running? - end - - # ============================================================================ - # 3. Start Lifecycle & Single-Run Contract - # ============================================================================ - - def test_start_successful_upload_transitions_to_bound - responses = [ - # Initiation response - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_1", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - # Chunk 1 (0-3) - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "active" }, - body: "" - ), - # Chunk 2 (4-7) - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "active" }, - body: "" - ), - # Final Chunk (8-9) - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"status":"completed"}' - ) - ] - - stub = ScriptedClientStub.new responses - session = build_session stub: stub, upload_size: 10, chunk_size: 4 - - result = start_session session - - assert_equal '{"status":"completed"}', result - assert session.bound? - assert_equal "https://upload.example.com/session_1", session.upload_url - refute session.running? - refute session.resumable? - assert_nil session.resume_handle - end - - def test_second_start_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_1", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"done":true}' - ) - ] - session = build_session( - stub: ScriptedClientStub.new(responses), - stream: StringIO.new("01"), - upload_size: 2, - chunk_size: 4 - ) - start_session session - - assert session.bound? - - err = assert_raises SessionStateError do - start_session session - end - assert_includes err.message, "Session has already executed a run" - end - - def test_resume_after_start_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_1", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"done":true}' - ) - ] - session = build_session( - stub: ScriptedClientStub.new(responses), - stream: StringIO.new("01"), - upload_size: 2, - chunk_size: 4 - ) - start_session session - - assert_nil session.resume_handle - refute session.resumable? - - handle = ResumeHandle.new upload_url: "https://upload.example.com/session_1", chunk_size: 4 - err = assert_raises SessionStateError do - session.resume resume_handle: handle - end - assert_includes err.message, "Session has already executed a run" - end - - # ============================================================================ - # 4. Resume Forms: Explicit URL or ResumeHandle - # ============================================================================ - - def test_resume_explicit_url_and_chunk_size_binds_and_executes - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"from_url":true}' - ) - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - - refute session.bound? - result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - - assert_equal '{"from_url":true}', result - assert session.bound? - assert_equal "https://upload.example.com/direct", session.upload_url - end - - def test_resume_resume_handle_binds_and_executes - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"from_handle":true}' - ) - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - - handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 - - refute session.bound? - result = session.resume resume_handle: handle - - assert_equal '{"from_handle":true}', result - assert session.bound? - assert_equal "https://upload.example.com/from_handle", session.upload_url - end - - def test_start_after_resume_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - - assert session.bound? - err = assert_raises SessionStateError do - start_session session - end - assert_includes err.message, "Session has already executed a run" - end - - def test_second_resume_raises_session_state_error - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4 - session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - - assert session.bound? - err = assert_raises SessionStateError do - session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - end - assert_includes err.message, "Session has already executed a run" - end - - # ============================================================================ - # 5. Argument Shape & Preconditions - # ============================================================================ - - def test_resume_without_arguments_raises_argument_error - session = build_session - err = assert_raises ArgumentError do - session.resume - end - assert_includes err.message, "Must provide either resume_handle or upload_url and chunk_size" - end - - def test_resume_mixing_arguments_raises_argument_error - session = build_session - handle = ResumeHandle.new upload_url: "https://example.com", chunk_size: 4 - - assert_raises ArgumentError do - session.resume resume_handle: handle, upload_url: "https://example.com" - end - - assert_raises ArgumentError do - session.resume resume_handle: handle, chunk_size: 4 - end - - assert_raises ArgumentError do - session.resume upload_url: "https://example.com" - end - - assert_raises ArgumentError do - session.resume chunk_size: 4 - end - - assert_raises ArgumentError do - session.resume handle - end - end - - def test_resume_with_non_zero_stream_pos_raises_argument_error - stream = StringIO.new "0123456789" - stream.seek 4 - - session = build_session stream: stream - handle = ResumeHandle.new upload_url: "https://upload.example.com/from_handle", chunk_size: 4 - - err = assert_raises ArgumentError do - session.resume resume_handle: handle - end - assert_includes err.message, "Stream must be positioned at byte 0 to resume an upload (got pos 4)" - end - - def test_resume_with_stream_without_pos_is_trusted - stream = StreamWithoutPos.new "01" - responses = [ - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"ok":true}') - ] - stub = ScriptedClientStub.new responses - session = build_session stub: stub, stream: stream, upload_size: 2, chunk_size: 4 - - result = session.resume upload_url: "https://upload.example.com/direct", chunk_size: 4 - assert_equal '{"ok":true}', result - end - - # ============================================================================ - # 6. Cross-Session Resumption - # ============================================================================ - - def test_cross_session_resumption_from_failed_run - stream = StringIO.new "0123456789" - stub1 = ScriptedClientStub.new [ - # Initiation succeeds - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_cross", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ), - # Chunk 1 returns 503 - FakeResponse.new(status: 503, headers: {}, body: "Service Unavailable"), - # Recovery query fails - Faraday::ConnectionFailed.new("network connection failed") - ] - - session1 = build_session stub: stub1, stream: stream, upload_size: 10, chunk_size: 4 - raised = assert_raises RequestFailedError do - start_session session1 - end - - assert session1.bound? - assert session1.resumable? - handle = session1.resume_handle - refute_nil handle - assert_equal handle, raised.resume_handle - assert_equal "https://upload.example.com/session_cross", handle.upload_url - assert_equal 4, handle.chunk_size - - # Prepare for session 2: rewind the stream to byte 0 - stream.rewind - assert_equal 0, stream.pos - - stub2 = ScriptedClientStub.new [ - # Recovery query on resume: server acknowledges 0 bytes received - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-size-received" => "0" - }, - body: "" - ), - # Chunk 1 - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), - # Chunk 2 - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "active" }, body: ""), - # Chunk 3 (final) - FakeResponse.new(status: 200, headers: { "x-goog-upload-status" => "final" }, body: '{"resumed":true}') - ] - - session2 = build_session stub: stub2, stream: stream, upload_size: 10, chunk_size: 4 - refute session2.bound? - - result = session2.resume resume_handle: handle - assert_equal '{"resumed":true}', result - assert session2.bound? - refute session2.resumable? - assert_nil session2.resume_handle - end - - # ============================================================================ - # 7. Concurrency & Running Guard - # ============================================================================ - - def test_running_guard_prevents_concurrent_runs - started_q = Queue.new - unblock_q = Queue.new - - blocking_proc = proc do - started_q.push :started - unblock_q.pop # wait until test signals to proceed - FakeResponse.new( - status: 200, - headers: { - "x-goog-upload-status" => "active", - "x-goog-upload-url" => "https://upload.example.com/session_block", - "x-goog-upload-chunk-granularity" => "4" - }, - body: "" - ) - end - - stub = ScriptedClientStub.new [ - blocking_proc, - FakeResponse.new( - status: 200, - headers: { "x-goog-upload-status" => "final" }, - body: '{"done":true}' - ) - ] - session = build_session( - stub: stub, - stream: StringIO.new("01"), - upload_size: 2, - chunk_size: 4 - ) - - worker = Thread.new do - start_session session - end - - started_q.pop # wait for worker thread to enter driver.run - assert session.running? - - # Concurrent call from another thread raises SessionStateError - handle = ResumeHandle.new upload_url: "https://upload.example.com/session_block", chunk_size: 4 - err = assert_raises SessionStateError do - session.resume resume_handle: handle - end - assert_includes err.message, "A run is already in progress for this session" - - err_start = assert_raises SessionStateError do - start_session session - end - assert_includes err_start.message, "A run is already in progress for this session" - - # Unblock worker thread - unblock_q.push :continue - result = worker.value - - assert_equal '{"done":true}', result - refute session.running? - end - - # ============================================================================ - # 8. Driver#upload_url Direct Verification - # ============================================================================ - - def test_driver_upload_url_across_statuses - dummy_client = ScriptedClientStub.new - config = StartUploadConfig.new( - initial_url: "https://example.com/upload", - stream: StringIO.new("data"), - upload_size: 4, - chunk_size: 4 - ) - driver = Driver.new client_stub: dummy_client, config: config - - assert_nil driver.upload_url - - # Active - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :transmission_sending, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_equal "https://upload.example.com/sess1", driver.resume_handle.upload_url - - # Rejected (resume_handle is nil, but upload_url remains readable) - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :rejected, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_nil driver.resume_handle - - # Cancelled (resume_handle is nil, but upload_url remains readable) - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :cancelled, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_nil driver.resume_handle - - # Success (resume_handle is nil, but upload_url remains readable) - driver.core.instance_variable_set( - :@state, - driver.core.state.with(status: :success, upload_url: "https://upload.example.com/sess1") - ) - assert_equal "https://upload.example.com/sess1", driver.upload_url - assert_nil driver.resume_handle - end -end diff --git a/gapic-common/test/gapic/resumable_upload_test.rb b/gapic-common/test/gapic/resumable_upload_test.rb new file mode 100644 index 0000000..5adafe9 --- /dev/null +++ b/gapic-common/test/gapic/resumable_upload_test.rb @@ -0,0 +1,625 @@ +# frozen_string_literal: true + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "test_helper" +require "gapic/rest" +require "stringio" + +class ResumableUploadTest < Minitest::Test + include Gapic::Rest::ResumableUpload + + FakeResponse = Struct.new :status, :headers, :body, keyword_init: true + + INITIAL_URL = "https://example.com/initiate" + INITIAL_BODY = '{"name":"test.txt"}' + SESSION_URL = "https://upload.example.com/session_1" + + class ScriptedClientStub + attr_reader :requests + + def initialize responses = [] + @responses = responses.dup + @requests = [] + end + + def make_post_request uri:, body:, params:, options:, method_name: nil + @requests << { uri: uri, body: body, params: params, options: options, method_name: method_name } + raise "Unexpected request: no scripted response left" if @responses.empty? + + res = @responses.shift + if res.is_a? Proc + res.call + elsif res.is_a? Exception + raise res + else + res + end + end + end + + # Stream that reports no position at all, which a resumed run has to trust. + class StreamWithoutPos + def initialize string + @io = StringIO.new string + end + + def read length = nil + @io.read length + end + end + + # Stream that records whether anything read from it, to pin down when a run first touches it. + class WatchedStream + attr_reader :reads + + def initialize string + @io = StringIO.new string + @reads = 0 + end + + def pos + @io.pos + end + + def read length = nil + @reads += 1 + @io.read length + end + end + + def setup + @proc_calls = [] + end + + # ============================================================================ + # Helpers + # ============================================================================ + + def build_upload stub: nil, response_type: nil, initial_request: nil, **kwargs + @stub = stub || ScriptedClientStub.new + Gapic::ResumableUpload.new( + client_stub_proc: lambda { + @proc_calls << :client_stub + @stub + }, + initial_request_proc: lambda { + @proc_calls << :initial_request + initial_request || [INITIAL_URL, INITIAL_BODY] + }, + response_type: response_type, + **kwargs + ) + end + + def start_upload upload, stream: nil, upload_size: 10, chunk_size: 4, **overrides + upload.start stream: stream || StringIO.new("0123456789"), + upload_size: upload_size, + chunk_size: chunk_size, + **overrides + end + + def driver_of upload + upload.instance_variable_get :@driver + end + + def config_of upload + driver_of(upload).instance_variable_get :@config + end + + def initiation_response url: SESSION_URL, granularity: 4 + FakeResponse.new( + status: 200, + headers: { + "x-goog-upload-status" => "active", + "x-goog-upload-url" => url, + "x-goog-upload-chunk-granularity" => granularity.to_s + }, + body: "" + ) + end + + def query_response received: 0 + FakeResponse.new( + status: 200, + headers: { "x-goog-upload-status" => "active", "x-goog-upload-size-received" => received.to_s }, + body: "" + ) + end + + def chunk_response + FakeResponse.new status: 200, headers: { "x-goog-upload-status" => "active" }, body: "" + end + + def final_response body = '{"text":"done"}' + FakeResponse.new status: 200, headers: { "x-goog-upload-status" => "final" }, body: body + end + + # Initiation plus a single finalizing chunk, enough for a two-byte stream. + def short_upload_responses body: '{"text":"done"}' + [initiation_response, final_response(body)] + end + + # Recovery query plus a single finalizing chunk, enough to resume a two-byte stream. + def short_resume_responses body: '{"text":"done"}' + [query_response, final_response(body)] + end + + def resume_handle_for url: SESSION_URL, chunk_size: 4 + ResumeHandle.new upload_url: url, chunk_size: chunk_size + end + + # ============================================================================ + # 1. Deferred procs + # ============================================================================ + + def test_start_calls_client_stub_proc_then_initial_request_proc + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal [:client_stub, :initial_request], @proc_calls + end + + def test_resume_never_calls_initial_request_proc + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses) + + upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: resume_handle_for + + assert_equal [:client_stub], @proc_calls + end + + def test_raising_client_stub_proc_surfaces_before_the_stream_is_read + stream = WatchedStream.new "0123456789" + upload = Gapic::ResumableUpload.new( + client_stub_proc: -> { raise ArgumentError, "REST is unavailable" }, + initial_request_proc: -> { [INITIAL_URL, INITIAL_BODY] }, + response_type: nil + ) + + error = assert_raises ArgumentError do + upload.start stream: stream, upload_size: 10 + end + + assert_equal "REST is unavailable", error.message + assert_equal 0, stream.reads + refute upload.running? + assert_nil driver_of(upload) + end + + def test_resume_rejects_a_non_zero_stream_before_calling_the_client_stub_proc + stream = StringIO.new "0123456789" + stream.seek 4 + upload = build_upload + + error = assert_raises ArgumentError do + upload.resume stream: stream, resume_handle: resume_handle_for + end + + assert_includes error.message, "Stream must be positioned at byte 0 to resume an upload (got pos 4)" + assert_empty @proc_calls + end + + # ============================================================================ + # 2. Configuration construction + # ============================================================================ + + def test_start_builds_a_start_config_from_constructor_and_run_arguments + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), + initial_headers: { "X-Goog-Test" => "yes", :symbol_key => 7 }, + start_retry_policy: { initial_delay: 0.01 } + + start_upload upload, stream: StringIO.new("01"), upload_size: 2, chunk_size: 4, + content_type: "text/plain", upload_timeout: 42 + + config = config_of upload + assert_instance_of StartUploadConfig, config + assert_equal INITIAL_URL, config.initial_url + assert_equal INITIAL_BODY, config.initial_body + assert_equal({ "X-Goog-Test" => "yes", "symbol_key" => "7" }, config.initial_headers) + assert_equal 4, config.chunk_size + assert_equal({ initial_delay: 0.01 }, config.start_retry_policy) + assert_equal 2, config.upload_size + assert_equal "text/plain", config.content_type + assert_equal 42, config.timeout + end + + def test_resume_builds_a_resume_config_from_the_handle + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses) + handle = resume_handle_for url: "https://upload.example.com/persisted", chunk_size: 8 + + upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: handle + + config = config_of upload + assert_instance_of ResumeUploadConfig, config + assert_equal "https://upload.example.com/persisted", config.upload_url + assert_equal 8, config.chunk_size + end + + def test_upload_timeout_is_omitted_when_unset + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_nil config_of(upload).timeout + end + + def test_plane_retry_policies_come_from_the_constructor + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), + control_plane_retry_policy: { initial_delay: 0.02 }, + data_plane_retry_policy: { initial_delay: 0.03 } + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + config = config_of upload + assert_equal({ initial_delay: 0.02 }, config.control_plane_retry_policy) + assert_equal({ initial_delay: 0.03 }, config.data_plane_retry_policy) + end + + def test_reserved_initial_header_raises_before_any_request + stub = ScriptedClientStub.new + upload = build_upload stub: stub, initial_headers: { "X-Goog-Upload-Header-Content-Type" => "image/png" } + + error = assert_raises ArgumentError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_match(/must not set protocol header/, error.message) + assert_empty stub.requests + assert_nil driver_of(upload) + end + + # ============================================================================ + # 3. Lifecycle + # ============================================================================ + + def test_readers_before_the_first_run + upload = build_upload + + assert_nil upload.resume_handle + refute upload.resumable? + refute upload.running? + end + + def test_successful_run_leaves_nothing_to_resume + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal '{"text":"done"}', result + refute upload.running? + refute upload.resumable? + assert_nil upload.resume_handle + end + + def test_handle_is_reusable_across_runs + responses = short_upload_responses(body: '{"text":"first"}') + + short_upload_responses(body: '{"text":"second"}') + upload = build_upload stub: ScriptedClientStub.new(responses) + + assert_equal '{"text":"first"}', start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + first_driver = driver_of upload + + assert_equal '{"text":"second"}', start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + refute_same first_driver, driver_of(upload) + end + + def test_concurrent_run_raises_session_state_error + started_q = Queue.new + unblock_q = Queue.new + blocking_proc = proc do + started_q.push :started + unblock_q.pop + initiation_response + end + upload = build_upload stub: ScriptedClientStub.new([blocking_proc, final_response]) + + worker = Thread.new { start_upload upload, stream: StringIO.new("01"), upload_size: 2 } + started_q.pop + assert upload.running? + + error = assert_raises SessionStateError do + upload.resume stream: StringIO.new("01"), resume_handle: resume_handle_for + end + assert_includes error.message, "A run is already in progress for this upload" + + assert_raises SessionStateError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + unblock_q.push :continue + assert_equal '{"text":"done"}', worker.value + refute upload.running? + end + + def test_run_slot_is_released_after_a_failed_run + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + upload = build_upload stub: stub + + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + refute upload.running? + assert upload.resumable? + end + + def test_run_slot_is_released_after_a_progress_callback_raises + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + on_progress = ->(_progress) { raise "callback exploded" } + + error = assert_raises RuntimeError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2, on_progress: on_progress + end + + assert_equal "callback exploded", error.message + refute upload.running? + end + + def test_run_slot_is_released_after_a_configuration_error + upload = build_upload start_retry_policy: "nonsense" + + assert_raises ArgumentError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + refute upload.running? + assert_nil driver_of(upload) + end + + def test_configuration_error_leaves_an_earlier_runs_resume_handle_intact + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + upload = build_upload stub: stub + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + handle = upload.resume_handle + refute_nil handle + + assert_raises ArgumentError do + upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: resume_handle_for(chunk_size: -1) + end + + assert_equal handle, upload.resume_handle + end + + # ============================================================================ + # 4. Resume forms + # ============================================================================ + + def test_resume_with_an_explicit_handle + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses(body: '{"text":"resumed"}')) + handle = resume_handle_for url: "https://upload.example.com/persisted", chunk_size: 4 + + result = upload.resume stream: StringIO.new("01"), upload_size: 2, resume_handle: handle + + assert_equal '{"text":"resumed"}', result + assert_equal "https://upload.example.com/persisted", @stub.requests.first[:uri] + end + + def test_bare_resume_reuses_the_retained_drivers_handle + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom"), + query_response, final_response('{"text":"resumed"}')] + upload = build_upload stub: stub + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + assert upload.resumable? + + result = upload.resume stream: StringIO.new("01"), upload_size: 2 + + assert_equal '{"text":"resumed"}', result + assert_equal SESSION_URL, stub.requests.last[:uri] + end + + def test_bare_resume_without_a_previous_run_raises_argument_error + upload = build_upload + + error = assert_raises ArgumentError do + upload.resume stream: StringIO.new("01") + end + + assert_includes error.message, "No upload to resume" + end + + def test_bare_resume_after_a_successful_run_raises_argument_error + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_raises ArgumentError do + upload.resume stream: StringIO.new("01") + end + end + + def test_bare_resume_after_an_unresumable_failure_raises_argument_error + upload = build_upload stub: ScriptedClientStub.new([Faraday::ConnectionFailed.new("boom")]) + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + refute upload.resumable? + + assert_raises ArgumentError do + upload.resume stream: StringIO.new("01") + end + end + + def test_resume_trusts_a_stream_that_reports_no_position + upload = build_upload stub: ScriptedClientStub.new(short_resume_responses) + + result = upload.resume stream: StreamWithoutPos.new("01"), + upload_size: 2, + resume_handle: resume_handle_for + + assert_equal '{"text":"done"}', result + end + + # ============================================================================ + # 5. Response decoding + # ============================================================================ + + def test_decodes_the_final_body_into_the_response_type + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: '{"text":"decoded"}')), + response_type: Gapic::Examples::Post + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_instance_of Gapic::Examples::Post, result + assert_equal "decoded", result.text + end + + def test_decodes_an_empty_final_body_into_an_empty_message + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: "")), + response_type: Gapic::Examples::Post + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal Gapic::Examples::Post.new, result + end + + def test_decodes_an_absent_final_body_into_an_empty_message + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: nil)), + response_type: Gapic::Examples::Post + + result = start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal Gapic::Examples::Post.new, result + end + + def test_malformed_final_body_raises_a_parse_error + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: '{"text":')), + response_type: Gapic::Examples::Post + + assert_raises Google::Protobuf::ParseError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + end + + def test_ignores_unknown_fields_in_the_final_body + body = '{"text":"decoded","unknown_field":true}' + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: body)), + response_type: Gapic::Examples::Post + + assert_equal "decoded", start_upload(upload, stream: StringIO.new("01"), upload_size: 2).text + end + + def test_no_response_type_returns_the_raw_body + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: "not json at all")) + + assert_equal "not json at all", start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + end + + def test_no_response_type_returns_nil_for_a_bodiless_response + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses(body: nil)) + + assert_nil start_upload(upload, stream: StringIO.new("01"), upload_size: 2) + end + + # ============================================================================ + # 6. Error handling + # ============================================================================ + + class WrappedError < StandardError; end + + def failing_upload **kwargs + stub = ScriptedClientStub.new [initiation_response, Faraday::ConnectionFailed.new("boom"), + Faraday::ConnectionFailed.new("boom")] + build_upload stub: stub, **kwargs + end + + def test_error_handler_replaces_the_raised_error + upload = failing_upload error_handler: ->(e) { WrappedError.new "wrapped: #{e.message}" } + + error = assert_raises WrappedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_includes error.message, "wrapped:" + end + + def test_wrapped_error_remains_rescuable_as_has_resume_handle + upload = failing_upload error_handler: ->(_e) { WrappedError.new "wrapped" } + + error = assert_raises HasResumeHandle do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_instance_of WrappedError, error + assert_equal upload.resume_handle, error.resume_handle + assert_equal SESSION_URL, error.resume_handle.upload_url + end + + def test_error_handler_returning_nil_reraises_the_original + upload = failing_upload error_handler: ->(_e) { nil } + + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + end + + def test_error_handler_returning_the_original_reraises_it + upload = failing_upload error_handler: ->(e) { e } + + assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + end + + def test_an_error_without_a_resume_handle_is_not_decorated + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), + error_handler: ->(_e) { WrappedError.new "wrapped" } + on_progress = ->(_progress) { raise "callback exploded" } + + error = assert_raises WrappedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2, on_progress: on_progress + end + + refute error.is_a?(HasResumeHandle) + end + + def test_no_error_handler_propagates_the_protocol_error + upload = failing_upload + + error = assert_raises RequestFailedError do + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + end + + assert_equal upload.resume_handle, error.resume_handle + end + + # ============================================================================ + # 7. Logging + # ============================================================================ + + def test_method_name_reaches_the_client_stub + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses), method_name: "create_media_upload" + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal ["create_media_upload.start", "create_media_upload.upload"], + @stub.requests.map { |request| request[:method_name] } + end + + def test_method_name_defaults_to_the_protocol_name + upload = build_upload stub: ScriptedClientStub.new(short_upload_responses) + + start_upload upload, stream: StringIO.new("01"), upload_size: 2 + + assert_equal ["ResumableUpload.start", "ResumableUpload.upload"], + @stub.requests.map { |request| request[:method_name] } + end +end