diff --git a/CHANGELOG.md b/CHANGELOG.md index 037b489..8a2c9cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## Unreleased + +### Added + +- Automatic retry with exponential backoff (10s, 20s, 40s, 60s; up to 5 attempts) when Cloud Logging returns a quota / rate-limit error (`ResourceExhausted`, `RATE_LIMIT_EXCEEDED`, e.g. the 60 read requests/min limit). A short notice is printed to stderr before each retry; retries stop if the context is cancelled. Interrupted pagination resumes from the last page token. + +### Changed + +- `--raw` now emits one compact JSON object per line (JSONL), consistent with `--json`, instead of pretty-printed multi-line objects. +- Query page size now matches `--limit` (capped at the API maximum of 1000), so small queries cost a single read request. + +### Fixed + +- `--where` values containing double quotes (e.g. `--where 'name="gcpCredentials"'`, as documented) or commas no longer fail with `bare " in non-quoted-field`. Each `--where` is now taken verbatim; multiple `--where` flags are still ANDed. +- Raw output mislabelled JSON payloads as `protoPayload`; they are now reported as `jsonPayload`, matching the field paths used in GCP filters (e.g. `jsonPayload.reason` for k8s events). Genuine proto payloads (e.g. audit logs) are still reported as `protoPayload`. + ## v1.4.1 - 2026-06-02 ### Added diff --git a/README.md b/README.md index a76a45a..f40f253 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A CLI tool for querying GCP Cloud Logging for [Taskcluster](https://taskcluster. - **Schema-aware queries** — Fetches log type definitions from Taskcluster references. Knows that `worker-stopped` has fields `workerId`, `workerPoolId`, `reason`, etc. - **Field shorthand** — Write `--where 'workerPoolId="proj-misc"'` instead of `--filter 'jsonPayload.Fields.workerPoolId="proj-misc"'` - **Auto service narrowing** — When a log type is unique to one service (e.g., `worker-stopped` → `worker-manager`), the tool automatically narrows the query -- **Multiple output formats** — Aligned columns (default), JSONL (`--json`), or raw GCP entries (`--raw`) +- **Multiple output formats** — Aligned columns (default), JSONL (`--json`), or full raw GCP entries as JSONL (`--raw`) - **Result caching** — Queries with absolute time windows are cached for 14 days, keyed by filter + time range hash - **Environment management** — Configure multiple Taskcluster deployments, auto-detect from `TASKCLUSTER_ROOT_URL` @@ -224,8 +224,12 @@ tc-logview query -e fx-ci --type task-claimed \ # JSONL output, pipe to jq tc-logview query -e fx-ci --type hook-fire --json | jq '.taskId' -# Raw GCP entries for debugging -tc-logview query -e fx-ci --type worker-stopped --raw --limit 5 +# Raw GCP entries for debugging (one compact JSON object per line) +tc-logview query -e fx-ci --type worker-stopped --raw --limit 5 | jq '.jsonPayload.Fields' + +# Multiple --where flags are ANDed; values may be quoted and contain commas +tc-logview query -e fx-ci --type monitor.apiMethod \ + --where 'name="gcpCredentials"' --where 'statusCode="500"' # Page through cached results tc-logview query -e fx-ci --type worker-stopped \ @@ -299,7 +303,7 @@ tc-logview query --type worker-stopped | `-e, --env` | Environment name | from `TASKCLUSTER_ROOT_URL` | | `--type` | Log type (e.g. `worker-stopped`, `monitor.error`) | | | `--service` | Filter by service (useful for shared types like `monitor.apiMethod`) | auto-detected if type is unique | -| `--where` | Field shorthand filter (e.g. `workerPoolId="proj-misc"`) | | +| `--where` | Field shorthand filter (e.g. `workerPoolId="proj-misc"`); repeatable, each value taken verbatim | | | `--filter` | Raw GCP filter expression | | | `--since` | Relative time window (`30m`, `2h`, `1d`) | `1h` | | `--from` | Absolute start time (RFC3339) | | @@ -307,7 +311,7 @@ tc-logview query --type worker-stopped | `--limit` | Max entries | `100` | | `--offset` | Skip N entries (cached results only) | `0` | | `--json` | JSONL output | | -| `--raw` | Full raw GCP entries | | +| `--raw` | Full raw GCP entries, one compact JSON object per line (JSONL) | | | `--no-cache` | Skip result cache | | ## How it works @@ -320,6 +324,14 @@ The tool builds GCP filters from three layers: 2. **Type** — `jsonPayload.Type` + `jsonPayload.serviceContext.service` (from `--type`, auto-narrowed) 3. **User** — `--where` (expanded to `jsonPayload.Fields.*`) and `--filter` (raw passthrough) +### Quotas and pagination + +Cloud Logging limits read requests (default 60 per minute per project). tc-logview sizes each +page to the remaining `--limit` (max 1000 per request), so a query with `--limit 100` costs one +read request. If the quota is exceeded (`ResourceExhausted` / `RATE_LIMIT_EXCEEDED`), the request +is retried up to 4 more times with exponential backoff (10s, 20s, 40s, 60s), with a short notice +printed to stderr before each retry. Cached results (see below) cost no read requests. + ### Caching - **References** (`~/.cache/tc-logview/references/`) — Cached indefinitely, refresh with `tc-logview sync` diff --git a/cmd/query.go b/cmd/query.go index 415ab0e..42a2f6a 100644 --- a/cmd/query.go +++ b/cmd/query.go @@ -49,7 +49,9 @@ var queryCmd = &cobra.Command{ func init() { queryCmd.Flags().StringVar(&queryType, "type", "", "log type (e.g. worker-stopped)") queryCmd.Flags().StringVar(&queryService, "service", "", "filter by service (for shared types)") - queryCmd.Flags().StringSliceVar(&queryWhere, "where", nil, "field shorthand filter (e.g. workerPoolId=\"proj-misc\")") + // StringArray (not StringSlice): each --where is taken verbatim. StringSlice + // CSV-parses values, which rejects quotes (name="x") and splits on commas. + queryCmd.Flags().StringArrayVar(&queryWhere, "where", nil, "field shorthand filter, repeatable (e.g. workerPoolId=\"proj-misc\")") queryCmd.Flags().StringVar(&queryFilter, "filter", "", "raw GCP filter expression") queryCmd.Flags().StringVar(&querySince, "since", "", "relative time window (e.g. 2h, 30m, 1d)") queryCmd.Flags().StringVar(&queryFrom, "from", "", "absolute start time (RFC3339)") @@ -57,7 +59,7 @@ func init() { queryCmd.Flags().IntVar(&queryLimit, "limit", 100, "max entries to return") queryCmd.Flags().IntVar(&queryOffset, "offset", 0, "skip N entries (cached results only)") queryCmd.Flags().BoolVar(&queryJSON, "json", false, "output as JSONL") - queryCmd.Flags().BoolVar(&queryRaw, "raw", false, "output full raw GCP entries") + queryCmd.Flags().BoolVar(&queryRaw, "raw", false, "output full raw GCP entries as JSONL (one compact object per line)") queryCmd.Flags().BoolVar(&queryNoCache, "no-cache", false, "skip cache") rootCmd.AddCommand(queryCmd) } diff --git a/cmd/query_test.go b/cmd/query_test.go new file mode 100644 index 0000000..2fe71e7 --- /dev/null +++ b/cmd/query_test.go @@ -0,0 +1,42 @@ +package cmd + +import ( + "slices" + "testing" + + "github.com/spf13/pflag" +) + +// TestQueryWhereFlagVerbatim guards against --where being CSV-parsed (as a +// StringSlice would): quoted values, unquoted values and values containing +// commas must all be passed through verbatim, one entry per flag. +func TestQueryWhereFlagVerbatim(t *testing.T) { + flag := queryCmd.Flags().Lookup("where") + prev := slices.Clone(queryWhere) + t.Cleanup(func() { + _ = flag.Value.(pflag.SliceValue).Replace(prev) + flag.Changed = false + }) + + args := []string{ + `--where`, `name="gcpCredentials"`, + `--where`, `runId=1`, + `--where`, `reason="a,b"`, + `--where`, `pod=x,y`, + } + if err := queryCmd.Flags().Parse(args); err != nil { + t.Fatalf("parsing --where flags: %v", err) + } + want := []string{`name="gcpCredentials"`, `runId=1`, `reason="a,b"`, `pod=x,y`} + if !slices.Equal(queryWhere, want) { + t.Errorf("queryWhere = %q, want %q", queryWhere, want) + } +} + +func TestParseWhereFieldNames(t *testing.T) { + got := parseWhereFieldNames([]string{`taskId="abc"`, `runId>=1`, `name="a,b"`}) + want := []string{"taskId", "runId", "name"} + if !slices.Equal(got, want) { + t.Errorf("parseWhereFieldNames = %q, want %q", got, want) + } +} diff --git a/go.mod b/go.mod index c9ea5af..c5f7fc2 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,12 @@ go 1.26.0 require ( cloud.google.com/go/logging v1.13.2 github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.9 github.com/tidwall/gjson v1.18.0 golang.org/x/oauth2 v0.35.0 google.golang.org/api v0.269.0 google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) @@ -28,7 +30,6 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.12 // indirect github.com/googleapis/gax-go/v2 v2.17.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/pflag v1.0.9 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect @@ -45,5 +46,4 @@ require ( golang.org/x/time v0.14.0 // indirect google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260217215200-42d3e9bedb6d // indirect - google.golang.org/grpc v1.79.3 // indirect ) diff --git a/internal/filter/builder_test.go b/internal/filter/builder_test.go index 0392f3a..11896a2 100644 --- a/internal/filter/builder_test.go +++ b/internal/filter/builder_test.go @@ -296,3 +296,23 @@ func TestBuild_PresetFilterUnknownField(t *testing.T) { t.Fatal("expected error for unknown field in preset context") } } + +func TestBuild_WhereQuotedUnquotedAndCommaValues(t *testing.T) { + got, err := Build(Params{ + Cluster: "c", + Where: []string{`name="gcpCredentials"`, `runId=1`, `message="a,b"`}, + FieldNames: []string{"name", "runId", "message"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for _, want := range []string{ + `jsonPayload.Fields.name="gcpCredentials"`, + `jsonPayload.Fields.runId=1`, + `jsonPayload.Fields.message="a,b"`, + } { + if !strings.Contains(got, want) { + t.Errorf("filter missing %q, got: %s", want, got) + } + } +} diff --git a/internal/format/raw.go b/internal/format/raw.go index 3d5d90a..72f9a4c 100644 --- a/internal/format/raw.go +++ b/internal/format/raw.go @@ -5,23 +5,17 @@ import ( "io" ) -// Raw formats log entries by pretty-printing each entry's Raw map as -// indented JSON, separated by newlines. +// Raw formats log entries as JSON Lines: each entry's full Raw map is written +// as one compact JSON object per line, suitable for piping to jq or other +// line-based tools. type Raw struct{} -// Format writes each entry's Raw map as pretty-printed JSON (2-space indent) -// to w, with a blank line between entries. +// Format writes each entry's Raw map as a single-line JSON object to w. func (r *Raw) Format(w io.Writer, entries []LogEntry, fieldOrder []string, total int) error { enc := json.NewEncoder(w) enc.SetEscapeHTML(false) - enc.SetIndent("", " ") - for i, e := range entries { - if i > 0 { - if _, err := io.WriteString(w, "\n"); err != nil { - return err - } - } + for _, e := range entries { if err := enc.Encode(e.Raw); err != nil { return err } diff --git a/internal/format/raw_test.go b/internal/format/raw_test.go new file mode 100644 index 0000000..14ed354 --- /dev/null +++ b/internal/format/raw_test.go @@ -0,0 +1,50 @@ +package format + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestRaw_OneCompactObjectPerLine(t *testing.T) { + entries := []LogEntry{ + {Raw: map[string]interface{}{ + "timestamp": "2026-03-10T12:00:00Z", + "jsonPayload": map[string]interface{}{"reason": "FailedPreStopHook", "msg": "a\nb "}, + }}, + {Raw: map[string]interface{}{"timestamp": "2026-03-10T12:00:01Z", "textPayload": "hello"}}, + } + + var buf bytes.Buffer + if err := (&Raw{}).Format(&buf, entries, nil, 2); err != nil { + t.Fatalf("Format: %v", err) + } + + lines := strings.Split(strings.TrimSuffix(buf.String(), "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d:\n%s", len(lines), buf.String()) + } + for i, line := range lines { + var m map[string]interface{} + if err := json.Unmarshal([]byte(line), &m); err != nil { + t.Errorf("line %d is not a JSON object: %v\n%s", i, err, line) + } + if strings.HasPrefix(line, " ") || strings.Contains(line, "\n") { + t.Errorf("line %d is not compact: %q", i, line) + } + } + if !strings.Contains(lines[0], "") { + t.Errorf("HTML characters should not be escaped, got: %s", lines[0]) + } +} + +func TestRaw_Empty(t *testing.T) { + var buf bytes.Buffer + if err := (&Raw{}).Format(&buf, nil, nil, 0); err != nil { + t.Fatalf("Format: %v", err) + } + if buf.Len() != 0 { + t.Errorf("expected no output, got %q", buf.String()) + } +} diff --git a/internal/gcp/client.go b/internal/gcp/client.go index 8a6ccfa..c1eeeae 100644 --- a/internal/gcp/client.go +++ b/internal/gcp/client.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "os" "strings" "time" @@ -13,8 +14,11 @@ import ( "golang.org/x/oauth2" "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" "github.com/taskcluster/tc-logview/internal/format" ) @@ -78,28 +82,118 @@ func AuthModeLabel(auth AuthConfig) string { } } +// maxPageSize is the largest page size Cloud Logging's entries.list accepts. +const maxPageSize = 1000 + +// Retry settings for quota (ResourceExhausted) errors. Cloud Logging's read +// quota is per minute (default 60 read requests/min), so the backoff starts at +// several seconds and grows to roughly a minute. Variables so tests can shrink +// them. +var ( + retryAttempts = 5 + retryInitialDelay = 10 * time.Second + retryMaxDelay = 60 * time.Second + // retryNotify receives a short human-readable notice before each retry. + retryNotify = func(msg string) { fmt.Fprintln(os.Stderr, msg) } +) + +// pageFetcher fetches one page of up to pageSize entries starting at +// pageToken, returning the entries and the next page token ("" when done). +type pageFetcher func(ctx context.Context, pageToken string, pageSize int) ([]*logging.Entry, string, error) + // Query executes a GCP Cloud Logging filter query and returns up to limit // entries, newest first (reversed before display). When resourceName is // non-empty, the query is scoped to that resource (e.g. a log view) instead of // the client's project; an empty resourceName preserves project-scope behavior. +// +// Pages are sized to the remaining limit (capped at the API maximum) so small +// queries cost a single read request. Quota errors are retried with backoff. func (c *Client) Query(ctx context.Context, filter, resourceName string, limit int) (*QueryResult, error) { - it := c.adminClient.Entries(ctx, queryOptions(filter, resourceName)...) + opts := queryOptions(filter, resourceName) + fetch := func(ctx context.Context, pageToken string, pageSize int) ([]*logging.Entry, string, error) { + // A fresh iterator per page: iterator errors are sticky, so a retry + // after a quota error needs a new one, resumed from the page token. + it := c.adminClient.Entries(ctx, opts...) + var entries []*logging.Entry + next, err := iterator.NewPager(it, pageSize, pageToken).NextPage(&entries) + return entries, next, err + } + + entries, err := collectEntries(ctx, fetch, limit) + if err != nil { + return nil, fmt.Errorf("error iterating log entries: %w", err) + } result := &QueryResult{} - for i := 0; i < limit; i++ { - entry, err := it.Next() - if err == iterator.Done { - break - } + for _, entry := range entries { + result.Entries = append(result.Entries, entryToMap(entry)) + } + result.Total = len(result.Entries) + return result, nil +} + +// collectEntries pages through fetch until limit entries are collected or the +// results are exhausted, retrying each page on quota errors. +func collectEntries(ctx context.Context, fetch pageFetcher, limit int) ([]*logging.Entry, error) { + var all []*logging.Entry + token := "" + for len(all) < limit { + pageSize := min(limit-len(all), maxPageSize) + var page []*logging.Entry + var next string + err := withRetry(ctx, func() error { + var err error + page, next, err = fetch(ctx, token, pageSize) + return err + }) if err != nil { - return nil, fmt.Errorf("error iterating log entries: %w", err) + return nil, err + } + all = append(all, page...) + if next == "" { + break } + token = next + } + if len(all) > limit { + all = all[:limit] + } + return all, nil +} - result.Entries = append(result.Entries, entryToMap(entry)) +// withRetry calls fn, retrying with exponential backoff while it returns a +// quota/rate-limit error. It stops early if ctx is cancelled. +func withRetry(ctx context.Context, fn func() error) error { + delay := retryInitialDelay + for attempt := 1; ; attempt++ { + err := fn() + if err == nil || !isQuotaError(err) || attempt >= retryAttempts { + return err + } + retryNotify(fmt.Sprintf("GCP Logging quota exceeded; retrying in %s (attempt %d/%d)...", + delay, attempt+1, retryAttempts)) + select { + case <-ctx.Done(): + return fmt.Errorf("%w (gave up waiting to retry: %v)", err, ctx.Err()) + case <-time.After(delay): + } + delay = min(delay*2, retryMaxDelay) } +} - result.Total = len(result.Entries) - return result, nil +// isQuotaError reports whether err is a Cloud Logging quota / rate-limit +// error (gRPC ResourceExhausted, or a RATE_LIMIT_EXCEEDED / quota message). +func isQuotaError(err error) bool { + if err == nil { + return false + } + if s, ok := status.FromError(err); ok && s.Code() == codes.ResourceExhausted { + return true + } + msg := err.Error() + return strings.Contains(msg, "ResourceExhausted") || + strings.Contains(msg, "RATE_LIMIT_EXCEEDED") || + strings.Contains(msg, "Quota exceeded") } // queryOptions builds the logadmin EntriesOptions for a query. When @@ -150,6 +244,11 @@ func entryToMap(entry *logging.Entry) map[string]interface{} { switch p := entry.Payload.(type) { case map[string]interface{}: m["jsonPayload"] = p + case *structpb.Struct: + // logadmin surfaces LogEntry.jsonPayload as a *structpb.Struct; it + // must be labelled jsonPayload (not protoPayload) so raw output + // matches the field paths used in GCP filters. + m["jsonPayload"] = p.AsMap() case string: m["textPayload"] = p case proto.Message: diff --git a/internal/gcp/retry_test.go b/internal/gcp/retry_test.go new file mode 100644 index 0000000..a40758b --- /dev/null +++ b/internal/gcp/retry_test.go @@ -0,0 +1,229 @@ +package gcp + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "cloud.google.com/go/logging" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" +) + +// fastRetry shrinks backoff delays and captures retry notices for a test. +func fastRetry(t *testing.T) *[]string { + t.Helper() + prevA, prevI, prevM, prevN := retryAttempts, retryInitialDelay, retryMaxDelay, retryNotify + t.Cleanup(func() { + retryAttempts, retryInitialDelay, retryMaxDelay, retryNotify = prevA, prevI, prevM, prevN + }) + var notices []string + retryAttempts = 4 + retryInitialDelay = time.Millisecond + retryMaxDelay = 2 * time.Millisecond + retryNotify = func(msg string) { notices = append(notices, msg) } + return ¬ices +} + +var quotaErr = status.Error(codes.ResourceExhausted, + "Quota exceeded for quota metric 'Read requests' and limit 'Read requests per minute'") + +func makeEntries(n int) []*logging.Entry { + es := make([]*logging.Entry, n) + for i := range es { + es[i] = &logging.Entry{InsertID: fmt.Sprintf("e%d", i)} + } + return es +} + +func TestIsQuotaError(t *testing.T) { + tests := []struct { + err error + want bool + }{ + {nil, false}, + {quotaErr, true}, + {fmt.Errorf("wrapped: %w", quotaErr), true}, + {errors.New("rpc error: code = ResourceExhausted desc = Quota exceeded"), true}, + {errors.New("googleapi: Error 429: RATE_LIMIT_EXCEEDED"), true}, + {status.Error(codes.PermissionDenied, "denied"), false}, + {errors.New("network unreachable"), false}, + } + for _, tt := range tests { + if got := isQuotaError(tt.err); got != tt.want { + t.Errorf("isQuotaError(%v) = %v, want %v", tt.err, got, tt.want) + } + } +} + +func TestWithRetry_SucceedsAfterQuotaErrors(t *testing.T) { + notices := fastRetry(t) + calls := 0 + err := withRetry(context.Background(), func() error { + calls++ + if calls < 3 { + return quotaErr + } + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if calls != 3 { + t.Errorf("calls = %d, want 3", calls) + } + if len(*notices) != 2 || !strings.Contains((*notices)[0], "quota exceeded") { + t.Errorf("expected 2 retry notices, got %q", *notices) + } +} + +func TestWithRetry_GivesUpAfterMaxAttempts(t *testing.T) { + fastRetry(t) + calls := 0 + err := withRetry(context.Background(), func() error { + calls++ + return quotaErr + }) + if !isQuotaError(err) { + t.Fatalf("expected quota error after giving up, got %v", err) + } + if calls != retryAttempts { + t.Errorf("calls = %d, want %d", calls, retryAttempts) + } +} + +func TestWithRetry_NonQuotaErrorNotRetried(t *testing.T) { + notices := fastRetry(t) + calls := 0 + want := status.Error(codes.PermissionDenied, "denied") + err := withRetry(context.Background(), func() error { + calls++ + return want + }) + if err != want || calls != 1 || len(*notices) != 0 { + t.Errorf("err=%v calls=%d notices=%q; want no retry", err, calls, *notices) + } +} + +func TestWithRetry_RespectsContext(t *testing.T) { + fastRetry(t) + retryInitialDelay = time.Hour + retryMaxDelay = time.Hour + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + start := time.Now() + err := withRetry(ctx, func() error { return quotaErr }) + if err == nil || !errors.Is(err, quotaErr) { + t.Fatalf("expected wrapped quota error, got %v", err) + } + if time.Since(start) > 5*time.Second { + t.Errorf("withRetry did not stop on context cancellation") + } +} + +func TestCollectEntries_PageSizeMatchesLimit(t *testing.T) { + fastRetry(t) + var sizes []int + fetch := func(_ context.Context, token string, size int) ([]*logging.Entry, string, error) { + sizes = append(sizes, size) + return makeEntries(size), "more", nil + } + got, err := collectEntries(context.Background(), fetch, 5) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 5 { + t.Errorf("got %d entries, want 5", len(got)) + } + if len(sizes) != 1 || sizes[0] != 5 { + t.Errorf("page sizes = %v, want a single page of 5", sizes) + } +} + +func TestCollectEntries_LargeLimitPagesAtMax(t *testing.T) { + fastRetry(t) + var sizes []int + fetch := func(_ context.Context, token string, size int) ([]*logging.Entry, string, error) { + sizes = append(sizes, size) + return makeEntries(size), "more", nil + } + got, err := collectEntries(context.Background(), fetch, 2500) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 2500 { + t.Errorf("got %d entries, want 2500", len(got)) + } + want := []int{1000, 1000, 500} + if fmt.Sprint(sizes) != fmt.Sprint(want) { + t.Errorf("page sizes = %v, want %v", sizes, want) + } +} + +func TestCollectEntries_StopsWhenExhausted(t *testing.T) { + fastRetry(t) + calls := 0 + fetch := func(_ context.Context, token string, size int) ([]*logging.Entry, string, error) { + calls++ + return makeEntries(3), "", nil + } + got, err := collectEntries(context.Background(), fetch, 100) + if err != nil || len(got) != 3 || calls != 1 { + t.Errorf("got %d entries, err=%v, calls=%d; want 3 entries in 1 call", len(got), err, calls) + } +} + +func TestCollectEntries_RetriesPageFromSameToken(t *testing.T) { + notices := fastRetry(t) + var tokens []string + fetch := func(_ context.Context, token string, size int) ([]*logging.Entry, string, error) { + tokens = append(tokens, token) + switch len(tokens) { + case 1: + return makeEntries(2), "t1", nil + case 2: + return nil, "", quotaErr + default: + return makeEntries(2), "", nil + } + } + got, err := collectEntries(context.Background(), fetch, 10) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 4 { + t.Errorf("got %d entries, want 4 (no duplicates or gaps)", len(got)) + } + if fmt.Sprint(tokens) != fmt.Sprint([]string{"", "t1", "t1"}) { + t.Errorf("tokens = %q, want retry from same token", tokens) + } + if len(*notices) != 1 { + t.Errorf("expected 1 retry notice, got %q", *notices) + } +} + +func TestEntryToMap_StructPayloadIsJsonPayload(t *testing.T) { + s, err := structpb.NewStruct(map[string]interface{}{ + "reason": "FailedPreStopHook", + "involvedObject": map[string]interface{}{"name": "pod-1"}, + }) + if err != nil { + t.Fatal(err) + } + m := entryToMap(&logging.Entry{Timestamp: time.Now(), Payload: s}) + if _, ok := m["protoPayload"]; ok { + t.Errorf("structpb payload must not be labelled protoPayload: %v", m) + } + jp, ok := m["jsonPayload"].(map[string]interface{}) + if !ok || jp["reason"] != "FailedPreStopHook" { + t.Fatalf("jsonPayload missing or wrong: %v", m["jsonPayload"]) + } + entry := ExtractFieldsWithPaths(m, []string{"pod"}, map[string]string{"pod": "jsonPayload.involvedObject.name"}) + if entry.Fields["pod"] != "pod-1" { + t.Errorf("pod: got %q", entry.Fields["pod"]) + } +} diff --git a/plugins/debug-tc-logs/skills/debug-tc-logs/SKILL.md b/plugins/debug-tc-logs/skills/debug-tc-logs/SKILL.md index a3bd8f9..190ab40 100644 --- a/plugins/debug-tc-logs/skills/debug-tc-logs/SKILL.md +++ b/plugins/debug-tc-logs/skills/debug-tc-logs/SKILL.md @@ -155,14 +155,17 @@ tc-logview query -e --type --since