Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions core/packages/gax/src/fallbackRest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ export function decodeResponse(
rpc: protobuf.Method,
ok: boolean,
response: Buffer | ArrayBuffer,
httpStatusCode?: number,
): {} {
// eslint-disable-next-line n/no-unsupported-features/node-builtins
const decodedString = new TextDecoder().decode(response);
Expand All @@ -99,6 +100,17 @@ export function decodeResponse(
const json = JSON.parse(decodedString);
if (!ok) {
const error = GoogleError.parseHttpError(json);
// `parseHttpError` reads the status out of the response body and maps it
// onto the gRPC `code`, keeping no record of the status the transport
// actually received — and the body's status can differ from it, or be
// missing entirely. Record the received one when the caller knows it.
//
// Optional because `decodeResponse` is also called from
// `streamArrayParser`, which only ever decodes an already-successful body
// and so has no status to pass.
if (httpStatusCode !== undefined) {
error.httpStatusCode = httpStatusCode;
}
throw error;
}
const message = serializer.fromProto3JSON(rpc.resolvedResponseType!, json);
Expand Down
18 changes: 14 additions & 4 deletions core/packages/gax/src/fallbackServiceStub.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
* limitations under the License.
*/

import type {Response as NodeFetchResponse} from 'node-fetch' with {'resolution-mode': 'import'};
import type {Response as NodeFetchResponse} from 'node-fetch' with {
'resolution-mode': 'import',
};

import {AuthClient, GoogleAuth, gaxios} from 'google-auth-library';
import * as serializer from 'proto3-json-serializer';
Expand All @@ -35,8 +37,7 @@
// - https://github.com/node-fetch/node-fetch#custom-agent
// - https://github.com/googleapis/gax-nodejs/pull/1534
let agentOption:
| ((parsedUrl: {protocol: string}) => HttpAgent | HttpsAgent)
| null = null;
((parsedUrl: {protocol: string}) => HttpAgent | HttpsAgent) | null = null;
if (isNodeJS()) {
const http = require('http');
const https = require('https');
Expand Down Expand Up @@ -221,6 +222,7 @@
rpc: protobuf.Method,
ok: boolean,
response: Buffer | ArrayBuffer,
httpStatusCode?: number,
) => {},
numericEnums: boolean,
minifyJson: boolean,
Expand Down Expand Up @@ -418,7 +420,7 @@
// state, as the handlers below do.
if (err && (timedOut || !cancelRequested)) {
if (callback) {
callback(err);

Check warning on line 423 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
}
streamArrayParser.emit('error', err);
}
Expand All @@ -426,16 +428,24 @@
);
return;
} else {
// Captured here because the decoded value below is also named
// `response` and shadows the fetch response.
const httpStatusCode = response.status;
return Promise.all([
Promise.resolve(response.ok),
response.arrayBuffer(),
])
.then(([ok, buffer]: [boolean, Buffer | ArrayBuffer]) => {

Check warning on line 438 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid nesting promises
const response = responseDecoder(rpc, ok, buffer);
const response = responseDecoder(
rpc,
ok,
buffer,
httpStatusCode,
);
callback!(null, response);
return;
})
.catch((err: Error) => {

Check warning on line 448 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid nesting promises
// The deadline can expire after the response headers arrive but
// before the body is fully read, which rejects here rather than
// in the outer handler.
Expand All @@ -458,7 +468,7 @@
// state we recorded.
if (timedOut || !cancelRequested) {
if (callback) {
callback(callErr);

Check warning on line 471 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
}
streamArrayParser.emit('error', callErr);
}
Expand Down Expand Up @@ -516,12 +526,12 @@
// nobody is listening to any more.
if (timedOut || !cancelRequested) {
if (callback) {
callback(err);

Check warning on line 529 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
}
streamArrayParser.emit('error', err);
}
} else if (callback) {
callback(err);

Check warning on line 534 in core/packages/gax/src/fallbackServiceStub.ts

View workflow job for this annotation

GitHub Actions / lint

Avoid calling back inside of a promise
} else {
throw err;
}
Expand Down
13 changes: 13 additions & 0 deletions core/packages/gax/src/googleError.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,19 @@

export class GoogleError extends Error {
code?: Status;
/**
* The HTTP response status received by the REST fallback transport.
*
* `code` holds the gRPC status the response was mapped to, which is lossy:
* `rpcCodeFromHttpStatusCode` collapses whole ranges (every unmapped 5xx
* becomes INTERNAL), so the original status cannot be recovered from it.
* Telemetry reports the two separately, so the received status is kept here
* as well.
*
* Undefined for gRPC calls, and for fallback failures that never produced a
* response at all, such as an expired deadline or a connection error.
*/
httpStatusCode?: number;
note?: string;
metadata?: Metadata;
statusDetails?: string | protobuf.Message<{}>[];
Expand Down Expand Up @@ -192,7 +205,7 @@
};

// Return true if proto is known in protobuf.
const isDetailKnownProto = (protobuf: any, detail: any): boolean => {

Check warning on line 208 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type

Check warning on line 208 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
try {
const typeName = getProtoTypeNameFromFullNameType(detail['@type']);
if (typeName === ANY_PROTO_TYPE_NAME) {
Expand All @@ -209,7 +222,7 @@
};

// Check if error is unknown type encoded.
const isUnknownTypeEncodedError = (error: any): boolean => {

Check warning on line 225 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
if (typeof error === 'object' && error && 'message' in error) {
return (
error.message.includes(UNKNOWN_TYPE_ENCONDED_ERROR_PREFIX) ||
Expand All @@ -222,7 +235,7 @@
// Build unknown proto as protobuf.Message<{}>.
const buildUnknownProtoAsAny = (
unknownProto: ProtobufAny,
anyProto: any,

Check warning on line 238 in core/packages/gax/src/googleError.ts

View workflow job for this annotation

GitHub Actions / lint

Unexpected any. Specify a different type
): protobuf.Message<{}> => {
return anyProto.create({
type_url: unknownProto.type_url,
Expand Down
72 changes: 71 additions & 1 deletion core/packages/gax/src/observability/TracerHelper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,13 @@
*/

import {EventEmitter} from 'events';
import {Span, SpanStatusCode, trace, Tracer} from '@opentelemetry/api';
import {
Attributes,
Span,
SpanStatusCode,
trace,
Tracer,
} from '@opentelemetry/api';
import {APICallback, GaxCallResult} from '../apitypes';
import {Status} from '../status';

Expand Down Expand Up @@ -105,6 +111,33 @@ function resolveErrorType(e: Error): string {
return e.constructor?.name ?? e.name;
}

/**
* Resolves the gRPC status reported for a failed call, as its name.
*
* Zero is treated as absent rather than as `OK`, for the same reason as in
* `resolveErrorType`: it is the proto3 default for an unset field, so a failed
* call must not be labelled `OK`.
*/
function resolveRpcStatusName(e: unknown): string {
const code = (e as {code?: unknown} | null)?.code;
if (
typeof code === 'number' &&
code !== Status.OK &&
Status[code] !== undefined
) {
return Status[code];
}
return Status[Status.UNKNOWN];
}
Comment on lines +121 to +131

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The resolveRpcStatusName helper currently only resolves numeric status codes. In JavaScript/TypeScript environments, errors (especially custom or third-party ones) may carry status codes as stringified numbers (e.g., '5') or as string status names (e.g., 'NOT_FOUND'). Supporting both stringified numbers and string status names would make the status resolution significantly more robust.

function resolveRpcStatusName(e: unknown): string {
  const code = (e as {code?: unknown} | null)?.code;
  if (typeof code === 'number') {
    if (code !== Status.OK && Status[code] !== undefined) {
      return Status[code];
    }
  } else if (typeof code === 'string') {
    const parsed = parseInt(code, 10);
    if (!isNaN(parsed)) {
      if (parsed !== Status.OK && Status[parsed] !== undefined) {
        return Status[parsed];
      }
    } else if (code !== 'OK' && typeof (Status as any)[code] === 'number') {
      return code;
    }
  }
  return Status[Status.UNKNOWN];
}


/**
* Reads the HTTP response status recorded on a fallback error.
*/
function resolveHttpStatusCode(e: unknown): number | undefined {
const code = (e as {httpStatusCode?: unknown} | null)?.httpStatusCode;
return typeof code === 'number' ? code : undefined;
}

/**
* Checks if a value behaves like a Promise or Thenable.
*
Expand Down Expand Up @@ -333,6 +366,13 @@ export function traceCall(
let spanEnded = false;
let errorRecorded = false;

// Resolved from the error when one is reported, and defaulted to success
// in endSpan otherwise. Held here rather than written immediately so that
// every completion path — promise, stream, callback, synchronous throw —
// emits them from the same place.
let rpcStatusName: string | undefined;
let httpStatusCode: number | undefined;

// Marks the span failed. Kept separate from recordError so paths that are
// failures but not exceptions can set the status without emitting a
// misleading exception event.
Expand All @@ -341,19 +381,49 @@ export function traceCall(
span.setStatus({code: SpanStatusCode.ERROR, message});
};

// The gRPC status is reported for both transports, because it is the one
// status gax resolves on every call and the only one a caller can compare
// across them. The transport-specific attribute is an alias of it on gRPC,
// and the received HTTP status on the fallback, which is a different value
// rather than a restatement of the same one.
//
// Written from one place so the two can never disagree.
const setStatusAttributes = () => {
const attributes: Attributes = {
'rpc.response.status_code': rpcStatusName,
};
if (dynamicArgs.rpcType === 'grpc') {
attributes['grpc.response.status_code'] = rpcStatusName;
} else if (httpStatusCode !== undefined) {
attributes['http.response.status_code'] = httpStatusCode;
}
span.setAttributes(attributes);
};

// Every path ends here, so the status is resolved in one place: ERROR if
// anything reported a failure, OK otherwise.
const endSpan = () => {
if (!spanEnded) {
spanEnded = true;
if (!errorRecorded) {
rpcStatusName = Status[Status.OK];
// Nothing carries the response status back on a successful fallback
// call, and success means a 2xx, so 200 is the only value available.
// A legacy Apiary 204 is therefore also reported as 200.
httpStatusCode = 200;
span.setStatus({code: SpanStatusCode.OK});
}
setStatusAttributes();
span.end();
}
};

const recordError = (e: unknown) => {
// Resolved for every failure, including non-Error throws: those carry no
// status, and resolveRpcStatusName reports UNKNOWN for them, which is
// the right answer for a call that failed for an unmapped reason.
rpcStatusName = resolveRpcStatusName(e);
httpStatusCode = resolveHttpStatusCode(e);
if (e instanceof Error) {
span.setAttributes({
'error.message': e.message,
Expand Down
50 changes: 50 additions & 0 deletions core/packages/gax/test/unit/grpc-fallback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,35 @@ describe('grpc-fallback', () => {
});
});

it('should record the received http status on the error', async () => {
const requestObject = {content: 'test-content'};

// The body reports 400 while the response itself is a 503. `code` is
// derived from the body, so a status read back off the error can only be
// the received one if the two differ.
setMockFallbackHttpResponse(
gaxGrpc,
new Response(
JSON.stringify({error: {code: 400, message: 'mismatched status'}}),
{status: 503},
),
);

const echoStub = await gaxGrpc.createStub(echoService, stubOptions);
await new Promise<void>((resolve, reject) => {
echoStub.echo(requestObject, {}, {}, (err?: Error) => {
try {
assert(err instanceof GoogleError);
assert.strictEqual(err.code, Status.INVALID_ARGUMENT);
assert.strictEqual(err.httpStatusCode, 503);
resolve();
} catch (e) {
reject(e);
}
});
});
});

it('should promote ErrorInfo if exist in fallback-rest error', async () => {
const requestObject = {content: 'test-content'};
// example of an actual google.rpc.Status error message returned by Translate API
Expand Down Expand Up @@ -1357,5 +1386,26 @@ describe('grpc-fallback', () => {
assert.strictEqual(err.code, Status.UNAVAILABLE);
assert.notStrictEqual(err.code as number, 503);
});

it('should decode a resolved error response and record its http status', async () => {
// 500 now passes `validateStatus`, so it resolves and is decoded, which
// is what lets the received HTTP status be recorded alongside the
// gRPC code the body maps to.
setMockFallbackHttpResponse(
gaxGrpc,
new Response(
JSON.stringify({
error: {code: 500, message: 'server blew up', status: 'INTERNAL'},
}),
{status: 500, headers: {'Content-Type': 'application/json'}},
),
);

const err = await callEcho();

assert(err instanceof GoogleError);
assert.strictEqual(err.code, Status.INTERNAL);
assert.strictEqual(err.httpStatusCode, 500);
});
});
});
117 changes: 117 additions & 0 deletions core/packages/gax/test/unit/otelHarness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,123 @@ export class OtelHarness {
);
}
}

/**
* The three response status attributes carried by a traced call.
*
* @param {ReadableSpan} span - The span to read.
* @returns {ResponseStatusAttributes} The attributes, each undefined if absent.
*/
responseStatus(span: ReadableSpan): ResponseStatusAttributes {
return {
rpc: span.attributes['rpc.response.status_code'] as string | undefined,
grpc: span.attributes['grpc.response.status_code'] as string | undefined,
http: span.attributes['http.response.status_code'] as number | undefined,
};
}

/**
* Asserts the response status attributes of a traced call.
*
* The transport-specific attribute is not named by the caller. It is derived
* from the span's own `gcp.method.type`, so a test cannot assert a
* combination the tracer is not supposed to produce — such as an HTTP status
* on a gRPC span. Both the presence of the attribute that applies and the
* absence of the one that does not are checked, because the second half is
* what catches an attribute leaking onto the wrong transport.
*
* `httpStatus` is only meaningful on a fallback span. Omitting it there
* asserts that no HTTP status was reported, which is the expected result for
* a failure that never received a response, such as an expired deadline.
*
* @param {object} expected - Expected status values.
* @param {string} expected.rpcStatus - gRPC status name, e.g. 'OK' or 'NOT_FOUND'.
* @param {number} [expected.httpStatus] - HTTP status expected on a fallback span.
* @param {object} [options] - Span selection.
* @param {string} [options.tracerName] - Restrict the lookup to one instrumentation scope.
* @param {ReadableSpan} [options.span] - Span to check; defaults to the only exported span.
*/
assertResponseStatus(
expected: {rpcStatus: string; httpStatus?: number},
options: {tracerName?: string; span?: ReadableSpan} = {},
): void {
const target = options.span ?? this.requireSingleSpan(options.tracerName);
const actual = this.responseStatus(target);
const transport = target.attributes['gcp.method.type'];
const where = `span '${target.name}'`;

assert.ok(
transport === 'grpc' || transport === 'http',
`${where} has gcp.method.type ${JSON.stringify(transport)}; the ` +
'transport-specific status attribute cannot be checked without it. ' +
'Was this span produced by traceCall?',
);

assert.strictEqual(
actual.rpc,
expected.rpcStatus,
`expected ${where} to report rpc.response.status_code ` +
`${JSON.stringify(expected.rpcStatus)}, got ${JSON.stringify(actual.rpc)}. ` +
'This attribute is reported on every call, on both transports.',
);

if (transport === 'grpc') {
assert.strictEqual(
actual.grpc,
expected.rpcStatus,
`expected ${where} to report grpc.response.status_code ` +
`${JSON.stringify(expected.rpcStatus)}, got ${JSON.stringify(actual.grpc)}. ` +
'On a gRPC span it mirrors rpc.response.status_code.',
);
assert.strictEqual(
actual.http,
undefined,
`${where} is a gRPC span but reported http.response.status_code ` +
`${JSON.stringify(actual.http)}. A gRPC call has no HTTP status, ` +
'not even a synthesized one.',
);
assert.strictEqual(
expected.httpStatus,
undefined,
'assertResponseStatus was given an expected httpStatus for a gRPC ' +
'span, which can never hold one. Drop it, or assert against a ' +
'fallback span.',
);
return;
}

assert.strictEqual(
actual.grpc,
undefined,
`${where} is a fallback span but reported grpc.response.status_code ` +
`${JSON.stringify(actual.grpc)}. The gRPC status is reported as ` +
'rpc.response.status_code there, not under the grpc.* name.',
);
assert.strictEqual(
actual.http,
expected.httpStatus,
expected.httpStatus === undefined
? `expected ${where} to report no http.response.status_code, got ` +
`${JSON.stringify(actual.http)}. It is only reported when a ` +
'response was actually received.'
: `expected ${where} to report http.response.status_code ` +
`${expected.httpStatus}, got ${JSON.stringify(actual.http)}. ` +
'This is the status the transport received, which is not ' +
'recoverable from the gRPC status it was mapped to.',
);
}
}

/**
* The response status attributes read off a traced span.
*/
export interface ResponseStatusAttributes {
/** `rpc.response.status_code`: gRPC status name, reported on both transports. */
rpc: string | undefined;
/** `grpc.response.status_code`: gRPC spans only. */
grpc: string | undefined;
/** `http.response.status_code`: fallback spans that received a response. */
http: number | undefined;
}

/**
Expand Down
Loading
Loading