/** * Created by ls on 9/7/2018. * * This `network.ts` is the NON-MEDIA API arm of network fetch requests. * It is built on `Network` object and provides standard functionality, such as: * 1. Parsing the body into specific format. * 2. Adding timing metrics onto blob. * * This should *only* be used for objects that should have timing metrics, i.e. requests to Non-MediaAPI endpoints * that will ultimately render some whole page. Otherwise, use `objectGraph.network.fetch` directly. * * @see `src/media/network.ts` for fetching from Media API endpoints */ import { FetchRequest, FetchResponse, HTTPTimingMetrics } from "@jet/environment/types/globals/net"; import { FetchTimingMetrics, MetricsFields } from "@jet/environment/types/metrics"; import { FetchTimingMetricsBuilder } from "@jet/environment/metrics"; import { isSome, Opt } from "@jet/environment/types/optional"; import * as serverData from "./models/server-data"; import { ParsedNetworkResponse } from "./models/data-structure"; import { Request } from "./models/request"; import * as urls from "./models/urls"; import * as urlBuilder from "./url-builder"; import { MediaConfigurationType, MediaTokenService } from "./models/mediapi-configuration"; import { HTTPMethod, HTTPCachePolicy, HTTPSigningStyle, HTTPHeaders } from "@jet/environment"; /** @public */ // eslint-disable-next-line @typescript-eslint/no-namespace export namespace ResponseMetadata { export const requestedUrl = "_jet-internal:metricsHelpers_requestedUrl"; /** * Symbol used to place timing metrics values onto fetch responses * without interfering with the data returned by the server. */ export const timingValues = "_jet-internal:metricsHelpers_timingValues"; /** * Key used to access the page information gathered from a response's headers */ export const pageInformation = "_jet-internal:metricsHelpers_pageInformation"; /** * Key used to access the content max-age gathered from a response's headers. */ export const contentMaxAge = "_jet-internal:responseMetadata_contentMaxAge"; } /** * Module's private fetch implementation built off `net` global. * * @param {FetchRequest} request describes fetch request. * @param {(value: string) => Type} parser Some function parsing response body `string` into specific type. * @returns {Promise} Promise resolving to specific object. * @throws {Error} Throws error if status code of request is not 200. * * @note Similar to `fetchWithToken` in `media` module, but excludes media token specific functionality. * Top level data fetches to endpoints that don't do redirects, and can benefit from metrics should * call methods that build off of this instead of calling `objectGraph.network.fetch(...)` directly. */ export async function fetch( configuration: MediaConfigurationType, request: FetchRequest, parser: (value: Opt) => Type, ): Promise { const response = await configuration.network.fetch(request); if (!response.ok) { throw Error(`Bad Status code ${response.status} for ${request.url}`); } const parseStartTime = Date.now(); const result = parser(response.body) as Type & ParsedNetworkResponse; const parseEndTime = Date.now(); // Build full network timing metrics. const completeTimingMetrics = networkTimingMetricsWithParseTime(response.metrics, parseStartTime, parseEndTime); if (serverData.isDefinedNonNull(completeTimingMetrics)) { result[ResponseMetadata.timingValues] = completeTimingMetrics; } result[ResponseMetadata.requestedUrl] = request.url.toString(); return result; } /** * Fetch from an endpoint with JSON response body. * * @param {FetchRequest} request to fetch from endpoint with JSON response.. * @returns {Promise} Promise resolving to body of response parsed as `Type`. * @throws {Error} Throws error if status code of request is not 200. */ export async function fetchJSON(configuration: MediaConfigurationType, request: FetchRequest): Promise { return await fetch(configuration, request, (body) => { if (isSome(body)) { return JSON.parse(body) as Type; } else { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions return {} as Type; } }); } /** * Fetch from an endpoint with XML response body. * * @param {FetchRequest} request to fetch from endpoint with XML response. * @returns {Promise} Promise resolving to body of response parsed as `Type`. * @throws {Error} Throws error if status code of request is not 200. */ export async function fetchPlist(configuration: MediaConfigurationType, request: FetchRequest): Promise { return await fetch(configuration, request, (body) => { if (isSome(body)) { return configuration.plist.parse(body) as Type; } else { throw new Error(`Could not fetch Plist, response body was not defined for ${request.url}`); } }); } /** * With network requests now being created and parsed in JS, different timing metrics are measured in both Native and JS. * This function populates the missing values from `HTTPTimingMetrics`'s native counterpart, `JSNetworkPerformanceMetrics`. * * @param {HTTPTimingMetrics[] | null} responseMetrics Array of response metrics provided by native. * @param {number} parseStartTime Time at which response body string parse began in JS. * @param {number} parseEndTime Time at which response body string parse ended in JS. * @returns {HTTPTimingMetrics | null} Fully populated timing metrics, or `null` if native response provided no metrics events to build off of. */ function networkTimingMetricsWithParseTime( responseMetrics: HTTPTimingMetrics[] | null, parseStartTime: number, parseEndTime: number, ): FetchTimingMetrics | null { // No metrics events to build from. if (serverData.isNull(responseMetrics) || responseMetrics.length === 0) { return null; } // Append parse times to first partial timing metrics from native. const firstPartialTimingMetrics: FetchTimingMetrics = { ...responseMetrics[0], parseStartTime: parseStartTime, parseEndTime: parseEndTime, }; // Timing metrics with all properties populated. return firstPartialTimingMetrics; } export type FetchOptions = { headers?: { [key: string]: string }; method?: "GET" | "HEAD" | "POST" | "PUT" | "DELETE" | "CONNECT" | "OPTIONS" | "TRACE" | "PATCH"; requestBodyString?: string; timeout?: number; // in seconds. Check for feature 'supportsRequestTimeoutOption'. /// When true the fetch wont throw if we dont get any data back for given request. allowEmptyDataResponse?: boolean; excludeIdentifierHeadersForAccount?: boolean; // Defaults to false alwaysIncludeAuthKitHeaders?: boolean; // Defaults to true alwaysIncludeMMeClientInfoAndDeviceHeaders?: boolean; // Defaults to true }; /** * Implements the MAPI fetch, building URL from MAPI Request and opaquely managing initial token request and refreshes. * * @param {MediaConfigurationType} configuration Base media API configuration. * @param {Request} request MAPI Request to fetch with. * @param {FetchOptions} [options] FetchOptions for the MAPI request. * @returns {Promise} Promise resolving to some type for given MAPI request. */ export async function fetchData( configuration: MediaConfigurationType, mediaToken: MediaTokenService, request: Request, options?: FetchOptions, ): Promise { const url = urlBuilder.buildURLFromRequest(configuration, request).toString(); const startTime = Date.now(); const token = await mediaToken.refreshToken(); const response = await fetchWithToken( configuration, mediaToken, url, token, options, false, configuration.fetchTimingMetricsBuilder, ); const endTime = Date.now(); if (request.canonicalUrl) { response[ResponseMetadata.requestedUrl] = request.canonicalUrl; } const roundTripTimeIncludingWaiting = endTime - startTime; if (roundTripTimeIncludingWaiting > 500) { console.warn("Fetch took too long (" + roundTripTimeIncludingWaiting.toString() + "ms) " + url); } return response; } export function redirectParametersInUrl(configuration: MediaConfigurationType, url: urls.URL): string[] { const redirectURLParams = configuration.redirectUrlWhitelistedQueryParams; return redirectURLParams.filter((param) => serverData.isDefinedNonNull(url.query?.[param])); } export type MediaAPIFetchRequest = { url: string; excludeIdentifierHeadersForAccount?: boolean; alwaysIncludeAuthKitHeaders?: boolean; alwaysIncludeMMeClientInfoAndDeviceHeaders?: boolean; method?: Opt; cache?: Opt; signingStyle?: Opt; headers?: Opt; timeout?: Opt; body?: Opt; }; /** * Given a built URL, token, and options, calls into native networking APIs to fetch content. * * @param {string} url URL to fetch data from. * @param {string} token MAPI token key. * @param {FetchOptions} options Fetch options for MAPI requests. * @param {boolean} isRetry flag indicating whether this is a fetch retry following a 401 request, and media token was refreshed. * @returns {Promise} Promise resolving to some type for given MAPI request. */ async function fetchWithToken( configuration: MediaConfigurationType, mediaToken: MediaTokenService, url: string, token: string, options: FetchOptions = {}, isRetry = false, fetchTimingMetricsBuilder: Opt, ): Promise { // Removes all affiliate/redirect params for caching (https://connectme.apple.com/docs/DOC-577671) const filteredURL = new urls.URL(url); const redirectParameters = redirectParametersInUrl(configuration, filteredURL); for (const param of redirectParameters) { filteredURL.removeParam(param); } const filteredUrlString = filteredURL.toString(); let headers = options.headers; if (headers == null) { headers = {}; } headers["Authorization"] = "Bearer " + token; const fetchRequest: MediaAPIFetchRequest = { url: filteredUrlString, excludeIdentifierHeadersForAccount: options.excludeIdentifierHeadersForAccount ?? false, alwaysIncludeAuthKitHeaders: options.alwaysIncludeAuthKitHeaders ?? true, alwaysIncludeMMeClientInfoAndDeviceHeaders: options.alwaysIncludeMMeClientInfoAndDeviceHeaders ?? true, headers: headers, method: options.method, body: options.requestBodyString, timeout: options.timeout, }; const response = await configuration.network.fetch(fetchRequest); try { if (response.status === 401 || response.status === 403) { if (isRetry) { throw Error("We refreshed the token but we still get 401 from the API"); } mediaToken.resetToken(); return await mediaToken.refreshToken().then(async (newToken) => { // Explicitly re-fetch with the original request so logging and metrics are correct return await fetchWithToken( configuration, mediaToken, url, newToken, options, true, fetchTimingMetricsBuilder, ); }); } else if (response.status === 404) { // item is not available in this storefront or perhaps not at all throw noContentError(); } else if (!response.ok) { const correlationKey = response.headers["x-apple-jingle-correlation-key"] ?? "N/A"; const error = new NetworkError( `Bad Status code ${response.status} (correlationKey: ${correlationKey}) for ${filteredUrlString}, original ${url}`, ); error.statusCode = response.status; throw error; } const parser = (resp: FetchResponse) => { const parseStartTime = Date.now(); let result: Type & ParsedNetworkResponse; if (serverData.isNull(resp.body) || resp.body === "") { if (resp.status === 204) { // 204 indicates a success, but the response will typically be empty // Create a fake result so that we don't throw an error when JSON parsing const emptyData: ParsedNetworkResponse = {}; result = emptyData as Type & ParsedNetworkResponse; } else { throw noContentError(); } } else { result = JSON.parse(resp.body) as Type & ParsedNetworkResponse; } const parseEndTime = Date.now(); result[ResponseMetadata.pageInformation] = serverData.asJSONData( getPageInformationFromResponse(configuration, resp), ); if (resp.metrics.length > 0) { const metrics: FetchTimingMetrics = { ...resp.metrics[0], parseStartTime: parseStartTime, parseEndTime: parseEndTime, }; result[ResponseMetadata.timingValues] = metrics; } else { const fallbackMetrics: FetchTimingMetrics = { pageURL: resp.url, parseStartTime, parseEndTime, }; result[ResponseMetadata.timingValues] = fallbackMetrics; } result[ResponseMetadata.contentMaxAge] = getContentTimeToLiveFromResponse(resp); // If we have an empty data object, throw a 204 (No Content). if ( Array.isArray(result.data) && serverData.isArrayDefinedNonNullAndEmpty(result.data) && !serverData.asBooleanOrFalse(options.allowEmptyDataResponse) ) { throw noContentError(); } result[ResponseMetadata.requestedUrl] = url; return result; }; if (isSome(fetchTimingMetricsBuilder)) { return fetchTimingMetricsBuilder.measureParsing(response, parser); } else { return parser(response); } } catch (e) { if (e instanceof NetworkError) { throw e; } throw new Error(`Error Fetching - filtered: ${filteredUrlString}, original: ${url}, ${e.name}, ${e.message}`); } } export class NetworkError extends Error { statusCode?: number; } function noContentError(): NetworkError { const error = new NetworkError(`No content`); error.statusCode = 204; return error; } const serverInstanceHeader = "x-apple-application-instance"; const environmentDataCenterHeader = "x-apple-application-site"; function getPageInformationFromResponse( configuration: MediaConfigurationType, response: FetchResponse, ): MetricsFields | null { const storeFrontHeader: string = configuration.storefrontIdentifier; let storeFront: Opt = null; if (serverData.isDefinedNonNullNonEmpty(storeFrontHeader)) { const storeFrontHeaderComponents: string[] = storeFrontHeader.split("-"); if (serverData.isDefinedNonNullNonEmpty(storeFrontHeaderComponents)) { storeFront = storeFrontHeaderComponents[0]; } } return { serverInstance: response.headers[serverInstanceHeader], storeFrontHeader: storeFrontHeader, language: configuration.bagLanguage, storeFront: storeFront, environmentDataCenter: response.headers[environmentDataCenterHeader], }; } function getContentTimeToLiveFromResponse(response: FetchResponse): Opt { const cacheControlHeaderKey = Object.keys(response.headers).find((key) => key.toLowerCase() === "cache-control"); if (serverData.isNull(cacheControlHeaderKey) || cacheControlHeaderKey === "") { return null; } const headerValue = response.headers[cacheControlHeaderKey]; if (serverData.isNullOrEmpty(headerValue)) { return null; } const matches = headerValue.match(/max-age=(\d+)/); if (serverData.isNull(matches) || matches.length < 2) { return null; } return serverData.asNumber(matches[1]); }