Source: lib/net/http_fetch_plugin.js

/*! @license
 * Shaka Player
 * Copyright 2016 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

goog.provide('shaka.net.HttpFetchPlugin');

goog.require('goog.asserts');
goog.require('shaka.log');
goog.require('shaka.net.HttpPluginUtils');
goog.require('shaka.net.NetworkingEngine');
goog.require('shaka.util.AbortableOperation');
goog.require('shaka.util.BufferUtils');
goog.require('shaka.util.Error');
goog.require('shaka.util.MapUtils');
goog.require('shaka.util.Timer');
goog.require('shaka.util.Uint8ArrayUtils');


/**
 * @summary A networking plugin to handle http and https URIs via the Fetch API.
 * @export
 */
shaka.net.HttpFetchPlugin = class {
  /**
   * @param {string} uri
   * @param {shaka.extern.Request} request
   * @param {shaka.net.NetworkingEngine.RequestType} requestType
   * @param {shaka.extern.ProgressUpdated} progressUpdated Called when a
   *   progress event happened.
   * @param {shaka.extern.HeadersReceived} headersReceived Called when the
   *   headers for the download are received, but before the body is.
   * @param {shaka.extern.SchemePluginConfig} config
   * @return {!shaka.extern.IAbortableOperation.<shaka.extern.Response>}
   * @export
   */
  static parse(uri, request, requestType, progressUpdated, headersReceived,
      config) {
    const headers = new shaka.net.HttpFetchPlugin.Headers_();
    try {
      shaka.util.MapUtils.asMap(request.headers).forEach((value, key) => {
        headers.append(key, value);
      });
    } catch (error) {
      shaka.log.debug('Failed to create headers:', error);
      return shaka.util.AbortableOperation.failed(
          new shaka.util.Error(
              shaka.util.Error.Severity.RECOVERABLE,
              shaka.util.Error.Category.NETWORK,
              shaka.util.Error.Code.HTTP_ERROR,
              uri, error, requestType));
    }

    const controller = new shaka.net.HttpFetchPlugin.AbortController_();

    /** @type {!RequestInit} */
    const init = {
      // Edge does not treat null as undefined for body; https://bit.ly/2luyE6x
      body: request.body || undefined,
      headers: headers,
      method: request.method,
      signal: controller.signal,
      credentials: request.allowCrossSiteCredentials ? 'include' : undefined,
    };

    /** @type {shaka.net.HttpFetchPlugin.AbortStatus} */
    const abortStatus = {
      canceled: false,
      timedOut: false,
    };

    const minBytes = config.minBytesForProgressEvents || 0;

    const pendingRequest = shaka.net.HttpFetchPlugin.request_(
        uri, request, requestType, init, abortStatus, progressUpdated,
        headersReceived, request.streamDataCallback, minBytes);

    /** @type {!shaka.util.AbortableOperation} */
    const op = new shaka.util.AbortableOperation(pendingRequest, () => {
      abortStatus.canceled = true;
      controller.abort();
      return Promise.resolve();
    });

    // The fetch API does not timeout natively, so do a timeout manually using
    // the AbortController.
    const timeoutMs = request.retryParameters.timeout;
    if (timeoutMs) {
      const timer = new shaka.util.Timer(() => {
        abortStatus.timedOut = true;
        controller.abort();
      });

      timer.tickAfter(timeoutMs / 1000);

      // To avoid calling |abort| on the network request after it finished, we
      // will stop the timer when the requests resolves/rejects.
      op.finally(() => {
        timer.stop();
      });
    }

    return op;
  }

  /**
   * @param {string} uri
   * @param {shaka.extern.Request} request
   * @param {shaka.net.NetworkingEngine.RequestType} requestType
   * @param {!RequestInit} init
   * @param {shaka.net.HttpFetchPlugin.AbortStatus} abortStatus
   * @param {shaka.extern.ProgressUpdated} progressUpdated
   * @param {shaka.extern.HeadersReceived} headersReceived
   * @param {?function(BufferSource):!Promise} streamDataCallback
   * @param {number} minBytes
   * @return {!Promise<!shaka.extern.Response>}
   * @private
   */
  static async request_(uri, request, requestType, init, abortStatus,
      progressUpdated, headersReceived, streamDataCallback, minBytes) {
    const fetch = shaka.net.HttpFetchPlugin.fetch_;
    let response;
    let arrayBuffer = new ArrayBuffer(0);
    let loaded = 0;
    let lastLoaded = 0;
    let headers = {};

    // Last time stamp when we got a progress event.
    let lastTime = Date.now();

    try {
      // The promise returned by fetch resolves as soon as the HTTP response
      // headers are available. The download itself isn't done until the promise
      // for retrieving the data (arrayBuffer, blob, etc) has resolved.
      response = await fetch(uri, init);
      // At this point in the process, we have the headers of the response, but
      // not the body yet.
      headers =
          shaka.net.HttpFetchPlugin.headersToGenericObject_(response.headers);
      headersReceived(headers);

      // Only stream the body of a successful response.  An error body (e.g. a
      // 404 for a low-latency segment requested just before it is available)
      // is not media, and the stream data callback's state outlives this
      // attempt: if a retry succeeds, the error body would be parsed together
      // with the segment, and nothing of it appended.
      if (!shaka.net.HttpPluginUtils.isSuccessStatus(response.status)) {
        streamDataCallback = null;
      }

      // In new versions of Chromium, HEAD requests now have a response body
      // that is null.
      // So just don't try to download the body at all, if it's a HEAD request,
      // to avoid null reference errors.
      // See: https://crbug.com/1297060
      if (init.method != 'HEAD') {
        goog.asserts.assert(response.body,
            'non-HEAD responses should have a body');

        const contentLengthRaw = response.headers.get('Content-Length');
        const contentLength =
            contentLengthRaw ? parseInt(contentLengthRaw, 10) : 0;

        // Fetch returning a ReadableStream response body is not currently
        // supported by all browsers.
        // Browser compatibility:
        // https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API
        // If it is not supported, returning the whole segment when
        // it's ready (as xhr)
        if (!response.body) {
          arrayBuffer = await response.arrayBuffer();
          const currentTime = Date.now();
          // If the time between last time and this time we got progress event
          // is long enough, or if a whole segment is downloaded, call
          // progressUpdated().
          progressUpdated(currentTime - lastTime, arrayBuffer.byteLength, 0);
        } else {
          // Read the body with a single reader, to observe the download
          // progress and to stream the data, and keep it as it comes.  Neither
          // cloning the response nor reading it a second time with
          // arrayBuffer(): both keep another copy of the body in memory while
          // it is downloaded.
          const {reader, byob} = shaka.net.HttpFetchPlugin.getReader_(
              response.body, contentLength, response.headers);

          /**
           * Streams the data of a read, and reports the progress.
           * @param {?Uint8Array} chunk The data, or null at the end.
           * @return {!Promise}
           */
          const onRead = async (chunk) => {
            if (chunk) {
              loaded += chunk.byteLength;
              if (streamDataCallback) {
                await streamDataCallback(chunk);
              }
            }

            const currentTime = Date.now();
            const chunkSize = loaded - lastLoaded;
            // If the time between last time and this time we got progress
            // event is long enough, or if a whole segment is downloaded, call
            // progressUpdated().
            if ((currentTime - lastTime > 100 && chunkSize >= minBytes) ||
                !chunk) {
              const numBytesRemaining = chunk ? contentLength - loaded : 0;
              progressUpdated(currentTime - lastTime, chunkSize,
                  numBytesRemaining);
              lastLoaded = loaded;
              lastTime = currentTime;
            }
          };

          if (byob) {
            arrayBuffer = await shaka.net.HttpFetchPlugin.readWithByob_(
                /** @type {!ReadableStreamBYOBReader} */ (reader),
                contentLength, onRead, !!streamDataCallback);
          } else {
            arrayBuffer = await shaka.net.HttpFetchPlugin.readWithDefault_(
                /** @type {!ReadableStreamDefaultReader} */ (reader),
                contentLength, onRead);
          }
        }
      }

      if (request.headers['Range']) {
        const range = request.headers['Range'].replace('bytes=', '').split('-')
            .filter((r) => r).map((r) => parseInt(r, 10));
        if (range.length == 2 &&
            arrayBuffer.byteLength != (range[1] - range[0] + 1)) {
          shaka.log.alwaysWarn(
              'Payload length does not match range requested bytes',
              request, response);
        }
      }
    } catch (error) {
      if (abortStatus.canceled) {
        throw new shaka.util.Error(
            shaka.util.Error.Severity.RECOVERABLE,
            shaka.util.Error.Category.NETWORK,
            shaka.util.Error.Code.OPERATION_ABORTED,
            uri, requestType);
      } else if (abortStatus.timedOut) {
        throw new shaka.util.Error(
            shaka.util.Error.Severity.RECOVERABLE,
            shaka.util.Error.Category.NETWORK,
            shaka.util.Error.Code.TIMEOUT,
            uri, requestType);
      } else {
        throw new shaka.util.Error(
            shaka.util.Error.Severity.RECOVERABLE,
            shaka.util.Error.Category.NETWORK,
            shaka.util.Error.Code.HTTP_ERROR,
            uri, error, requestType);
      }
    }

    return shaka.net.HttpPluginUtils.makeResponse(headers, arrayBuffer,
        response.status, uri, response.url, request, requestType);
  }

  /**
   * Gets a reader of the body.  A BYOB ("bring your own buffer") reader is
   * used when the browser supports it for fetch bodies and the size of the
   * body is known: the data is then read straight into the buffer of the
   * response, without allocating every chunk.  Otherwise, a default reader.
   *
   * @param {!ReadableStream} body
   * @param {number} contentLength
   * @param {!Headers} headers
   * @return {{
   *   reader: (!ReadableStreamDefaultReader|!ReadableStreamBYOBReader),
   *   byob: boolean,
   * }}
   * @private
   */
  static getReader_(body, contentLength, headers) {
    const BYOBReader = shaka.net.HttpFetchPlugin.ReadableStreamBYOBReader_;
    // With a compressed response, Content-Length is the size before decoding,
    // not the size of the body.
    if (BYOBReader && contentLength > 0 && !headers.get('Content-Encoding')) {
      let reader = null;
      try {
        reader = body.getReader({mode: 'byob'});
      } catch (e) {
        // The body is not a byte stream (e.g. Chrome 89 to 115).  The stream
        // is not locked, so a default reader can still be used.
      }
      if (reader) {
        // An implementation that ignores the mode gives a default reader.  The
        // stream is locked by it, so it has to be used anyway.
        return {reader, byob: reader instanceof BYOBReader};
      }
    }
    return {reader: body.getReader(), byob: false};
  }

  /**
   * Reads a body with a default reader.
   *
   * @param {!ReadableStreamDefaultReader} reader
   * @param {number} contentLength
   * @param {function(?Uint8Array):!Promise} onRead
   * @return {!Promise<!ArrayBuffer>}
   * @private
   */
  static async readWithDefault_(reader, contentLength, onRead) {
    const body = new shaka.net.HttpFetchPlugin.BodyBuffer_(contentLength);
    let done = false;
    while (!done) {
      // The chunks of a stream can only be read one after another.
      // eslint-disable-next-line no-await-in-loop
      const readObj = await reader.read();
      done = readObj.done;
      const chunk = done ? null : /** @type {!Uint8Array} */ (readObj.value);
      if (chunk) {
        body.append(chunk);
      }
      // Chunks are passed on in order, one at a time.
      // eslint-disable-next-line no-await-in-loop
      await onRead(chunk);
    }
    return body.getArrayBuffer();
  }

  /**
   * Reads a body with a BYOB reader, into a buffer of the expected length.
   * Every read transfers that buffer: the view given to read() is detached,
   * and the same memory comes back as the buffer of the result.
   *
   * @param {!ReadableStreamBYOBReader} reader
   * @param {number} contentLength
   * @param {function(?Uint8Array):!Promise} onRead
   * @param {boolean} copyChunks Whether onRead gets copies of the data.  The
   *   data read is a view of the buffer, which the next read detaches, so it
   *   must be copied if it is kept, as a stream data callback may do.
   * @return {!Promise<!ArrayBuffer>}
   * @private
   */
  static async readWithByob_(reader, contentLength, onRead, copyChunks) {
    // The statics are not used through an alias of the class: in an async
    // function, the compiler collapses them but keeps the alias.
    let buffer = new ArrayBuffer(contentLength);
    let offset = 0;
    // If the body is larger than expected, the rest is read into new buffers,
    // and everything is joined at the end.
    /** @type {?shaka.net.HttpFetchPlugin.BodyBuffer_} */
    let overflow = null;
    let done = false;
    while (!done) {
      const intoBuffer = !overflow && offset < contentLength;
      // Once the buffer is full, a read is still needed to see the end.
      const view = intoBuffer ?
          new Uint8Array(buffer, offset, contentLength - offset) :
          new Uint8Array(shaka.net.HttpFetchPlugin.BYOB_OVERFLOW_CHUNK_SIZE_);
      // The chunks of a stream can only be read one after another.
      // eslint-disable-next-line no-await-in-loop
      const readObj = await reader.read(view);
      const value = readObj.value;
      if (intoBuffer && value) {
        buffer = shaka.util.BufferUtils.getTransferredBuffer(value);
      }
      done = readObj.done;
      let chunk = null;
      if (!done) {
        goog.asserts.assert(value, 'A read that is not done has data');
        chunk = /** @type {!Uint8Array} */ (value);
        if (intoBuffer) {
          offset += chunk.byteLength;
        } else {
          if (!overflow) {
            overflow = new shaka.net.HttpFetchPlugin.BodyBuffer_(0);
            overflow.append(new Uint8Array(buffer, 0, offset));
          }
          overflow.append(chunk);
        }
        if (copyChunks) {
          chunk = chunk.slice();
        }
      }
      // Chunks are passed on in order, one at a time.
      // eslint-disable-next-line no-await-in-loop
      await onRead(chunk);
    }
    if (overflow) {
      return overflow.getArrayBuffer();
    }
    // No copy is made if the body has the expected size.
    return shaka.util.BufferUtils.toArrayBuffer(
        new Uint8Array(buffer, 0, offset));
  }

  /**
   * @param {!Headers} headers
   * @return {!Object<string, string>}
   * @private
   */
  static headersToGenericObject_(headers) {
    const headersObj = {};
    headers.forEach((value, key) => {
      // Since Edge incorrectly return the header with a leading new line
      // character ('\n'), we trim the header here.
      headersObj[key.trim()] = value;
    });
    return headersObj;
  }

  /**
   * Determine if the Fetch API is supported in the browser. Note: this is
   * deliberately exposed as a method to allow the client app to use the same
   * logic as Shaka when determining support.
   * @return {boolean}
   * @export
   */
  static isSupported() {
    // On Edge, ReadableStream exists, but attempting to construct it results in
    // an error. See https://bit.ly/2zwaFLL
    // So this has to check that ReadableStream is present AND usable.
    if (window.ReadableStream) {
      try {
        new ReadableStream({}); // eslint-disable-line no-new
      } catch (e) {
        return false;
      }
    } else {
      return false;
    }
    // Old fetch implementations hasn't body and ReadableStream implementation
    // See: https://github.com/shaka-project/shaka-player/issues/5088
    if (window.Response) {
      const response = new Response('');
      if (!response.body) {
        return false;
      }
    } else {
      return false;
    }
    return !!(window.fetch && !('polyfill' in window.fetch) &&
      window.AbortController);
  }
};


/**
 * @typedef {{
 *   canceled: boolean,
 *   timedOut: boolean,
 * }}
 * @property {boolean} canceled
 *   Indicates if the request was canceled.
 * @property {boolean} timedOut
 *   Indicates if the request timed out.
 */
shaka.net.HttpFetchPlugin.AbortStatus;


/**
 * Collects the chunks of a response body.  When the size of the body is known,
 * the chunks are written into a buffer of that size, so that the body is held
 * once, and no copy is made at the end.  Otherwise, or if the body turns out to
 * be larger (e.g. a compressed response, where Content-Length is the size
 * before decoding), the chunks are kept and joined at the end.
 *
 * @private
 */
shaka.net.HttpFetchPlugin.BodyBuffer_ = class {
  /**
   * @param {number} expectedLength The Content-Length, or 0 if unknown.
   */
  constructor(expectedLength) {
    /** @private {?Uint8Array} */
    this.buffer_ = null;
    if (expectedLength > 0) {
      try {
        this.buffer_ = new Uint8Array(expectedLength);
      } catch (e) {
        // An invalid or too large length.  Keep the chunks instead.
      }
    }

    /** @private {number} */
    this.length_ = 0;

    /** @private {!Array<!Uint8Array>} */
    this.chunks_ = [];
  }

  /**
   * @param {!Uint8Array} chunk
   */
  append(chunk) {
    if (this.buffer_ &&
        this.length_ + chunk.byteLength > this.buffer_.byteLength) {
      // Larger than expected, so the chunks are joined at the end.
      this.chunks_.push(this.buffer_.subarray(0, this.length_));
      this.buffer_ = null;
    }
    if (this.buffer_) {
      this.buffer_.set(chunk, this.length_);
    } else {
      this.chunks_.push(chunk);
    }
    this.length_ += chunk.byteLength;
  }

  /**
   * @return {!ArrayBuffer}
   */
  getArrayBuffer() {
    const BufferUtils = shaka.util.BufferUtils;
    if (this.buffer_) {
      // No copy is made if the body has the expected size.
      return BufferUtils.toArrayBuffer(this.buffer_.subarray(0, this.length_));
    }
    return BufferUtils.toArrayBuffer(
        shaka.util.Uint8ArrayUtils.concatRange(this.chunks_));
  }
};


/**
 * The size of the buffers read into with a BYOB reader once the body is larger
 * than its Content-Length.
 *
 * @const {number}
 * @private
 */
shaka.net.HttpFetchPlugin.BYOB_OVERFLOW_CHUNK_SIZE_ = 65536;


/**
 * Overridden in unit tests, but compiled out in production.  It is an
 * interface in the externs, so it can't be typed as a constructor.
 *
 * @const {(Function|undefined)}
 * @private
 */
shaka.net.HttpFetchPlugin.ReadableStreamBYOBReader_ =
    window['ReadableStreamBYOBReader'];


/**
 * Overridden in unit tests, but compiled out in production.
 *
 * @const {function(string, !RequestInit)}
 * @private
 */
shaka.net.HttpFetchPlugin.fetch_ = window.fetch;


/**
 * Overridden in unit tests, but compiled out in production.
 *
 * @const {function(new: AbortController)}
 * @private
 */
shaka.net.HttpFetchPlugin.AbortController_ = window.AbortController;


/**
 * Overridden in unit tests, but compiled out in production.
 *
 * @const {function(new: Headers)}
 * @private
 */
shaka.net.HttpFetchPlugin.Headers_ = window.Headers;


if (shaka.net.HttpFetchPlugin.isSupported()) {
  shaka.net.NetworkingEngine.registerScheme(
      'http', shaka.net.HttpFetchPlugin.parse,
      shaka.net.NetworkingEngine.PluginPriority.PREFERRED,
      /* progressSupport= */ true);
  shaka.net.NetworkingEngine.registerScheme(
      'https', shaka.net.HttpFetchPlugin.parse,
      shaka.net.NetworkingEngine.PluginPriority.PREFERRED,
      /* progressSupport= */ true);
  shaka.net.NetworkingEngine.registerScheme(
      'blob', shaka.net.HttpFetchPlugin.parse,
      shaka.net.NetworkingEngine.PluginPriority.PREFERRED,
      /* progressSupport= */ true);
}