/*! @license
* Shaka Player
* Copyright 2016 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
goog.provide('shaka.cea.Mp4CeaParser');
goog.require('goog.asserts');
goog.require('shaka.cea.CeaUtils');
goog.require('shaka.cea.SeiProcessor');
goog.require('shaka.log');
goog.require('shaka.media.ClosedCaptionParser');
goog.require('shaka.util.DataViewReader');
goog.require('shaka.util.Error');
goog.require('shaka.util.Mp4Parser');
goog.require('shaka.util.Mp4BoxParsers');
/**
* MPEG4 stream parser used for extracting 708 closed captions data.
* @implements {shaka.extern.ICeaParser}
* @export
*/
shaka.cea.Mp4CeaParser = class {
constructor() {
/**
* SEI data processor.
* @private
* @const {!shaka.cea.SeiProcessor}
*/
this.seiProcessor_ = new shaka.cea.SeiProcessor();
/**
* Map of track id to corresponding timescale.
* @private {!Map<number, number>}
*/
this.trackIdToTimescale_ = new Map();
/**
* Default sample duration, as specified by the TREX box.
* @private {number}
*/
this.defaultSampleDuration_ = 0;
/**
* Default sample size, as specified by the TREX box.
* @private {number}
*/
this.defaultSampleSize_ = 0;
/**
* @private {?number}
*/
this.c608TrackId_ = null;
/**
* @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
*/
this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
}
/**
* Parses the init segment. Gets Default Sample Duration and Size from the
* TREX box, and constructs a map of Track IDs to timescales. Each TRAK box
* contains a track header (TKHD) containing track ID, and a media header box
* (MDHD) containing the timescale for the track
* @override
*/
init(initSegment) {
const Mp4Parser = shaka.util.Mp4Parser;
const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
const trackIds = [];
const timescales = [];
let currentTrackId = null;
const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
new Mp4Parser()
.boxes([...Mp4Parser.SAMPLE_TABLE_PATH, 'mvex'], Mp4Parser.children)
.fullBox('trex', (box) => {
const parsedTREXBox = shaka.util.Mp4BoxParsers.parseTREX(
box.reader);
this.defaultSampleDuration_ = parsedTREXBox.defaultSampleDuration;
this.defaultSampleSize_ = parsedTREXBox.defaultSampleSize;
})
.fullBox('tkhd', (box) => {
goog.asserts.assert(
box.version != null,
'TKHD is a full box and should have a valid version.');
const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
box.reader, box.version);
currentTrackId = parsedTKHDBox.trackId;
trackIds.push(parsedTKHDBox.trackId);
})
.fullBox('mdhd', (box) => {
goog.asserts.assert(
box.version != null,
'MDHD is a full box and should have a valid version.');
const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
box.reader, box.version);
timescales.push(parsedMDHDBox.timescale);
})
.fullBox('stsd', Mp4Parser.sampleDescription)
// These are the various boxes that signal a codec.
.boxes(Mp4Parser.AVC_HEVC_VVC_DV, codecBoxParser)
.boxes(Mp4Parser.AV1_DV, codecBoxParser)
// This signals an encrypted sample, which we can go inside of to find
// the codec used.
.box('encv', Mp4Parser.visualSampleEntry)
.box('sinf', Mp4Parser.children)
.box('frma', (box) => {
const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
this.setBitstreamFormat_(codec);
})
// Raw CEA-608 (Apple format)
.box('c608', (box) => {
this.bitstreamFormat_ =
shaka.cea.Mp4CeaParser.BitstreamFormat.C608_TRACK;
this.c608TrackId_ = currentTrackId;
})
.parse(initSegment, /* partialOkay= */ true, /* stopOnPartial= */ true);
// At least one track should exist, and each track should have a
// corresponding Id in TKHD box, and timescale in its MDHD box
if (!trackIds.length|| !timescales.length ||
trackIds.length != timescales.length) {
throw new shaka.util.Error(
shaka.util.Error.Severity.CRITICAL,
shaka.util.Error.Category.TEXT,
shaka.util.Error.Code.INVALID_MP4_CEA);
}
if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
shaka.log.alwaysWarn(
'Unable to determine bitstream format for CEA parsing!');
}
// Populate the map from track Id to timescale
trackIds.forEach((trackId, idx) => {
this.trackIdToTimescale_.set(trackId, timescales[idx]);
});
}
/**
* Parses each video segment. In fragmented MP4s, MOOF and MDAT come in
* pairs. The following logic gets the necessary info from MOOFs to parse
* MDATs (base media decode time, sample sizes/offsets/durations, etc),
* and then parses the MDAT boxes for CEA-708 packets using this information.
* CEA-708 packets are returned in the callback.
* @override
*/
parse(mediaSegment) {
const Mp4Parser = shaka.util.Mp4Parser;
const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
// We don't know how to extract SEI from this.
return [];
}
/** @type {!Array<!shaka.extern.ICeaParser.CaptionPacket>} **/
const captionPackets = [];
let moofOffset = 0;
/** @type {!Array<!shaka.cea.Mp4CeaParser.ParsedTRAF>} */
let parsedTRAFs = [];
new Mp4Parser()
.box('moof', (box) => {
moofOffset = box.start;
// traf box parsing is reset on each moof.
parsedTRAFs = [];
Mp4Parser.children(box);
})
.box('traf', (box) => {
parsedTRAFs.push({
trackId: null,
baseMediaDecodeTime: null,
defaultSampleDuration: this.defaultSampleDuration_,
defaultSampleSize: this.defaultSampleSize_,
parsedTRUNs: [],
timescale: shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE,
});
Mp4Parser.children(box);
})
.fullBox('trun', (box) => {
goog.asserts.assert(
box.version != null && box.flags != null,
'TRUN is a full box and should have a valid version & flags.');
const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
box.reader, box.version, box.flags);
lastTRAF.parsedTRUNs.push(parsedTRUN);
})
.fullBox('tfhd', (box) => {
goog.asserts.assert(
box.flags != null,
'TFHD is a full box and should have valid flags.');
const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
const parsedTFHD = shaka.util.Mp4BoxParsers.parseTFHD(
box.reader, box.flags);
lastTRAF.trackId = parsedTFHD.trackId;
// If specified, defaultSampleDuration and defaultSampleSize
// override the ones specified in the TREX box
lastTRAF.defaultSampleDuration = parsedTFHD.defaultSampleDuration ||
this.defaultSampleDuration_;
lastTRAF.defaultSampleSize = parsedTFHD.defaultSampleSize ||
this.defaultSampleSize_;
const trackId = parsedTFHD.trackId;
// Get the timescale from the track Id
if (this.trackIdToTimescale_.has(trackId)) {
lastTRAF.timescale = this.trackIdToTimescale_.get(trackId);
}
})
.fullBox('tfdt', (box) => {
goog.asserts.assert(
box.version != null,
'TFDT is a full box and should have a valid version.');
const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
const parsedTFDT = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
box.reader, box.version);
lastTRAF.baseMediaDecodeTime = parsedTFDT.baseMediaDecodeTime;
})
.box('mdat', (box) => {
const offset = moofOffset - box.start - 8;
const initialPosition = box.reader.getPosition();
for (const parsedTRAF of parsedTRAFs) {
if (this.bitstreamFormat_ == BitstreamFormat.C608_TRACK &&
parsedTRAF.trackId !== this.c608TrackId_) {
continue;
}
if (parsedTRAF.baseMediaDecodeTime === null) {
// This field should have been populated by the Base Media Decode
// Time in the tfdt box.
shaka.log.alwaysWarn(
'Unable to find base media decode time for CEA captions!');
continue;
}
box.reader.seek(initialPosition);
if (this.bitstreamFormat_ === BitstreamFormat.C608_TRACK) {
this.parseC608Mdat_(box.reader,
parsedTRAF.baseMediaDecodeTime,
parsedTRAF.timescale,
parsedTRAF.defaultSampleDuration,
parsedTRAF.defaultSampleSize,
offset,
parsedTRAF.parsedTRUNs,
captionPackets);
} else {
this.parseMdat_(box.reader,
parsedTRAF.baseMediaDecodeTime,
parsedTRAF.timescale,
parsedTRAF.defaultSampleDuration,
parsedTRAF.defaultSampleSize,
offset,
parsedTRAF.parsedTRUNs,
captionPackets);
}
}
})
.parse(mediaSegment, /* partialOkay= */ false,
/* stopOnPartial= */ true);
return captionPackets;
}
/**
* Parse MDAT box.
* @param {!shaka.util.DataViewReader} reader
* @param {number} time
* @param {number} timescale
* @param {number} defaultSampleDuration
* @param {number} defaultSampleSize
* @param {number} offset
* @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
* @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
* @private
*/
parseMdat_(reader, time, timescale, defaultSampleDuration,
defaultSampleSize, offset, parsedTRUNs, captionPackets) {
const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
const CeaUtils = shaka.cea.CeaUtils;
goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
'Bitstream format should have been checked before now!');
// We can't assume the samples described across multiple trun boxes are
// contiguous in the mdat. A track's runs may be interleaved with another
// track's data in a shared mdat, so each run is read starting from its own
// data offset, and we only walk the NAL units that belong to each sample
// (bounded by the sample size). Reading past a run would pick up bytes from
// the other track and misinterpret them as NAL units.
let currentTime = time;
for (const parsedTRUN of parsedTRUNs) {
if (parsedTRUN.dataOffset != null) {
reader.seek(offset + parsedTRUN.dataOffset);
}
for (const sample of parsedTRUN.sampleData) {
// The fields in each ParsedTRUNSample are nullable. For sample size and
// duration we use the defaults provided by the TREX/TFHD boxes, and for
// the composition time offset we default to 0.
const timeOffset = sample.sampleCompositionTimeOffset || 0;
const pts = (currentTime + timeOffset) / timescale;
let sampleSize = sample.sampleSize || defaultSampleSize;
// AV1 has no NAL units: one sample is a bare sequence of OBUs, so it
// is read whole and walked separately.
if (this.bitstreamFormat_ == BitstreamFormat.AV1) {
try {
const sampleData = reader.readBytes(
sampleSize,
// Don't clone. The sample is temporary and is not stored.
/* clone= */ false);
this.parseAv1Sample_(sampleData, pts, captionPackets);
} catch (e) {
// It is necessary to ignore this error because it can break the
// start of playback even if the user does not want to see the
// subtitles.
return;
}
currentTime += sample.sampleDuration || defaultSampleDuration;
continue;
}
while (sampleSize > 0 && reader.hasMoreData()) {
const naluSize = reader.readUint32();
const naluHeader = reader.readUint8();
let naluType = null;
let isSeiMessage = false;
let naluHeaderSize = 1;
switch (this.bitstreamFormat_) {
case BitstreamFormat.H264:
naluType = naluHeader & 0x1f;
isSeiMessage =
CeaUtils.isSeiNaluType(CeaUtils.CodecFamily.H264, naluType);
break;
// H.265 and H.266 share the same NAL unit header parsing and only
// differ in which codec family's SEI types to match against.
case BitstreamFormat.H265:
case BitstreamFormat.H266: {
naluHeaderSize = 2;
reader.skip(1);
naluType = (naluHeader >> 1) & 0x3f;
const family = this.bitstreamFormat_ === BitstreamFormat.H265 ?
CeaUtils.CodecFamily.H265 : CeaUtils.CodecFamily.H266;
isSeiMessage = CeaUtils.isSeiNaluType(family, naluType);
break;
}
default:
return;
}
if (isSeiMessage) {
const nalu = reader.readBytes(
naluSize - naluHeaderSize,
// Don't clone. The nalu is temporary and is not stored.
/* clone= */ false);
for (const packet of this.seiProcessor_.process(nalu)) {
captionPackets.push({
packet,
pts,
format: shaka.media.ClosedCaptionParser.CaptionPacketFormat.SEI,
});
}
} else {
try {
reader.skip(naluSize - naluHeaderSize);
} catch (e) {
// It is necessary to ignore this error because it can break the
// start of playback even if the user does not want to see the
// subtitles.
return;
}
}
sampleSize -= (naluSize + 4);
}
currentTime += sample.sampleDuration || defaultSampleDuration;
}
}
}
/**
* Parses the OBUs of a single AV1 coded sample (one temporal unit) and
* extracts the caption payload of every metadata_itu_t_t35 OBU.
*
* AV1 has no NAL units, so the cc_data() that AVC/HEVC carry in an SEI
* message rides in a metadata OBU placed before the frame OBU instead. The
* T.35/GA94 header and the cc_data() below it are identical to the SEI case,
* so only the envelope differs and the extracted payload is handed to the
* decoder as an SEI packet. OBUs carry no emulation prevention bytes, which
* is why the payload does not go through the SEI processor.
*
* @param {!Uint8Array} sample
* @param {number} pts
* @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
* @private
*/
parseAv1Sample_(sample, pts, captionPackets) {
const Mp4CeaParser = shaka.cea.Mp4CeaParser;
let offset = 0;
while (offset < sample.length) {
// obu_header(): 1 forbidden bit, 4 bits obu_type, obu_extension_flag,
// obu_has_size_field, and 1 reserved bit.
const obuHeader = sample[offset++];
const obuType = (obuHeader >> 3) & 0x0f;
const obuExtensionFlag = (obuHeader >> 2) & 0x01;
const obuHasSizeField = (obuHeader >> 1) & 0x01;
if (obuExtensionFlag) {
// obu_extension_header() is a single byte we have no use for.
offset++;
}
let obuEnd;
if (obuHasSizeField) {
const obuSize = Mp4CeaParser.readLeb128_(sample, offset);
if (!obuSize) {
return;
}
offset += obuSize.size;
obuEnd = offset + obuSize.value;
} else {
// Without a size field the OBU extends to the end of the sample, so it
// is necessarily the last one.
obuEnd = sample.length;
}
if (obuEnd > sample.length) {
// Malformed sample; anything past this point would be misread.
return;
}
if (obuType == Mp4CeaParser.OBU_TYPE_METADATA_) {
const metadataType = Mp4CeaParser.readLeb128_(sample, offset);
if (metadataType &&
metadataType.value == Mp4CeaParser.METADATA_TYPE_ITUT_T35_) {
// What follows is itu_t_t35(), the same payload an SEI message of
// type user_data_registered_itu_t_t35 carries. The trailing bits
// byte at the end of the OBU is harmless: the decoder reads the
// caption constructs by count.
captionPackets.push({
packet: sample.subarray(offset + metadataType.size, obuEnd),
pts,
format: shaka.media.ClosedCaptionParser.CaptionPacketFormat.SEI,
});
}
}
offset = obuEnd;
}
}
/**
* Reads the AV1 leb128() at the given offset. Returns null if the value is
* truncated or malformed, in which case the rest of the sample cannot be
* trusted either.
*
* @param {!Uint8Array} data
* @param {number} offset
* @return {?{value: number, size: number}}
* @private
*/
static readLeb128_(data, offset) {
let value = 0;
// A leb128() is at most 8 bytes, each holding 7 bits of the value plus a
// continuation bit. Multiplication is used instead of a shift because the
// value can exceed 32 bits.
for (let i = 0; i < 8; i++) {
if (offset + i >= data.length) {
return null;
}
const byte = data[offset + i];
value += (byte & 0x7f) * Math.pow(2, 7 * i);
if (!(byte & 0x80)) {
return {value, size: i + 1};
}
}
return null;
}
/**
* Parse MDAT box for C608 format
* @param {!shaka.util.DataViewReader} reader
* @param {number} time
* @param {number} timescale
* @param {number} defaultSampleDuration
* @param {number} defaultSampleSize
* @param {number} offset
* @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
* @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
* @private
*/
parseC608Mdat_(reader, time, timescale, defaultSampleDuration,
defaultSampleSize, offset, parsedTRUNs, captionPackets) {
let currentTime = time;
// We can't assume the samples described across multiple trun boxes are
// contiguous in the mdat. A track's runs may be interleaved with another
// track's data in a shared mdat (e.g. Apple HLS interleaves the CEA-608
// 'c608' track's runs between the video runs), so each run must be read
// starting from its own data offset. Reading them contiguously would pick
// up bytes belonging to the other track, which can pass the CEA-608 parity
// check and inject garbage characters into the captions.
for (const parsedTRUN of parsedTRUNs) {
if (parsedTRUN.dataOffset != null) {
reader.seek(offset + parsedTRUN.dataOffset);
}
for (const sample of parsedTRUN.sampleData) {
const size = sample.sampleSize || defaultSampleSize;
const pts = currentTime / timescale;
const bytes = reader.readBytes(size, /* clone= */ false);
captionPackets.push({
packet: bytes,
pts,
format: shaka.media.ClosedCaptionParser.CaptionPacketFormat.RAW608,
});
currentTime += sample.sampleDuration || defaultSampleDuration;
}
}
}
/**
* @param {string} codec A fourcc for a codec.
* @private
*/
setBitstreamFormat_(codec) {
if (shaka.cea.Mp4CeaParser.CodecBitstreamMap_.has(codec)) {
this.bitstreamFormat_ =
shaka.cea.Mp4CeaParser.CodecBitstreamMap_.get(codec);
}
}
};
/** @enum {number} */
shaka.cea.Mp4CeaParser.BitstreamFormat = {
UNKNOWN: 0,
H264: 1,
H265: 2,
H266: 3,
C608_TRACK: 4,
AV1: 5,
};
/**
* OBU_METADATA, as defined by the AV1 bitstream specification.
* @const {number}
* @private
*/
shaka.cea.Mp4CeaParser.OBU_TYPE_METADATA_ = 5;
/**
* METADATA_TYPE_ITUT_T35, as defined by the AV1 bitstream specification.
* @const {number}
* @private
*/
shaka.cea.Mp4CeaParser.METADATA_TYPE_ITUT_T35_ = 4;
/** @private {Map<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = new Map()
// AVC
.set('avc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
.set('avc3', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
// Dolby Vision based in AVC
.set('dvav', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
.set('dva1', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
// HEVC
.set('hev1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
.set('hvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
// Dolby Vision based in HEVC
.set('dvh1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
.set('dvhe', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
// VVC
.set('vvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
.set('vvi1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
// Dolby Vision based in VVC
.set('dvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
.set('dvi1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
// AV1
.set('av01', shaka.cea.Mp4CeaParser.BitstreamFormat.AV1)
// Dolby Vision based in AV1
.set('dav1', shaka.cea.Mp4CeaParser.BitstreamFormat.AV1);
/**
* @typedef {{
* trackId: ?number,
* baseMediaDecodeTime: ?number,
* defaultSampleDuration: number,
* defaultSampleSize: number,
* parsedTRUNs: !Array<shaka.util.ParsedTRUNBox>,
* timescale: number,
* }}
*
* @property {?number} trackId
* @property {?number} baseMediaDecodeTime
* @property {number} defaultSampleDuration
* @property {number} defaultSampleSize
* @property {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
* @property {?number} timescale
*/
shaka.cea.Mp4CeaParser.ParsedTRAF;
shaka.media.ClosedCaptionParser.registerParser('video/mp4',
() => new shaka.cea.Mp4CeaParser());