/*! @license
* Shaka Player
* Copyright 2016 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
goog.provide('shaka.cea.Mp4CeaParser');
goog.require('goog.asserts');
goog.require('shaka.cea.CeaUtils');
goog.require('shaka.cea.SeiProcessor');
goog.require('shaka.log');
goog.require('shaka.media.ClosedCaptionParser');
goog.require('shaka.util.DataViewReader');
goog.require('shaka.util.Error');
goog.require('shaka.util.Mp4Parser');
goog.require('shaka.util.Mp4BoxParsers');
/**
* MPEG4 stream parser used for extracting 708 closed captions data.
* @implements {shaka.extern.ICeaParser}
* @export
*/
shaka.cea.Mp4CeaParser = class {
constructor() {
/**
* SEI data processor.
* @private
* @const {!shaka.cea.SeiProcessor}
*/
this.seiProcessor_ = new shaka.cea.SeiProcessor();
/**
* Map of track id to corresponding timescale.
* @private {!Map<number, number>}
*/
this.trackIdToTimescale_ = new Map();
/**
* Default sample duration, as specified by the TREX box.
* @private {number}
*/
this.defaultSampleDuration_ = 0;
/**
* Default sample size, as specified by the TREX box.
* @private {number}
*/
this.defaultSampleSize_ = 0;
/**
* @private {?number}
*/
this.c608TrackId_ = null;
/**
* @private {shaka.cea.Mp4CeaParser.BitstreamFormat}
*/
this.bitstreamFormat_ = shaka.cea.Mp4CeaParser.BitstreamFormat.UNKNOWN;
}
/**
* Parses the init segment. Gets Default Sample Duration and Size from the
* TREX box, and constructs a map of Track IDs to timescales. Each TRAK box
* contains a track header (TKHD) containing track ID, and a media header box
* (MDHD) containing the timescale for the track
* @override
*/
init(initSegment) {
const Mp4Parser = shaka.util.Mp4Parser;
const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
const trackIds = [];
const timescales = [];
let currentTrackId = null;
const codecBoxParser = (box) => this.setBitstreamFormat_(box.name);
new Mp4Parser()
.boxes([...Mp4Parser.SAMPLE_TABLE_PATH, 'mvex'], Mp4Parser.children)
.fullBox('trex', (box) => {
const parsedTREXBox = shaka.util.Mp4BoxParsers.parseTREX(
box.reader);
this.defaultSampleDuration_ = parsedTREXBox.defaultSampleDuration;
this.defaultSampleSize_ = parsedTREXBox.defaultSampleSize;
})
.fullBox('tkhd', (box) => {
goog.asserts.assert(
box.version != null,
'TKHD is a full box and should have a valid version.');
const parsedTKHDBox = shaka.util.Mp4BoxParsers.parseTKHD(
box.reader, box.version);
currentTrackId = parsedTKHDBox.trackId;
trackIds.push(parsedTKHDBox.trackId);
})
.fullBox('mdhd', (box) => {
goog.asserts.assert(
box.version != null,
'MDHD is a full box and should have a valid version.');
const parsedMDHDBox = shaka.util.Mp4BoxParsers.parseMDHD(
box.reader, box.version);
timescales.push(parsedMDHDBox.timescale);
})
.fullBox('stsd', Mp4Parser.sampleDescription)
// These are the various boxes that signal a codec.
.boxes(Mp4Parser.AVC_HEVC_VVC_DV, codecBoxParser)
// This signals an encrypted sample, which we can go inside of to find
// the codec used.
.box('encv', Mp4Parser.visualSampleEntry)
.box('sinf', Mp4Parser.children)
.box('frma', (box) => {
const {codec} = shaka.util.Mp4BoxParsers.parseFRMA(box.reader);
this.setBitstreamFormat_(codec);
})
// Raw CEA-608 (Apple format)
.box('c608', (box) => {
this.bitstreamFormat_ =
shaka.cea.Mp4CeaParser.BitstreamFormat.C608_TRACK;
this.c608TrackId_ = currentTrackId;
})
.parse(initSegment, /* partialOkay= */ true, /* stopOnPartial= */ true);
// At least one track should exist, and each track should have a
// corresponding Id in TKHD box, and timescale in its MDHD box
if (!trackIds.length|| !timescales.length ||
trackIds.length != timescales.length) {
throw new shaka.util.Error(
shaka.util.Error.Severity.CRITICAL,
shaka.util.Error.Category.TEXT,
shaka.util.Error.Code.INVALID_MP4_CEA);
}
if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
shaka.log.alwaysWarn(
'Unable to determine bitstream format for CEA parsing!');
}
// Populate the map from track Id to timescale
trackIds.forEach((trackId, idx) => {
this.trackIdToTimescale_.set(trackId, timescales[idx]);
});
}
/**
* Parses each video segment. In fragmented MP4s, MOOF and MDAT come in
* pairs. The following logic gets the necessary info from MOOFs to parse
* MDATs (base media decode time, sample sizes/offsets/durations, etc),
* and then parses the MDAT boxes for CEA-708 packets using this information.
* CEA-708 packets are returned in the callback.
* @override
*/
parse(mediaSegment) {
const Mp4Parser = shaka.util.Mp4Parser;
const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
if (this.bitstreamFormat_ == BitstreamFormat.UNKNOWN) {
// We don't know how to extract SEI from this.
return [];
}
/** @type {!Array<!shaka.extern.ICeaParser.CaptionPacket>} **/
const captionPackets = [];
let moofOffset = 0;
/** @type {!Array<!shaka.cea.Mp4CeaParser.ParsedTRAF>} */
let parsedTRAFs = [];
new Mp4Parser()
.box('moof', (box) => {
moofOffset = box.start;
// traf box parsing is reset on each moof.
parsedTRAFs = [];
Mp4Parser.children(box);
})
.box('traf', (box) => {
parsedTRAFs.push({
trackId: null,
baseMediaDecodeTime: null,
defaultSampleDuration: this.defaultSampleDuration_,
defaultSampleSize: this.defaultSampleSize_,
parsedTRUNs: [],
timescale: shaka.cea.CeaUtils.DEFAULT_TIMESCALE_VALUE,
});
Mp4Parser.children(box);
})
.fullBox('trun', (box) => {
goog.asserts.assert(
box.version != null && box.flags != null,
'TRUN is a full box and should have a valid version & flags.');
const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
const parsedTRUN = shaka.util.Mp4BoxParsers.parseTRUN(
box.reader, box.version, box.flags);
lastTRAF.parsedTRUNs.push(parsedTRUN);
})
.fullBox('tfhd', (box) => {
goog.asserts.assert(
box.flags != null,
'TFHD is a full box and should have valid flags.');
const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
const parsedTFHD = shaka.util.Mp4BoxParsers.parseTFHD(
box.reader, box.flags);
lastTRAF.trackId = parsedTFHD.trackId;
// If specified, defaultSampleDuration and defaultSampleSize
// override the ones specified in the TREX box
lastTRAF.defaultSampleDuration = parsedTFHD.defaultSampleDuration ||
this.defaultSampleDuration_;
lastTRAF.defaultSampleSize = parsedTFHD.defaultSampleSize ||
this.defaultSampleSize_;
const trackId = parsedTFHD.trackId;
// Get the timescale from the track Id
if (this.trackIdToTimescale_.has(trackId)) {
lastTRAF.timescale = this.trackIdToTimescale_.get(trackId);
}
})
.fullBox('tfdt', (box) => {
goog.asserts.assert(
box.version != null,
'TFDT is a full box and should have a valid version.');
const lastTRAF = parsedTRAFs[parsedTRAFs.length - 1];
const parsedTFDT = shaka.util.Mp4BoxParsers.parseTFDTInaccurate(
box.reader, box.version);
lastTRAF.baseMediaDecodeTime = parsedTFDT.baseMediaDecodeTime;
})
.box('mdat', (box) => {
const offset = moofOffset - box.start - 8;
const initialPosition = box.reader.getPosition();
for (const parsedTRAF of parsedTRAFs) {
if (this.bitstreamFormat_ == BitstreamFormat.C608_TRACK &&
parsedTRAF.trackId !== this.c608TrackId_) {
continue;
}
if (parsedTRAF.baseMediaDecodeTime === null) {
// This field should have been populated by the Base Media Decode
// Time in the tfdt box.
shaka.log.alwaysWarn(
'Unable to find base media decode time for CEA captions!');
continue;
}
box.reader.seek(initialPosition);
if (this.bitstreamFormat_ === BitstreamFormat.C608_TRACK) {
this.parseC608Mdat_(box.reader,
parsedTRAF.baseMediaDecodeTime,
parsedTRAF.timescale,
parsedTRAF.defaultSampleDuration,
parsedTRAF.defaultSampleSize,
offset,
parsedTRAF.parsedTRUNs,
captionPackets);
} else {
this.parseMdat_(box.reader,
parsedTRAF.baseMediaDecodeTime,
parsedTRAF.timescale,
parsedTRAF.defaultSampleDuration,
parsedTRAF.defaultSampleSize,
offset,
parsedTRAF.parsedTRUNs,
captionPackets);
}
}
})
.parse(mediaSegment, /* partialOkay= */ false,
/* stopOnPartial= */ true);
return captionPackets;
}
/**
* Parse MDAT box.
* @param {!shaka.util.DataViewReader} reader
* @param {number} time
* @param {number} timescale
* @param {number} defaultSampleDuration
* @param {number} defaultSampleSize
* @param {number} offset
* @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
* @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
* @private
*/
parseMdat_(reader, time, timescale, defaultSampleDuration,
defaultSampleSize, offset, parsedTRUNs, captionPackets) {
const BitstreamFormat = shaka.cea.Mp4CeaParser.BitstreamFormat;
const CeaUtils = shaka.cea.CeaUtils;
goog.asserts.assert(this.bitstreamFormat_ != BitstreamFormat.UNKNOWN,
'Bitstream format should have been checked before now!');
// We can't assume the samples described across multiple trun boxes are
// contiguous in the mdat. A track's runs may be interleaved with another
// track's data in a shared mdat, so each run is read starting from its own
// data offset, and we only walk the NAL units that belong to each sample
// (bounded by the sample size). Reading past a run would pick up bytes from
// the other track and misinterpret them as NAL units.
let currentTime = time;
for (const parsedTRUN of parsedTRUNs) {
if (parsedTRUN.dataOffset != null) {
reader.seek(offset + parsedTRUN.dataOffset);
}
for (const sample of parsedTRUN.sampleData) {
// The fields in each ParsedTRUNSample are nullable. For sample size and
// duration we use the defaults provided by the TREX/TFHD boxes, and for
// the composition time offset we default to 0.
const timeOffset = sample.sampleCompositionTimeOffset || 0;
const pts = (currentTime + timeOffset) / timescale;
let sampleSize = sample.sampleSize || defaultSampleSize;
while (sampleSize > 0 && reader.hasMoreData()) {
const naluSize = reader.readUint32();
const naluHeader = reader.readUint8();
let naluType = null;
let isSeiMessage = false;
let naluHeaderSize = 1;
switch (this.bitstreamFormat_) {
case BitstreamFormat.H264:
naluType = naluHeader & 0x1f;
isSeiMessage =
CeaUtils.isSeiNaluType(CeaUtils.CodecFamily.H264, naluType);
break;
// H.265 and H.266 share the same NAL unit header parsing and only
// differ in which codec family's SEI types to match against.
case BitstreamFormat.H265:
case BitstreamFormat.H266: {
naluHeaderSize = 2;
reader.skip(1);
naluType = (naluHeader >> 1) & 0x3f;
const family = this.bitstreamFormat_ === BitstreamFormat.H265 ?
CeaUtils.CodecFamily.H265 : CeaUtils.CodecFamily.H266;
isSeiMessage = CeaUtils.isSeiNaluType(family, naluType);
break;
}
default:
return;
}
if (isSeiMessage) {
const nalu = reader.readBytes(
naluSize - naluHeaderSize,
// Don't clone. The nalu is temporary and is not stored.
/* clone= */ false);
for (const packet of this.seiProcessor_.process(nalu)) {
captionPackets.push({
packet,
pts,
format: shaka.media.ClosedCaptionParser.CaptionPacketFormat.SEI,
});
}
} else {
try {
reader.skip(naluSize - naluHeaderSize);
} catch (e) {
// It is necessary to ignore this error because it can break the
// start of playback even if the user does not want to see the
// subtitles.
return;
}
}
sampleSize -= (naluSize + 4);
}
currentTime += sample.sampleDuration || defaultSampleDuration;
}
}
}
/**
* Parse MDAT box for C608 format
* @param {!shaka.util.DataViewReader} reader
* @param {number} time
* @param {number} timescale
* @param {number} defaultSampleDuration
* @param {number} defaultSampleSize
* @param {number} offset
* @param {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
* @param {!Array<!shaka.extern.ICeaParser.CaptionPacket>} captionPackets
* @private
*/
parseC608Mdat_(reader, time, timescale, defaultSampleDuration,
defaultSampleSize, offset, parsedTRUNs, captionPackets) {
let currentTime = time;
// We can't assume the samples described across multiple trun boxes are
// contiguous in the mdat. A track's runs may be interleaved with another
// track's data in a shared mdat (e.g. Apple HLS interleaves the CEA-608
// 'c608' track's runs between the video runs), so each run must be read
// starting from its own data offset. Reading them contiguously would pick
// up bytes belonging to the other track, which can pass the CEA-608 parity
// check and inject garbage characters into the captions.
for (const parsedTRUN of parsedTRUNs) {
if (parsedTRUN.dataOffset != null) {
reader.seek(offset + parsedTRUN.dataOffset);
}
for (const sample of parsedTRUN.sampleData) {
const size = sample.sampleSize || defaultSampleSize;
const pts = currentTime / timescale;
const bytes = reader.readBytes(size, /* clone= */ false);
captionPackets.push({
packet: bytes,
pts,
format: shaka.media.ClosedCaptionParser.CaptionPacketFormat.RAW608,
});
currentTime += sample.sampleDuration || defaultSampleDuration;
}
}
}
/**
* @param {string} codec A fourcc for a codec.
* @private
*/
setBitstreamFormat_(codec) {
if (shaka.cea.Mp4CeaParser.CodecBitstreamMap_.has(codec)) {
this.bitstreamFormat_ =
shaka.cea.Mp4CeaParser.CodecBitstreamMap_.get(codec);
}
}
};
/** @enum {number} */
shaka.cea.Mp4CeaParser.BitstreamFormat = {
UNKNOWN: 0,
H264: 1,
H265: 2,
H266: 3,
C608_TRACK: 4,
};
/** @private {Map<string, shaka.cea.Mp4CeaParser.BitstreamFormat>} */
shaka.cea.Mp4CeaParser.CodecBitstreamMap_ = new Map()
// AVC
.set('avc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
.set('avc3', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
// Dolby Vision based in AVC
.set('dvav', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
.set('dva1', shaka.cea.Mp4CeaParser.BitstreamFormat.H264)
// HEVC
.set('hev1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
.set('hvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
// Dolby Vision based in HEVC
.set('dvh1', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
.set('dvhe', shaka.cea.Mp4CeaParser.BitstreamFormat.H265)
// VVC
.set('vvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
.set('vvi1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
// Dolby Vision based in VVC
.set('dvc1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266)
.set('dvi1', shaka.cea.Mp4CeaParser.BitstreamFormat.H266);
/**
* @typedef {{
* trackId: ?number,
* baseMediaDecodeTime: ?number,
* defaultSampleDuration: number,
* defaultSampleSize: number,
* parsedTRUNs: !Array<shaka.util.ParsedTRUNBox>,
* timescale: number,
* }}
*
* @property {?number} trackId
* @property {?number} baseMediaDecodeTime
* @property {number} defaultSampleDuration
* @property {number} defaultSampleSize
* @property {!Array<shaka.util.ParsedTRUNBox>} parsedTRUNs
* @property {?number} timescale
*/
shaka.cea.Mp4CeaParser.ParsedTRAF;
shaka.media.ClosedCaptionParser.registerParser('video/mp4',
() => new shaka.cea.Mp4CeaParser());