Source: lib/transmuxer/h264.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.transmuxer.H264');
  9. goog.require('shaka.util.ExpGolomb');
  10. goog.require('shaka.util.Uint8ArrayUtils');
  13. /**
  14.  * H.264 utils
  15.  */
  16. shaka.transmuxer.H264 = class {
  17.   /**
  18.    * Read a sequence parameter set and return some interesting video
  19.    * properties. A sequence parameter set is the H264 metadata that
  20.    * describes the properties of upcoming video frames.
  21.    *
  22.    * @param {!Array.<shaka.extern.VideoNalu>} nalus
  23.    * @return {?{height: number, width: number, videoConfig: !Uint8Array,
  24.    *            hSpacing: number, vSpacing: number}}
  25.    */
  26.   static parseInfo(nalus) {
  27.     const H264 = shaka.transmuxer.H264;
  28.     if (!nalus.length) {
  29.       return null;
  30.     }
  31.     const spsNalu = nalus.find((nalu) => {
  32.       return nalu.type == H264.NALU_TYPE_SPS_;
  33.     });
  34.     const ppsNalu = nalus.find((nalu) => {
  35.       return nalu.type == H264.NALU_TYPE_PPS_;
  36.     });
  37.     if (!spsNalu || !ppsNalu) {
  38.       return null;
  39.     }
  41.     const expGolombDecoder = new shaka.util.ExpGolomb(spsNalu.data);
  42.     // profile_idc
  43.     const profileIdc = expGolombDecoder.readUnsignedByte();
  44.     // constraint_set[0-5]_flag
  45.     expGolombDecoder.readUnsignedByte();
  46.     // level_idc u(8)
  47.     expGolombDecoder.readUnsignedByte();
  48.     // seq_parameter_set_id
  49.     expGolombDecoder.skipExpGolomb();
  51.     // some profiles have more optional data we don't need
  52.     if (H264.PROFILES_WITH_OPTIONAL_SPS_DATA_.includes(profileIdc)) {
  53.       const chromaFormatIdc = expGolombDecoder.readUnsignedExpGolomb();
  54.       if (chromaFormatIdc === 3) {
  55.         // separate_colour_plane_flag
  56.         expGolombDecoder.skipBits(1);
  57.       }
  58.       // bit_depth_luma_minus8
  59.       expGolombDecoder.skipExpGolomb();
  60.       // bit_depth_chroma_minus8
  61.       expGolombDecoder.skipExpGolomb();
  62.       // qpprime_y_zero_transform_bypass_flag
  63.       expGolombDecoder.skipBits(1);
  64.       // seq_scaling_matrix_present_flag
  65.       if (expGolombDecoder.readBoolean()) {
  66.         const scalingListCount = (chromaFormatIdc !== 3) ? 8 : 12;
  67.         for (let i = 0; i < scalingListCount; i++) {
  68.           // seq_scaling_list_present_flag[ i ]
  69.           if (expGolombDecoder.readBoolean()) {
  70.             if (i < 6) {
  71.               expGolombDecoder.skipScalingList(16);
  72.             } else {
  73.               expGolombDecoder.skipScalingList(64);
  74.             }
  75.           }
  76.         }
  77.       }
  78.     }
  80.     // log2_max_frame_num_minus4
  81.     expGolombDecoder.skipExpGolomb();
  82.     const picOrderCntType = expGolombDecoder.readUnsignedExpGolomb();
  84.     if (picOrderCntType === 0) {
  85.       // log2_max_pic_order_cnt_lsb_minus4
  86.       expGolombDecoder.readUnsignedExpGolomb();
  87.     } else if (picOrderCntType === 1) {
  88.       // delta_pic_order_always_zero_flag
  89.       expGolombDecoder.skipBits(1);
  90.       // offset_for_non_ref_pic
  91.       expGolombDecoder.skipExpGolomb();
  92.       // offset_for_top_to_bottom_field
  93.       expGolombDecoder.skipExpGolomb();
  94.       const numRefFramesInPicOrderCntCycle =
  95.           expGolombDecoder.readUnsignedExpGolomb();
  96.       for (let i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
  97.         // offset_for_ref_frame[ i ]
  98.         expGolombDecoder.skipExpGolomb();
  99.       }
  100.     }
  102.     // max_num_ref_frames
  103.     expGolombDecoder.skipExpGolomb();
  104.     // gaps_in_frame_num_value_allowed_flag
  105.     expGolombDecoder.skipBits(1);
  107.     const picWidthInMbsMinus1 =
  108.         expGolombDecoder.readUnsignedExpGolomb();
  109.     const picHeightInMapUnitsMinus1 =
  110.         expGolombDecoder.readUnsignedExpGolomb();
  112.     const frameMbsOnlyFlag = expGolombDecoder.readBits(1);
  113.     if (frameMbsOnlyFlag === 0) {
  114.       // mb_adaptive_frame_field_flag
  115.       expGolombDecoder.skipBits(1);
  116.     }
  117.     // direct_8x8_inference_flag
  118.     expGolombDecoder.skipBits(1);
  120.     let frameCropLeftOffset = 0;
  121.     let frameCropRightOffset = 0;
  122.     let frameCropTopOffset = 0;
  123.     let frameCropBottomOffset = 0;
  125.     // frame_cropping_flag
  126.     if (expGolombDecoder.readBoolean()) {
  127.       frameCropLeftOffset = expGolombDecoder.readUnsignedExpGolomb();
  128.       frameCropRightOffset = expGolombDecoder.readUnsignedExpGolomb();
  129.       frameCropTopOffset = expGolombDecoder.readUnsignedExpGolomb();
  130.       frameCropBottomOffset = expGolombDecoder.readUnsignedExpGolomb();
  131.     }
  133.     let hSpacing = 1;
  134.     let vSpacing = 1;
  136.     // vui_parameters_present_flag
  137.     if (expGolombDecoder.readBoolean()) {
  138.       // aspect_ratio_info_present_flag
  139.       if (expGolombDecoder.readBoolean()) {
  140.         const aspectRatioIdc = expGolombDecoder.readUnsignedByte();
  141.         const hSpacingTable = [
  142.           1, 12, 10, 16, 40, 24, 20, 32, 80, 18, 15, 64, 160, 4, 3, 2,
  143.         ];
  144.         const vSpacingTable = [
  145.           1, 11, 11, 11, 33, 11, 11, 11, 33, 11, 11, 33, 99, 3, 2, 1,
  146.         ];
  147.         if (aspectRatioIdc > 0 && aspectRatioIdc <= 16) {
  148.           hSpacing = hSpacingTable[aspectRatioIdc - 1];
  149.           vSpacing = vSpacingTable[aspectRatioIdc - 1];
  150.         } else if (aspectRatioIdc === 255) {
  151.           hSpacing = expGolombDecoder.readBits(16);
  152.           vSpacing = expGolombDecoder.readBits(16);
  153.         }
  154.       }
  155.     }
  157.     const height = ((2 - frameMbsOnlyFlag) *
  158.         (picHeightInMapUnitsMinus1 + 1) * 16) - (frameCropTopOffset * 2) -
  159.         (frameCropBottomOffset * 2);
  160.     const width = ((picWidthInMbsMinus1 + 1) * 16) -
  161.         frameCropLeftOffset * 2 - frameCropRightOffset * 2;
  163.     // assemble the SPSs
  164.     let sps = [];
  165.     const spsData = spsNalu.fullData;
  166.     sps.push((spsData.byteLength >>> 8) & 0xff);
  167.     sps.push(spsData.byteLength & 0xff);
  168.     sps = sps.concat(...spsData);
  170.     // assemble the PPSs
  171.     let pps = [];
  172.     const ppsData = ppsNalu.fullData;
  173.     pps.push((ppsData.byteLength >>> 8) & 0xff);
  174.     pps.push(ppsData.byteLength & 0xff);
  175.     pps = pps.concat(...ppsData);
  177.     const videoConfig = new Uint8Array(
  178.         [
  179.           0x01, // version
  180.           sps[3], // profile
  181.           sps[4], // profile compat
  182.           sps[5], // level
  183.           0xfc | 3, // lengthSizeMinusOne, hard-coded to 4 bytes
  184.           0xe0 | 1, // 3bit reserved (111) + numOfSequenceParameterSets
  185.         ].concat(sps).concat([
  186.           1, // numOfPictureParameterSets
  187.         ]).concat(pps));
  189.     return {
  190.       height,
  191.       width,
  192.       videoConfig,
  193.       hSpacing,
  194.       vSpacing,
  195.     };
  196.   }
  198.   /**
  199.    * @param {!Array.<shaka.extern.MPEG_PES>} videoData
  200.    * @return {!Array.<shaka.extern.VideoSample>}
  201.    */
  202.   static getVideoSamples(videoData) {
  203.     const H264 = shaka.transmuxer.H264;
  205.     /** @type {!Array.<shaka.extern.VideoSample>} */
  206.     const videoSamples = [];
  207.     /** @type {?shaka.extern.VideoSample} */
  208.     let lastVideoSample = null;
  209.     /** @type {boolean} */
  210.     let audFound = false;
  212.     const addLastVideoSample = () => {
  213.       if (!lastVideoSample) {
  214.         return;
  215.       }
  216.       if (!lastVideoSample.nalus.length || !lastVideoSample.frame) {
  217.         return;
  218.       }
  219.       const nalusData = [];
  220.       for (const nalu of lastVideoSample.nalus) {
  221.         const size = nalu.fullData.byteLength;
  222.         const naluLength = new Uint8Array(4);
  223.         naluLength[0] = (size >> 24) & 0xff;
  224.         naluLength[1] = (size >> 16) & 0xff;
  225.         naluLength[2] = (size >> 8) & 0xff;
  226.         naluLength[3] = size & 0xff;
  227.         nalusData.push(naluLength);
  228.         nalusData.push(nalu.fullData);
  229.       }
  230.       lastVideoSample.data = shaka.util.Uint8ArrayUtils.concat(...nalusData);
  231.       videoSamples.push(lastVideoSample);
  232.     };
  234.     const createLastVideoSample = (pes) => {
  235.       lastVideoSample = {
  236.         data: new Uint8Array([]),
  237.         frame: false,
  238.         isKeyframe: false,
  239.         pts: pes.pts,
  240.         dts: pes.dts,
  241.         nalus: [],
  242.       };
  243.     };
  245.     for (let i = 0; i < videoData.length; i++) {
  246.       const pes = videoData[i];
  247.       const nalus = pes.nalus;
  248.       let spsFound = false;
  250.       // If new NAL units found and last sample still there, let's push ...
  251.       // This helps parsing streams with missing AUD
  252.       // (only do this if AUD never found)
  253.       if (lastVideoSample && nalus.length && !audFound) {
  254.         addLastVideoSample();
  255.         createLastVideoSample(pes);
  256.       }
  258.       for (const nalu of pes.nalus) {
  259.         let push = false;
  260.         switch (nalu.type) {
  261.           case H264.NALU_TYPE_NDR_: {
  262.             let isKeyframe = false;
  263.             push = true;
  264.             const data = nalu.data;
  265.             // Only check slice type to detect KF in case SPS found in same
  266.             // packet (any keyframe is preceded by SPS ...)
  267.             if (spsFound && data.length > 4) {
  268.               // retrieve slice type by parsing beginning of NAL unit (follow
  269.               // H264 spec,slice_header definition) to detect keyframe embedded
  270.               // in NDR
  271.               const sliceType = new shaka.util.ExpGolomb(data).readSliceType();
  272.               // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
  273.               // SI slice : A slice that is coded using intra prediction only
  274.               // and using quantisation of the prediction samples.
  275.               // An SI slice can be coded such that its decoded samples can be
  276.               // constructed identically to an SP slice.
  277.               // I slice: A slice that is not an SI slice that is decoded using
  278.               // intra prediction only.
  279.               if (sliceType === 2 || sliceType === 4 ||
  280.                   sliceType === 7 || sliceType === 9) {
  281.                 isKeyframe = true;
  282.               }
  283.             }
  284.             if (isKeyframe) {
  285.               // If we have non-keyframe data already, that cannot belong to
  286.               // the same frame as a keyframe, so force a push
  287.               if (lastVideoSample &&
  288.                   lastVideoSample.frame && !lastVideoSample.isKeyframe) {
  289.                 addLastVideoSample();
  290.                 lastVideoSample = null;
  291.               }
  292.             }
  293.             if (!lastVideoSample) {
  294.               createLastVideoSample(pes);
  295.             }
  296.             lastVideoSample.frame = true;
  297.             lastVideoSample.isKeyframe = isKeyframe;
  298.             break;
  299.           }
  300.           case H264.NALU_TYPE_IDR_: {
  301.             push = true;
  302.             // Handle PES not starting with AUD
  303.             // If we have frame data already, that cannot belong to the same
  304.             // frame, so force a push
  305.             if (lastVideoSample &&
  306.                 lastVideoSample.frame && !lastVideoSample.isKeyframe) {
  307.               addLastVideoSample();
  308.               lastVideoSample = null;
  309.             }
  310.             if (!lastVideoSample) {
  311.               createLastVideoSample(pes);
  312.             }
  313.             lastVideoSample.frame = true;
  314.             lastVideoSample.isKeyframe = true;
  315.             break;
  316.           }
  317.           case H264.NALU_TYPE_SEI_:
  318.             push = true;
  319.             break;
  320.           case H264.NALU_TYPE_SPS_:
  321.             push = true;
  322.             spsFound = true;
  323.             break;
  324.           case H264.NALU_TYPE_PPS_:
  325.             push = true;
  326.             break;
  327.           case H264.NALU_TYPE_AUD_:
  328.             push = true;
  329.             audFound = true;
  330.             if (lastVideoSample && lastVideoSample.frame) {
  331.               addLastVideoSample();
  332.               lastVideoSample = null;
  333.             }
  334.             if (!lastVideoSample) {
  335.               createLastVideoSample(pes);
  336.             }
  337.             break;
  338.           case H264.NALU_TYPE_FILLER_DATA_:
  339.             push = true;
  340.             break;
  341.           default:
  342.             push = false;
  343.             break;
  344.         }
  345.         if (lastVideoSample && push) {
  346.           lastVideoSample.nalus.push(nalu);
  347.         }
  348.       }
  349.     }
  351.     // If last PES packet, push samples
  352.     addLastVideoSample();
  354.     return videoSamples;
  355.   }
  356. };
  359. /**
  360.  * NALU type for NDR for H.264.
  361.  * @const {number}
  362.  * @private
  363.  */
  364. shaka.transmuxer.H264.NALU_TYPE_NDR_ = 0x01;
  367. /**
  368.  * NALU type for Instantaneous Decoder Refresh (IDR) for H.264.
  369.  * @const {number}
  370.  * @private
  371.  */
  372. shaka.transmuxer.H264.NALU_TYPE_IDR_ = 0x05;
  375. /**
  376.  * NALU type for Supplemental Enhancement Information (SEI) for H.264.
  377.  * @const {number}
  378.  * @private
  379.  */
  380. shaka.transmuxer.H264.NALU_TYPE_SEI_ = 0x06;
  383. /**
  384.  * NALU type for Sequence Parameter Set (SPS) for H.264.
  385.  * @const {number}
  386.  * @private
  387.  */
  388. shaka.transmuxer.H264.NALU_TYPE_SPS_ = 0x07;
  391. /**
  392.  * NALU type for Picture Parameter Set (PPS) for H.264.
  393.  * @const {number}
  394.  * @private
  395.  */
  396. shaka.transmuxer.H264.NALU_TYPE_PPS_ = 0x08;
  399. /**
  400.  * NALU type for Access Unit Delimiter (AUD) for H.264.
  401.  * @const {number}
  402.  * @private
  403.  */
  404. shaka.transmuxer.H264.NALU_TYPE_AUD_ = 0x09;
  407. /**
  408.  * NALU type for Filler Data for H.264.
  409.  * @const {number}
  410.  * @private
  411.  */
  412. shaka.transmuxer.H264.NALU_TYPE_FILLER_DATA_ = 0x0c;
  415. /**
  416.  * Values of profile_idc that indicate additional fields are included in the
  417.  * SPS.
  418.  * see Recommendation ITU-T H.264 (4/2013)
  419.  * 7.3.2.1.1 Sequence parameter set data syntax
  420.  *
  421.  * @const {!Array.<number>}
  422.  * @private
  423.  */
  424. shaka.transmuxer.H264.PROFILES_WITH_OPTIONAL_SPS_DATA_ =
  425.     [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134];