Merge remote-tracking branch 'origin/development' into fix/#4867

Author: dsilhavy

index.d.ts

@@ -984,6 +984,8 @@ export interface Representation {
     bitsPerPixel: number;
     codecPrivateData: string | null;
     codecs: string | null;
+    dependencyId: string | null;
+    dependentRepresentation: object | null;
     fragmentDuration: number | null;
     frameRate: number;
     height: number;
@@ -2868,6 +2870,7 @@ export interface conformanceViolationConstants {
 export interface Constants {
     STREAM: 'stream',
     VIDEO: 'video',
+    ENHANCEMENT: 'enhancement',
     AUDIO: 'audio',
     TEXT: 'text',
     MUXED: 'muxed',
@@ -5907,13 +5910,43 @@ export interface StreamProcessor {
 
     selectMediaInfo(selectionInput: object): Promise<any>;
 
+    setEnhancementStreamProcessor(value: StreamProcessor): void;
+
     setExplicitBufferingTime(value: number): void;
 
     setMediaSource(mediaSource: MediaSource): void;
 
     updateStreamInfo(newStreamInfo: StreamInfo): Promise<any>;
 }
 
+export interface ExternalMediaSource {
+    duration: number | null;
+
+    readyState: string;
+
+    addSourceBuffer(mimeType: string): ExternalSourceBuffer;
+
+    close(): void;
+
+    endOfStream(): void;
+
+    open(): void;
+
+    removeSourceBuffer(sourceBuffer: ExternalSourceBuffer): void;
+
+    reset(): void;
+}
+
+export interface ExternalSourceBuffer {
+    buffered: TimeRanges;
+
+    abort(): void;
+
+    appendBuffer(segmentData: ArrayBuffer, segmentStartTime: number, segmentEndTime: number): void;
+
+    remove(start: number, end: number): void;
+}
+
 export interface XlinkLoader {
     load(url: string, element: any, resolveObject: object): void;
 

samples/dash-if-reference-player/app/contributors.json

@@ -68,6 +68,11 @@
             "name": "Broadpeak",
             "logo": "app/img/broadpeak.png",
             "link": "https://broadpeak.tv/"
+        },
+        {
+            "name": "V-Nova",
+            "logo": "app/img/v-nova.png",
+            "link": "https://v-nova.com/"
         }
     ]
 }

samples/dash-if-reference-player/app/css/main.css

@@ -153,6 +153,7 @@ a:hover {
 
 .dash-video-player {
     background-color: #000000;
+    position: relative;
 }
 
 .col-md-9 video {
@@ -179,6 +180,10 @@ a:hover {
     margin-top: -5px !important;
 }
 
+.element-hidden {
+    display: none !important;
+}
+
 .btn-play-pause,
 .control-icon-layout {
     padding: 4px 10px !important;

samples/dash-if-reference-player/app/img/v-nova.png

@@ -238,6 +238,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
 
     $scope.conformanceViolations = [];
 
+    $scope.enhancementDecoder = null;
+
     var defaultExternalSettings = {
         mpd: encodeURIComponent('https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps.mpd'),
         loop: true,
@@ -294,6 +296,12 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
     $scope.audioLiveLatency = 0;
     $scope.audioPlaybackRate = 1.00;
 
+    $scope.activePeriod = '';
+    $scope.bufferingPeriod = '';
+
+    $scope.mpdType = '';
+    $scope.numberOfPeriods = 0;
+
     // Starting Options
     $scope.autoPlaySelected = true;
     $scope.autoLoadSelected = false;
@@ -434,6 +442,12 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
 
     $scope.player.on(dashjs.MediaPlayer.events.MANIFEST_LOADED, function (e) {
         $scope.isDynamic = e.data.type === 'dynamic';
+        if (e.data.Period) {
+            $scope.numberOfPeriods = e.data.Period.length;
+        }
+        if (e.data.type) {
+            $scope.mpdType = e.data.type;
+        }
     }, $scope);
 
 
@@ -450,11 +464,16 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         $scope[e.mediaType + 'Bitrate'] = bitrate;
         $scope.plotPoint('pendingIndex', e.mediaType, e.newQuality + 1, getTimeForPlot());
         $scope.safeApply();
+
+        if (e.currentRepresentation && e.currentRepresentation.adaptation && e.currentRepresentation.adaptation.period) {
+            $scope.bufferingPeriod = e.currentRepresentation.adaptation.period.id;
+        }
     }, $scope);
 
 
     $scope.player.on(dashjs.MediaPlayer.events.PERIOD_SWITCH_COMPLETED, function (e) {
         $scope.currentStreamInfo = e.toStreamInfo;
+        $scope.activePeriod = e.toStreamInfo.id;
     }, $scope);
 
     $scope.player.on(dashjs.MediaPlayer.events.QUALITY_CHANGE_RENDERED, function (e) {
@@ -1035,6 +1054,126 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         });
     };
 
+    $scope.toggleEnhancementEnabled = function () {
+        const video = document.querySelector('video');
+        const canvas = document.querySelector('canvas');
+
+        if ($scope.enhancementEnabled) {
+            canvas.classList.remove('element-hidden');
+            video.classList.add('element-hidden');
+        } else {
+            canvas.classList.add('element-hidden');
+            video.classList.remove('element-hidden');
+        }
+    };
+
+    $scope.setupEnhancementDecoder = function () {
+        /**
+         * MPEG-5 LCEVC Integration for Dash.js Player.
+         *
+         * These are the changes needed for passing the correct
+         * data to lcevc_dec.js and trigger the correct methods
+         * at the correct time.
+         */
+
+        /**
+         * Let the LCEVC Decoder Library make the decision as to when to switch, based on the currently
+         * rendered frame. If disabled, the player needs to signal LCEVC when there is a render change
+         * after an ABR switch happens.
+         *
+         * @readonly
+         * @enum {number}
+         * @public
+         */
+        const AutoRenderMode = {
+            DISABLED: 0,
+            ENABLED: 1
+        };
+
+        dashjs.Extensions = {
+            ...dashjs.Extensions,
+            /**
+             * Attaches LCEVC functionality and methods to the provided Dash.js player instance.
+             *
+             * @param {object} player the Dash.js player instance to attach LCEVC to
+             */
+            useLcevc: function useLcevc(player) {
+                if (!player) {
+                    throw new TypeError('The provided Dash.js player instance was null or undefined.');
+                }
+                const { LCEVCdec } = window;
+                if (!LCEVCdec) {
+                    throw new TypeError('LCEVC Decoder Libraries could not be loaded.');
+                }
+
+                let abrIndex = -1;
+
+                player.attachLcevc = function attachLcevc(media, canvas, LCEVCdecConfig) {
+                    player.LCEVCdec = new LCEVCdec.LCEVCdec(
+                        media,
+                        canvas,
+                        LCEVCdecConfig
+                    );
+
+                    /* Signal profile information and switches to LCEVCdecJS */
+                    player.on(dashjs.MediaPlayer.events.QUALITY_CHANGE_REQUESTED, handleQualityChange);
+                    player.on(dashjs.MediaPlayer.events.FRAGMENT_LOADING_COMPLETED, handleFragmentLoadingCompleted);
+                    player.on(dashjs.MediaPlayer.events.REPRESENTATION_SWITCH, handleRepresentationSwitch);
+                    player.on('externalSourceBufferUpdateStart', handleBufferUpdates);
+                };
+
+                function handleFragmentLoadingCompleted(event) {
+                    if (event.mediaType === 'enhancement') {
+                        abrIndex = event.request.representation.absoluteIndex;
+                    }
+                }
+
+                function handleQualityChange(event) {
+                    if (event.mediaType === 'video' || event.mediaType === 'enhancement') {
+                        const index = event.newRepresentation.absoluteIndex;
+                        console.log('>>> requested:', event.mediaType, index);
+                        player.LCEVCdec.setLevelSwitching(index, AutoRenderMode.ENABLED);
+                    }
+                }
+
+                function handleRepresentationSwitch(event) {
+                    if (event.mediaType === 'video' || event.mediaType === 'enhancement') {
+                        const rep = event.currentRepresentation;
+                        const index = rep.absoluteIndex;
+                        // Workaround for very first representation played for which no QUALITY_CHANGE_REQUESTED arrives
+                        if (rep && rep.dependentRepresentation) {
+                            console.log('>>> rep switch:', event.mediaType, index);
+                            player.LCEVCdec.setLevelSwitching(index, AutoRenderMode.ENABLED);
+                        }
+                    }
+                }
+
+                function handleBufferUpdates(event) {
+                    if (event.request === 'appendBuffer') {
+                        player.LCEVCdec.appendBuffer(event.data, 'video', abrIndex, 0, /* isMuxed */ false);
+                    }
+                    else if (event.request === 'remove') {
+                        player.LCEVCdec.flushBuffer(event.start, event.end);
+                    }
+                }
+            }
+        };
+
+        const video = document.querySelector('video');
+        const canvas = document.querySelector('canvas');
+        const LCEVCdecConfig = {
+            dynamicPerformanceScaling: false
+        };
+
+        window.LCEVCdec.ready.then(() => {
+            /* Attach LCEVC to the Dash.js player instance */
+            const player = $scope.player;
+            dashjs.Extensions.useLcevc(player);
+            player.attachLcevc(video, canvas, LCEVCdecConfig);
+            $scope.enhancementDecoder = player.LCEVCdec;
+        });
+    };
+
     $scope.toggleCmsdApplyMb = function () {
         $scope.player.updateSettings({
             streaming: {
@@ -1104,7 +1243,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
                     liveDelay: $scope.defaultLiveDelay
                 },
                 abr: {},
-                cmcd: {}
+                cmcd: {},
+                enhancement: {}
             }
         };
 
@@ -1170,6 +1310,21 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         config.streaming.cmcd.rtpSafetyFactor = $scope.cmcdRtpSafetyFactor ? $scope.cmcdRtpSafetyFactor : null;
         config.streaming.cmcd.enabledKeys = $scope.cmcdEnabledKeys ? $scope._getFormatedCmcdEnabledKeys() : [];
 
+        // Cleanup enhancement decoder if it exists from previous playback
+        if ($scope.enhancementDecoder) {
+            $scope.enhancementDecoder.close();
+            $scope.enhancementDecoder = null;
+        }
+
+        // Setup enhancement decoder if checkbox is checked or if stream is from V-Nova
+        if ($scope.enhancementEnabled || $scope.selectedItem.provider === 'v-nova') {
+            config.streaming.enhancement.enabled = true;
+            $scope.enhancementEnabled = true;
+            $scope.setupEnhancementDecoder();
+        }
+
+        $scope.toggleEnhancementEnabled();
+
         $scope.player.updateSettings(config);
 
         $scope.controlbar.reset();
@@ -1343,11 +1498,11 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
                             protectionData[input.drmKeySystem]['httpRequestHeaders'] = input.httpRequestHeaders;
                         }
 
-                        if(input.audioRobustness){
+                        if (input.audioRobustness) {
                             protectionData[input.drmKeySystem]['audioRobustness'] = input.audioRobustness;
                         }
 
-                        if(input.videoRobustness){
+                        if (input.videoRobustness) {
                             protectionData[input.drmKeySystem]['videoRobustness'] = input.videoRobustness;
                         }
                     } else {
@@ -1391,11 +1546,11 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
                         protectionData[input.drmKeySystem]['httpRequestHeaders'] = input.httpRequestHeaders;
                     }
 
-                    if(input.audioRobustness){
+                    if (input.audioRobustness) {
                         protectionData[input.drmKeySystem]['audioRobustness'] = input.audioRobustness;
                     }
 
-                    if(input.videoRobustness){
+                    if (input.videoRobustness) {
                         protectionData[input.drmKeySystem]['videoRobustness'] = input.videoRobustness;
                     }
                 }
@@ -2039,8 +2194,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         else if (value === 'null') typedValue = null;
         else if (value === 'undefined') typedValue = undefined;
         else integerRegEx.test(value) ? typedValue = parseInt(value) :
-            (floatRegEx.test(value) ? typedValue = parseFloat(value) :
-                typedValue = value);
+                (floatRegEx.test(value) ? typedValue = parseFloat(value) :
+                    typedValue = value);
 
         return typedValue;
     }

samples/dash-if-reference-player/app/main.js

@@ -93,6 +93,11 @@
             "acronym": "ARTE",
             "name": "ARTE",
             "url": "https://www.arte.tv/en/"
+        },
+        "v-nova": {
+            "acronym": "V-Nova",
+            "name": "V-Nova",
+            "url": "https://v-nova.com/"
         }
     },
     "items": [
@@ -1303,6 +1308,21 @@
                     "provider": "microsoft"
                 }
             ]
+        },
+        {
+            "name": "MPEG-5 Part 2 - LCEVC",
+            "submenu": [
+                {
+                    "url": "https://s3.eu-west-1.amazonaws.com/origin-prod-lon-v-nova.com/lcevcDualTrack/1080p30_4Mbps_no_dR/master.mpd",
+                    "name": "Scalable Carriage",
+                    "provider": "v-nova"
+                },
+                {
+                    "url": "https://s3.eu-west-1.amazonaws.com/origin-prod-lon-v-nova.com/lcevcDualTrack/1080p30_4Mbps_with_dR/master.mpd",
+                    "name": "Scalable with Debug Residuals",
+                    "provider": "v-nova"
+                }
+            ]
         }
     ]
 }