Feature: MPEG-5 LCEVC Scalable support

index.d.ts

@@ -984,6 +984,8 @@ export interface Representation {
     bitsPerPixel: number;
     codecPrivateData: string | null;
     codecs: string | null;
+    dependencyId: string | null;
+    dependentRepresentation: object | null;
     fragmentDuration: number | null;
     frameRate: number;
     height: number;
@@ -2868,6 +2870,7 @@ export interface conformanceViolationConstants {
 export interface Constants {
     STREAM: 'stream',
     VIDEO: 'video',
+    ENHANCEMENT: 'enhancement',
     AUDIO: 'audio',
     TEXT: 'text',
     MUXED: 'muxed',
@@ -5907,13 +5910,43 @@ export interface StreamProcessor {
 
     selectMediaInfo(selectionInput: object): Promise<any>;
 
+    setEnhancementStreamProcessor(value: StreamProcessor): void;
+
     setExplicitBufferingTime(value: number): void;
 
     setMediaSource(mediaSource: MediaSource): void;
 
     updateStreamInfo(newStreamInfo: StreamInfo): Promise<any>;
 }
 
+export interface ExternalMediaSource {
+    duration: number | null;
+
+    readyState: string;
+
+    addSourceBuffer(mimeType: string): ExternalSourceBuffer;
+
+    close(): void;
+
+    endOfStream(): void;
+
+    open(): void;
+
+    removeSourceBuffer(sourceBuffer: ExternalSourceBuffer): void;
+
+    reset(): void;
+}
+
+export interface ExternalSourceBuffer {
+    buffered: TimeRanges;
+
+    abort(): void;
+
+    appendBuffer(segmentData: ArrayBuffer, segmentStartTime: number, segmentEndTime: number): void;
+
+    remove(start: number, end: number): void;
+}
+
 export interface XlinkLoader {
     load(url: string, element: any, resolveObject: object): void;
 

samples/dash-if-reference-player/app/contributors.json

@@ -68,6 +68,11 @@
             "name": "Broadpeak",
             "logo": "app/img/broadpeak.png",
             "link": "https://broadpeak.tv/"
+        },
+        {
+            "name": "V-Nova",
+            "logo": "app/img/v-nova.png",
+            "link": "https://v-nova.com/"
         }
     ]
 }

samples/dash-if-reference-player/app/css/main.css

@@ -153,6 +153,7 @@ a:hover {
 
 .dash-video-player {
     background-color: #000000;
+    position: relative;
 }
 
 .col-md-9 video {
@@ -179,6 +180,10 @@ a:hover {
     margin-top: -5px !important;
 }
 
+.element-hidden {
+    display: none !important;
+}
+
 .btn-play-pause,
 .control-icon-layout {
     padding: 4px 10px !important;

samples/dash-if-reference-player/app/main.js

@@ -238,6 +238,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
 
     $scope.conformanceViolations = [];
 
+    $scope.enhancementDecoder = null;
+
     var defaultExternalSettings = {
         mpd: encodeURIComponent('https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps.mpd'),
         loop: true,
@@ -1035,6 +1037,126 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         });
     };
 
+    $scope.toggleEnhancementEnabled = function () {
+        const video = document.querySelector('video');
+        const canvas = document.querySelector('canvas');
+
+        if ($scope.enhancementEnabled) {
+            canvas.classList.remove('element-hidden');
+            video.classList.add('element-hidden');
+        } else {
+            canvas.classList.add('element-hidden');
+            video.classList.remove('element-hidden');
+        }
+    };
+
+    $scope.setupEnhancementDecoder = function () {
+        /**
+         * MPEG-5 LCEVC Integration for Dash.js Player.
+         *
+         * These are the changes needed for passing the correct
+         * data to lcevc_dec.js and trigger the correct methods
+         * at the correct time.
+         */
+
+        /**
+         * Let the LCEVC Decoder Library make the decision as to when to switch, based on the currently
+         * rendered frame. If disabled, the player needs to signal LCEVC when there is a render change
+         * after an ABR switch happens.
+         *
+         * @readonly
+         * @enum {number}
+         * @public
+         */
+        const AutoRenderMode = {
+            DISABLED: 0,
+            ENABLED: 1
+        };
+
+        dashjs.Extensions = {
+            ...dashjs.Extensions,
+            /**
+             * Attaches LCEVC functionality and methods to the provided Dash.js player instance.
+             *
+             * @param {object} player the Dash.js player instance to attach LCEVC to
+             */
+            useLcevc: function useLcevc(player) {
+                if (!player) {
+                    throw new TypeError('The provided Dash.js player instance was null or undefined.');
+                }
+                const { LCEVCdec } = window;
+                if (!LCEVCdec) {
+                    throw new TypeError('LCEVC Decoder Libraries could not be loaded.');
+                }
+
+                let abrIndex = -1;
+
+                player.attachLcevc = function attachLcevc(media, canvas, LCEVCdecConfig) {
+                    player.LCEVCdec = new LCEVCdec.LCEVCdec(
+                        media,
+                        canvas,
+                        LCEVCdecConfig
+                    );
+
+                    /* Signal profile information and switches to LCEVCdecJS */
+                    player.on(dashjs.MediaPlayer.events.QUALITY_CHANGE_REQUESTED, handleQualityChange);
+                    player.on(dashjs.MediaPlayer.events.FRAGMENT_LOADING_COMPLETED, handleFragmentLoadingCompleted);
+                    player.on(dashjs.MediaPlayer.events.REPRESENTATION_SWITCH, handleRepresentationSwitch);
+                    player.on('externalSourceBufferUpdateStart', handleBufferUpdates);
+                };
+
+                function handleFragmentLoadingCompleted(event) {
+                    if (event.mediaType === 'enhancement') {
+                        abrIndex = event.request.representation.absoluteIndex;
+                    }
+                }
+
+                function handleQualityChange(event) {
+                    if (event.mediaType === 'video' || event.mediaType === 'enhancement') {
+                        const index = event.newRepresentation.absoluteIndex;
+                        console.log('>>> requested:', event.mediaType, index);
+                        player.LCEVCdec.setLevelSwitching(index, AutoRenderMode.ENABLED);
+                    }
+                }
+
+                function handleRepresentationSwitch(event) {
+                    if (event.mediaType === 'video' || event.mediaType === 'enhancement') {
+                        const rep = event.currentRepresentation;
+                        const index = rep.absoluteIndex;
+                        // Workaround for very first representation played for which no QUALITY_CHANGE_REQUESTED arrives
+                        if (rep && rep.dependentRepresentation) {
+                            console.log('>>> rep switch:', event.mediaType, index);
+                            player.LCEVCdec.setLevelSwitching(index, AutoRenderMode.ENABLED);
+                        }
+                    }
+                }
+
+                function handleBufferUpdates(event) {
+                    if (event.request === 'appendBuffer') {
+                        player.LCEVCdec.appendBuffer(event.data, 'video', abrIndex, 0, /* isMuxed */ false);
+                    }
+                    else if (event.request === 'remove') {
+                        player.LCEVCdec.flushBuffer(event.start, event.end);
+                    }
+                }
+            }
+        };
+
+        const video = document.querySelector('video');
+        const canvas = document.querySelector('canvas');
+        const LCEVCdecConfig = {
+            dynamicPerformanceScaling: false
+        };
+
+        window.LCEVCdec.ready.then(() => {
+            /* Attach LCEVC to the Dash.js player instance */
+            const player = $scope.player;
+            dashjs.Extensions.useLcevc(player);
+            player.attachLcevc(video, canvas, LCEVCdecConfig);
+            $scope.enhancementDecoder = player.LCEVCdec;
+        });
+    };
+
     $scope.toggleCmsdApplyMb = function () {
         $scope.player.updateSettings({
             streaming: {
@@ -1104,7 +1226,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
                     liveDelay: $scope.defaultLiveDelay
                 },
                 abr: {},
-                cmcd: {}
+                cmcd: {},
+                enhancement: {}
             }
         };
 
@@ -1170,6 +1293,21 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         config.streaming.cmcd.rtpSafetyFactor = $scope.cmcdRtpSafetyFactor ? $scope.cmcdRtpSafetyFactor : null;
         config.streaming.cmcd.enabledKeys = $scope.cmcdEnabledKeys ? $scope._getFormatedCmcdEnabledKeys() : [];
 
+        // Cleanup enhancement decoder if it exists from previous playback
+        if ($scope.enhancementDecoder) {
+            $scope.enhancementDecoder.close();
+            $scope.enhancementDecoder = null;
+        }
+
+        // Setup enhancement decoder if checkbox is checked or if stream is from V-Nova
+        if ($scope.enhancementEnabled || $scope.selectedItem.provider === 'v-nova') {
+            config.streaming.enhancement.enabled = true;
+            $scope.enhancementEnabled = true;
+            $scope.setupEnhancementDecoder();
+        }
+
+        $scope.toggleEnhancementEnabled();
+
         $scope.player.updateSettings(config);
 
         $scope.controlbar.reset();

samples/dash-if-reference-player/app/sources.json

@@ -93,6 +93,11 @@
             "acronym": "ARTE",
             "name": "ARTE",
             "url": "https://www.arte.tv/en/"
+        },
+        "v-nova": {
+            "acronym": "V-Nova",
+            "name": "V-Nova",
+            "url": "https://v-nova.com/"
         }
     },
     "items": [
@@ -1303,6 +1308,21 @@
                     "provider": "microsoft"
                 }
             ]
+        },
+        {
+            "name": "MPEG-5 Part 2 - LCEVC",
+            "submenu": [
+                {
+                    "url": "https://s3.eu-west-1.amazonaws.com/origin-prod-lon-v-nova.com/lcevcDualTrack/1080p30_4Mbps_no_dR/master.mpd",
+                    "name": "Scalable Carriage",
+                    "provider": "v-nova"
+                },
+                {
+                    "url": "https://s3.eu-west-1.amazonaws.com/origin-prod-lon-v-nova.com/lcevcDualTrack/1080p30_4Mbps_with_dR/master.mpd",
+                    "name": "Scalable with Debug Residuals",
+                    "provider": "v-nova"
+                }
+            ]
         }
     ]
 }

samples/dash-if-reference-player/index.html

@@ -34,6 +34,9 @@
     <script src="app/main.js"></script>
     <script src="app/rules/ThroughputRule.js"></script>
 
+    <!-- LCEVC Decoder -->
+    <script src="https://unpkg.com/lcevc_dec.js@latest/dist/lcevc_dec.min.js"></script>
+
     <!-- Google Cast -->
     <script src="https://www.gstatic.com/cv/js/sender/v1/cast_sender.js?loadCastFramework=1"></script>
 
@@ -1003,6 +1006,17 @@
                        ng-change="updateCmsdEtpWeightRatio()">
             </div>
         </div>
+        <div class="options-item">
+            <div class="options-item-title">Enhancement Layer</div>
+            <div class="options-item-body">
+                <label class="topcoat-checkbox" data-toggle="tooltip" data-placement="right"
+                       title="Enables enhancement layer. The provided stream should contain two adaptation sets, one containing the base layer and the second containing an enhancement layer. Checkbox selection will be applied the next time a stream is loaded.">
+                    <input type="checkbox" ng-model="enhancementEnabled" ng-change=""
+                           ng-checked="enhancementEnabled">
+                    Enable Enhancement Layer
+                </label>
+            </div>
+        </div>
 
     </div>
 
@@ -1011,6 +1025,8 @@
         <div class="dash-video-player col-md-9">
             <div id="videoContainer" class="videoContainer">
                 <video disableRemotePlayback="true"></video>
+                <canvas id="enhacementCanvas" class="enhancementCanvas element-hidden"></canvas>
+                <div style="min-height: 1px;"></div>
                 <div id="video-caption"></div>
                 <div id="cast-msg" ng-if="isCasting">
                     {{ castPlayerState === 'IDLE' ? 'Ready to cast stream' : castPlayerState }}