Feature: MPEG-5 LCEVC Scalable support

index.d.ts

@@ -965,6 +965,8 @@ declare namespace dashjs {
         bitsPerPixel: number;
         codecPrivateData: string | null;
         codecs: string | null;
+        dependencyId: string | null;
+        dependentRepresentation: object | null;
         fragmentDuration: number | null;
         frameRate: number;
         height: number;
@@ -1911,6 +1913,10 @@ declare namespace dashjs {
                     etpWeightRatio?: number
                 }
             },
+            enhancement?: {
+                enabled?: boolean,
+                codecs?: Array<string>
+            },
             defaultSchemeIdUri?: {
                 viewpoint?: string,
                 audioChannelConfiguration?: string,
@@ -2836,6 +2842,7 @@ declare namespace dashjs {
     export interface Constants {
         STREAM: 'stream',
         VIDEO: 'video',
+        ENHANCEMENT: 'enhancement',
         AUDIO: 'audio',
         TEXT: 'text',
         MUXED: 'muxed',
@@ -5874,12 +5881,42 @@ declare namespace dashjs {
 
         selectMediaInfo(selectionInput: object): Promise<any>;
 
+        setEnhancementStreamProcessor(value: StreamProcessor): void;
+
         setExplicitBufferingTime(value: number): void;
 
         setMediaSource(mediaSource: MediaSource): void;
 
         updateStreamInfo(newStreamInfo: StreamInfo): Promise<any>;
     }
+
+    export interface ExternalMediaSource {
+        duration: number | null;
+
+        readyState: string;
+
+        addSourceBuffer(mimeType: string): ExternalSourceBuffer;
+
+        close(): void;
+
+        endOfStream(): void;
+
+        open(): void;
+
+        removeSourceBuffer(sourceBuffer: ExternalSourceBuffer): void;
+
+        reset(): void;
+    }
+
+    export interface ExternalSourceBuffer {
+        buffered: TimeRanges;
+
+        abort(): void;
+
+        appendBuffer(segmentData: ArrayBuffer, segmentStartTime: number, segmentEndTime: number): void;
+
+        remove(start: number, end: number): void;
+    }
 
     export interface XlinkLoader {
         load(url: string, element: any, resolveObject: object): void;

samples/dash-if-reference-player/app/contributors.json

@@ -68,6 +68,11 @@
             "name": "Broadpeak",
             "logo": "app/img/broadpeak.png",
             "link": "https://broadpeak.tv/"
+        },
+        {
+            "name": "V-Nova",
+            "logo": "app/img/v-nova.png",
+            "link": "https://v-nova.com/"
         }
     ]
 }

samples/dash-if-reference-player/app/css/main.css

@@ -153,6 +153,9 @@ a:hover {
 
 .dash-video-player {
     background-color: #000000;
+    position: relative;
+    margin: 0 auto;
+    line-height: 1.0;
 }
 
 .col-md-9 video {
@@ -178,6 +181,10 @@ a:hover {
     margin-top: -5px !important;
 }
 
+.element-hidden {
+    display: none !important;
+}
+
 .btn-play-pause,
 .control-icon-layout {
     padding: 4px 10px !important;

samples/dash-if-reference-player/app/main.js

@@ -243,6 +243,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
 
     $scope.conformanceViolations = [];
 
+    $scope.enhancementDecoder = null;
+
     var defaultExternalSettings = {
         mpd: encodeURIComponent('https://dash.akamaized.net/akamai/bbb_30fps/bbb_30fps.mpd'),
         loop: true,
@@ -1040,6 +1042,126 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         });
     };
 
+    $scope.toggleEnhancementEnabled = function () {
+        const video = document.querySelector('video');
+        const canvas = document.querySelector('canvas');
+
+        if ($scope.enhancementEnabled) {
+            canvas.classList.remove('element-hidden');
+            video.classList.add('element-hidden');
+        } else {
+            canvas.classList.add('element-hidden');
+            video.classList.remove('element-hidden');
+        }
+    };
+
+    $scope.setupEnhancementDecoder = function () {
+        /**
+         * MPEG-5 LCEVC Integration for Dash.js Player.
+         *
+         * These are the changes needed for passing the correct
+         * data to lcevc_dec.js and trigger the correct methods
+         * at the correct time.
+         */
+
+        /**
+         * Let the LCEVC Decoder Library make the decision as to when to switch, based on the currently
+         * rendered frame. If disabled, the player needs to signal LCEVC when there is a render change
+         * after an ABR switch happens.
+         *
+         * @readonly
+         * @enum {number}
+         * @public
+         */
+        const AutoRenderMode = {
+            DISABLED: 0,
+            ENABLED: 1
+        };
+
+        dashjs.Extensions = {
+            ...dashjs.Extensions,
+            /**
+             * Attaches LCEVC functionality and methods to the provided Dash.js player instance.
+             *
+             * @param {object} player the Dash.js player instance to attach LCEVC to
+             */
+            useLcevc: function useLcevc(player) {
+                if (!player) {
+                    throw new TypeError('The provided Dash.js player instance was null or undefined.');
+                }
+                const { LCEVCdec } = window;
+                if (!LCEVCdec) {
+                    throw new TypeError('LCEVC Decoder Libraries could not be loaded.');
+                }
+
+                let abrIndex = -1;
+
+                player.attachLcevc = function attachLcevc(media, canvas, LCEVCdecConfig) {
+                    player.LCEVCdec = new LCEVCdec.LCEVCdec(
+                        media,
+                        canvas,
+                        LCEVCdecConfig
+                    );
+
+                    /* Signal profile information and switches to LCEVCdecJS */
+                    player.on(dashjs.MediaPlayer.events.QUALITY_CHANGE_REQUESTED, handleQualityChange);
+                    player.on(dashjs.MediaPlayer.events.FRAGMENT_LOADING_COMPLETED, handleFragmentLoadingCompleted);
+                    player.on(dashjs.MediaPlayer.events.REPRESENTATION_SWITCH, handleRepresentationSwitch);
+                    player.on('externalSourceBufferUpdateStart', handleBufferUpdates);
+                };
+
+                function handleFragmentLoadingCompleted(event) {
+                    if (event.mediaType === 'enhancement') {
+                        abrIndex = event.request.representation.absoluteIndex;
+                    }
+                }
+
+                function handleQualityChange(event) {
+                    if (event.mediaType === 'video' || event.mediaType === 'enhancement') {
+                        const index = event.newRepresentation.absoluteIndex;
+                        console.log('>>> requested:', event.mediaType, index);
+                        player.LCEVCdec.setLevelSwitching(index, AutoRenderMode.ENABLED);
+                    }
+                }
+
+                function handleRepresentationSwitch(event) {
+                    if (event.mediaType === 'video' || event.mediaType === 'enhancement') {
+                        const rep = event.currentRepresentation;
+                        const index = rep.absoluteIndex;
+                        // Workaround for very first representation played for which no QUALITY_CHANGE_REQUESTED arrives
+                        if (rep && rep.dependentRepresentation) {
+                            console.log('>>> rep switch:', event.mediaType, index);
+                            player.LCEVCdec.setLevelSwitching(index, AutoRenderMode.ENABLED);
+                        }
+                    }
+                }
+
+                function handleBufferUpdates(event) {
+                    if (event.request === 'appendBuffer') {
+                        player.LCEVCdec.appendBuffer(event.data, 'video', abrIndex, 0, /* isMuxed */ false);
+                    }
+                    else if (event.request === 'remove') {
+                        player.LCEVCdec.flushBuffer(event.start, event.end);
+                    }
+                }
+            }
+        };
+
+        const video = document.querySelector('video');
+        const canvas = document.querySelector('canvas');
+        const LCEVCdecConfig = {
+            dynamicPerformanceScaling: false
+        };
+
+        window.LCEVCdec.ready.then(() => {
+            /* Attach LCEVC to the Dash.js player instance */
+            const player = $scope.player;
+            dashjs.Extensions.useLcevc(player);
+            player.attachLcevc(video, canvas, LCEVCdecConfig);
+            $scope.enhancementDecoder = player.LCEVCdec;
+        });
+    };
+
     $scope.toggleCmsdApplyMb = function () {
         $scope.player.updateSettings({
             streaming: {
@@ -1109,7 +1231,8 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
                     liveDelay: $scope.defaultLiveDelay
                 },
                 abr: {},
-                cmcd: {}
+                cmcd: {},
+                enhancement: {}
             }
         };
 
@@ -1175,6 +1298,21 @@ app.controller('DashController', ['$scope', '$window', 'sources', 'contributors'
         config.streaming.cmcd.rtpSafetyFactor = $scope.cmcdRtpSafetyFactor ? $scope.cmcdRtpSafetyFactor : null;
         config.streaming.cmcd.enabledKeys = $scope.cmcdEnabledKeys ? $scope._getFormatedCmcdEnabledKeys() : [];
 
+        // Cleanup enhancement decoder if it exists from previous playback
+        if ($scope.enhancementDecoder) {
+            $scope.enhancementDecoder.close();
+            $scope.enhancementDecoder = null;
+        }
+
+        // Setup enhancement decoder if checkbox is checked or if stream is from V-Nova
+        if ($scope.enhancementEnabled || $scope.selectedItem.provider === 'v-nova') {
+            config.streaming.enhancement.enabled = true;
+            $scope.enhancementEnabled = true;
+            $scope.setupEnhancementDecoder();
+        }
+
+        $scope.toggleEnhancementEnabled();
+
         $scope.player.updateSettings(config);
 
         $scope.controlbar.reset();

samples/dash-if-reference-player/app/sources.json

@@ -93,6 +93,11 @@
             "acronym": "ARTE",
             "name": "ARTE",
             "url": "https://www.arte.tv/en/"
+        },
+        "v-nova": {
+            "acronym": "V-Nova",
+            "name": "V-Nova",
+            "url": "https://v-nova.com/"
         }
     },
     "items": [
@@ -1303,6 +1308,21 @@
                     "provider": "microsoft"
                 }
             ]
+        },
+        {
+            "name": "MPEG-5 Part 2 - LCEVC",
+            "submenu": [
+                {
+                    "url": "https://s3.eu-west-1.amazonaws.com/origin-prod-lon-v-nova.com/lcevcDualTrack/1080p30_4Mbps_no_dR/master.mpd",
+                    "name": "Scalable Carriage",
+                    "provider": "v-nova"
+                },
+                {
+                    "url": "https://s3.eu-west-1.amazonaws.com/origin-prod-lon-v-nova.com/lcevcDualTrack/1080p30_4Mbps_with_dR/master.mpd",
+                    "name": "Scalable with Debug Residuals",
+                    "provider": "v-nova"
+                }
+            ]
         }
     ]
 }

samples/dash-if-reference-player/index.html

@@ -34,6 +34,9 @@
     <script src="app/main.js"></script>
     <script src="app/rules/ThroughputRule.js"></script>
 
+    <!-- LCEVC Decoder -->
+    <script src="https://unpkg.com/lcevc_dec.js@latest/dist/lcevc_dec.min.js"></script>
+
     <!-- Google Cast -->
     <script src="https://www.gstatic.com/cv/js/sender/v1/cast_sender.js?loadCastFramework=1"></script>
 
@@ -922,6 +925,17 @@
                        ng-change="updateCmsdEtpWeightRatio()">
             </div>
         </div>
+        <div class="options-item">
+            <div class="options-item-title">Enhancement</div>
+            <div class="options-item-body">
+                <label class="topcoat-checkbox" data-toggle="tooltip" data-placement="right"
+                       title="Enables enhancement. The provided stream should contain two adaptation sets, one containing the base layer and the second containing an enhancement layer. Checkbox selection will be applied the next time a stream is loaded.">
+                    <input type="checkbox" ng-model="enhancementEnabled" ng-change="toggleEnhancementEnabled()"
+                           ng-checked="enhancementEnabled">
+                    Enable Enhancement
+                </label>
+            </div>
+        </div>
 
     </div>
 
@@ -930,6 +944,8 @@
         <div class="dash-video-player col-md-9">
             <div id="videoContainer" class="videoContainer">
                 <video disableRemotePlayback="true"></video>
+                <canvas id="enhacementCanvas" class="enhancementCanvas element-hidden"></canvas>
+                <div style="min-height: 1px;"></div>
                 <div id="video-caption"></div>
                 <div id="cast-msg" ng-if="isCasting">
                     {{ castPlayerState === 'IDLE' ? 'Ready to cast stream' : castPlayerState }}