avcodec/qsvdec: Implement SEI parsing for QSV decoders

Signed-off-by: softworkz <[email protected]>

Author: softworkz

libavcodec/Makefile

@@ -146,7 +146,7 @@ OBJS-$(CONFIG_MSS34DSP)                += mss34dsp.o
 OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
 OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
 OBJS-$(CONFIG_QSV)                     += qsv.o
-OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o
+OBJS-$(CONFIG_QSVDEC)                  += qsvdec.o h264_sei.o hevc_sei.o
 OBJS-$(CONFIG_QSVENC)                  += qsvenc.o
 OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
 OBJS-$(CONFIG_RDFT)                    += rdft.o

libavcodec/qsvdec.c

@@ -41,6 +41,7 @@
 #include "libavutil/time.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/film_grain_params.h"
+#include <libavutil/reverse.h>
 
 #include "avcodec.h"
 #include "codec_internal.h"
@@ -49,6 +50,9 @@
 #include "hwconfig.h"
 #include "qsv.h"
 #include "qsv_internal.h"
+#include "h264_sei.h"
+#include "hevc_ps.h"
+#include "hevc_sei.h"
 
 #if QSV_ONEVPL
 #include <mfxdispatcher.h>
@@ -66,6 +70,8 @@ static const AVRational mfx_tb = { 1, 90000 };
     AV_NOPTS_VALUE : pts_tb.num ? \
     av_rescale_q(mfx_pts, mfx_tb, pts_tb) : mfx_pts)
 
+#define PAYLOAD_BUFFER_SIZE 65535
+
 typedef struct QSVAsyncFrame {
     mfxSyncPoint *sync;
     QSVFrame     *frame;
@@ -107,6 +113,9 @@ typedef struct QSVContext {
 
     mfxExtBuffer **ext_buffers;
     int         nb_ext_buffers;
+
+    mfxU8 payload_buffer[PAYLOAD_BUFFER_SIZE];
+    AVBufferRef *a53_buf_ref;
 } QSVContext;
 
 static const AVCodecHWConfigInternal *const qsv_hw_configs[] = {
@@ -628,6 +637,299 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
 }
 #endif
 
+static int find_start_offset(mfxU8 data[4])
+{
+    if (data[0] == 0 && data[1] == 0 && data[2] == 1)
+        return 3;
+
+    if (data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1)
+        return 4;
+
+    return 0;
+}
+
+static int parse_sei_h264(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+    H264SEIContext sei = { 0 };
+    GetBitContext gb = { 0 };
+    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
+    mfxU64 ts;
+    int ret;
+
+    while (1) {
+        int start;
+        memset(payload.Data, 0, payload.BufSize);
+
+        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+            return 0;
+        }
+        if (ret != MFX_ERR_NONE)
+            return ret;
+
+        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+            break;
+
+        start = find_start_offset(payload.Data);
+
+        switch (payload.Type) {
+            case SEI_TYPE_BUFFERING_PERIOD:
+            case SEI_TYPE_PIC_TIMING:
+                continue;
+        }
+
+        if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
+            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+        else {
+            ret = ff_h264_sei_decode(&sei, &gb, NULL, avctx);
+
+            if (ret < 0)
+                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+            else
+                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n", payload.Type, payload.NumBit);
+        }
+    }
+
+    if (out)
+        return ff_h264_set_sei_to_frame(avctx, &sei, out, NULL, 0);
+
+    return 0;
+}
+
+static int parse_sei_hevc(AVCodecContext* avctx, QSVContext* q, QSVFrame* out)
+{
+    HEVCSEI sei = { 0 };
+    HEVCParamSets ps = { 0 };
+    GetBitContext gb = { 0 };
+    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
+    mfxFrameSurface1 *surface = &out->surface;
+    mfxU64 ts;
+    int ret, has_logged = 0;
+
+    while (1) {
+        int start;
+        memset(payload.Data, 0, payload.BufSize);
+
+        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+            return 0;
+        }
+        if (ret != MFX_ERR_NONE)
+            return ret;
+
+        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+            break;
+
+        if (!has_logged) {
+            has_logged = 1;
+            av_log(avctx, AV_LOG_VERBOSE, "-----------------------------------------\n");
+            av_log(avctx, AV_LOG_VERBOSE, "Start reading SEI - payload timestamp: %llu - surface timestamp: %llu\n", ts, surface->Data.TimeStamp);
+        }
+
+        if (ts != surface->Data.TimeStamp) {
+            av_log(avctx, AV_LOG_WARNING, "GetPayload timestamp (%llu) does not match surface timestamp: (%llu)\n", ts, surface->Data.TimeStamp);
+        }
+
+        start = find_start_offset(payload.Data);
+
+        av_log(avctx, AV_LOG_VERBOSE, "parsing SEI type: %3d  Numbits %3d  Start: %d\n", payload.Type, payload.NumBit, start);
+
+        switch (payload.Type) {
+            case SEI_TYPE_BUFFERING_PERIOD:
+            case SEI_TYPE_PIC_TIMING:
+                continue;
+            case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
+                // There seems to be a bug in MSDK
+                payload.NumBit -= 8;
+
+                break;
+            case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
+                // There seems to be a bug in MSDK
+                payload.NumBit = 48;
+
+                break;
+            case SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35:
+                // There seems to be a bug in MSDK
+                if (payload.NumBit == 552)
+                    payload.NumBit = 528;
+                break;
+        }
+
+        if (init_get_bits(&gb, &payload.Data[start], payload.NumBit - start * 8) < 0)
+            av_log(avctx, AV_LOG_ERROR, "Error initializing bitstream reader SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+        else {
+            ret = ff_hevc_decode_nal_sei(&gb, avctx, &sei, &ps, HEVC_NAL_SEI_PREFIX);
+
+            if (ret < 0)
+                av_log(avctx, AV_LOG_WARNING, "Failed to parse SEI type: %d  Numbits %d error: %d\n", payload.Type, payload.NumBit, ret);
+            else
+                av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d\n", payload.Type, payload.NumBit);
+        }
+    }
+
+    if (has_logged) {
+        av_log(avctx, AV_LOG_VERBOSE, "End reading SEI\n");
+    }
+
+    if (out && out->frame)
+        return ff_hevc_set_sei_to_frame(avctx, &sei, out->frame, avctx->framerate, 0, &ps.sps->vui, ps.sps->bit_depth, ps.sps->bit_depth_chroma);
+
+    return 0;
+}
+
+#define A53_MAX_CC_COUNT 2000
+
+static int mpeg_decode_a53_cc(AVCodecContext *avctx, QSVContext *s,
+                              const uint8_t *p, int buf_size)
+{
+    if (buf_size >= 6 &&
+        p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' &&
+        p[4] == 3 && (p[5] & 0x40)) {
+        /* extract A53 Part 4 CC data */
+        unsigned cc_count = p[5] & 0x1f;
+        if (cc_count > 0 && buf_size >= 7 + cc_count * 3) {
+            const uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
+            const uint64_t new_size = (old_size + cc_count
+                                            * UINT64_C(3));
+            int ret;
+
+            if (new_size > 3*A53_MAX_CC_COUNT)
+                return AVERROR(EINVAL);
+
+            ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
+            if (ret >= 0)
+                memcpy(s->a53_buf_ref->data + old_size, p + 7, cc_count * UINT64_C(3));
+
+            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+        }
+        return 1;
+    } else if (buf_size >= 2 && p[0] == 0x03 && (p[1]&0x7f) == 0x01) {
+        /* extract SCTE-20 CC data */
+        GetBitContext gb;
+        unsigned cc_count = 0;
+        int ret;
+
+        init_get_bits8(&gb, p + 2, buf_size - 2);
+        cc_count = get_bits(&gb, 5);
+        if (cc_count > 0) {
+            uint64_t old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
+            uint64_t new_size = (old_size + cc_count * UINT64_C(3));
+            if (new_size > 3 * A53_MAX_CC_COUNT)
+                return AVERROR(EINVAL);
+
+            ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
+            if (ret >= 0) {
+                uint8_t field, cc1, cc2;
+                uint8_t *cap = s->a53_buf_ref->data;
+
+                memset(s->a53_buf_ref->data + old_size, 0, cc_count * 3);
+                for (unsigned i = 0; i < cc_count && get_bits_left(&gb) >= 26; i++) {
+                    skip_bits(&gb, 2); // priority
+                    field = get_bits(&gb, 2);
+                    skip_bits(&gb, 5); // line_offset
+                    cc1 = get_bits(&gb, 8);
+                    cc2 = get_bits(&gb, 8);
+                    skip_bits(&gb, 1); // marker
+
+                    if (!field) { // forbidden
+                        cap[0] = cap[1] = cap[2] = 0x00;
+                    } else {
+                        field = (field == 2 ? 1 : 0);
+                        ////if (!s1->mpeg_enc_ctx.top_field_first) field = !field;
+                        cap[0] = 0x04 | field;
+                        cap[1] = ff_reverse[cc1];
+                        cap[2] = ff_reverse[cc2];
+                    }
+                    cap += 3;
+                }
+            }
+            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+        }
+        return 1;
+    } else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01 && p[3] == 0xf8) {
+        int cc_count = 0;
+        int i, ret;
+        // There is a caption count field in the data, but it is often
+        // incorrect.  So count the number of captions present.
+        for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6)
+            cc_count++;
+        // Transform the DVD format into A53 Part 4 format
+        if (cc_count > 0) {
+            int old_size = s->a53_buf_ref ? s->a53_buf_ref->size : 0;
+            uint64_t new_size = (old_size + cc_count
+                                            * UINT64_C(6));
+            if (new_size > 3*A53_MAX_CC_COUNT)
+                return AVERROR(EINVAL);
+
+            ret = av_buffer_realloc(&s->a53_buf_ref, new_size);
+            if (ret >= 0) {
+                uint8_t field1 = !!(p[4] & 0x80);
+                uint8_t *cap = s->a53_buf_ref->data;
+                p += 5;
+                for (i = 0; i < cc_count; i++) {
+                    cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd;
+                    cap[1] = p[1];
+                    cap[2] = p[2];
+                    cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd;
+                    cap[4] = p[4];
+                    cap[5] = p[5];
+                    cap += 6;
+                    p += 6;
+                }
+            }
+            avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS;
+        }
+        return 1;
+    }
+    return 0;
+}
+
+static int parse_sei_mpeg12(AVCodecContext* avctx, QSVContext* q, AVFrame* out)
+{
+    mfxPayload payload = { 0, .Data = &q->payload_buffer[0], .BufSize = sizeof(q->payload_buffer) - AV_INPUT_BUFFER_PADDING_SIZE };
+    mfxU64 ts;
+    int ret;
+
+    while (1) {
+        int start;
+
+        memset(payload.Data, 0, payload.BufSize);
+        ret = MFXVideoDECODE_GetPayload(q->session, &ts, &payload);
+        if (ret == MFX_ERR_NOT_ENOUGH_BUFFER) {
+            av_log(avctx, AV_LOG_WARNING, "Warning: Insufficient buffer on GetPayload(). Size: %"PRIu64" Needed: %d\n", sizeof(q->payload_buffer), payload.BufSize);
+            return 0;
+        }
+        if (ret != MFX_ERR_NONE)
+            return ret;
+
+        if (payload.NumBit == 0 || payload.NumBit >= payload.BufSize * 8)
+            break;
+
+        start = find_start_offset(payload.Data);
+
+        start++;
+
+        mpeg_decode_a53_cc(avctx, q, &payload.Data[start], (int)((payload.NumBit + 7) / 8) - start);
+
+        av_log(avctx, AV_LOG_DEBUG, "mfxPayload Type: %d  Numbits %d start %d -> %.s\n", payload.Type, payload.NumBit, start, (char *)(&payload.Data[start]));
+    }
+
+    if (!out)
+        return 0;
+
+    if (q->a53_buf_ref) {
+
+        AVFrameSideData *sd = av_frame_new_side_data_from_buf(out, AV_FRAME_DATA_A53_CC, q->a53_buf_ref);
+        if (!sd)
+            av_buffer_unref(&q->a53_buf_ref);
+        q->a53_buf_ref = NULL;
+    }
+
+    return 0;
+}
+
 static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
                       AVFrame *frame, int *got_frame,
                       const AVPacket *avpkt)
@@ -664,6 +966,8 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
                                               insurf, &outsurf, sync);
         if (ret == MFX_WRN_DEVICE_BUSY)
             av_usleep(500);
+        else if (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO)
+            parse_sei_mpeg12(avctx, q, NULL);
 
     } while (ret == MFX_WRN_DEVICE_BUSY || ret == MFX_ERR_MORE_SURFACE);
 
@@ -705,6 +1009,23 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
             return AVERROR_BUG;
         }
 
+        switch (avctx->codec_id) {
+        case AV_CODEC_ID_MPEG2VIDEO:
+            ret = parse_sei_mpeg12(avctx, q, out_frame->frame);
+            break;
+        case AV_CODEC_ID_H264:
+            ret = parse_sei_h264(avctx, q, out_frame->frame);
+            break;
+        case AV_CODEC_ID_HEVC:
+            ret = parse_sei_hevc(avctx, q, out_frame);
+            break;
+        default:
+            ret = 0;
+        }
+
+        if (ret < 0)
+            av_log(avctx, AV_LOG_ERROR, "Error parsing SEI data: %d\n", ret);
+
         out_frame->queued += 1;
 
         aframe = (QSVAsyncFrame){ sync, out_frame };