VPLAY-12333 mp4demux hardening

mp4demux/MP4Demux.h

@@ -79,18 +79,20 @@ enum mp4LogLevel
  */
 enum Mp4ParseError
 {
-	MP4_PARSE_OK = 0,                              /**< No error */
-	MP4_PARSE_ERROR_INVALID_BOX,                   /**< Invalid box encountered */
-	MP4_PARSE_ERROR_INVALID_CONSTANT_IV_SIZE,      /**< Invalid constant IV size */
-	MP4_PARSE_ERROR_SAMPLE_COUNT_MISMATCH,         /**< Sample count mismatch */
-	MP4_PARSE_ERROR_UNSUPPORTED_ENCRYPTION_SCHEME, /**< Invalid auxiliary info type */
-	MP4_PARSE_ERROR_MISSING_DATA_OFFSET,           /**< Missing data offset in TRUN */
-	MP4_PARSE_ERROR_INVALID_PADDING,               /**< Invalid padding value */
-	MP4_PARSE_ERROR_UNSUPPORTED_SAMPLE_ENTRY_COUNT,/**< Unsupported sample entry count */
+	MP4_PARSE_OK,                                  /**< No error */
+	MP4_PARSE_ERROR_INVALID_BOX,                   /**< Invalid box header size */
+	MP4_PARSE_ERROR_INVALID_IV_SIZE,               /**< Invalid  IV size (expected 8 or 16) */
+	MP4_PARSE_ERROR_SAMPLE_COUNT_MISMATCH,         /**< Explicit sample count doesn't match implicit sample count */
+	MP4_PARSE_ERROR_UNSUPPORTED_ENCRYPTION_SCHEME, /**< Expected cenc or cbcs */
+	MP4_PARSE_ERROR_INVALID_PADDING,               /**< Unexpected Video Padding field (should be 0xffff) */
+	MP4_PARSE_ERROR_UNSUPPORTED_SAMPLE_ENTRY_COUNT,/**< Zero sample entry count */
 	MP4_PARSE_ERROR_UNSUPPORTED_STREAM_FORMAT,     /**< Unsupported stream format */
 	MP4_PARSE_ERROR_INVALID_ESDS_TAG,              /**< Invalid ESDS tag */
-	MP4_PARSE_ERROR_DATA_BOUNDARY_MISMATCH,        /**< Data boundary mismatch */
-	MP4_PARSE_ERROR_INVALID_INPUT                 /**< Invalid input to parse function */
+	MP4_PARSE_ERROR_DATA_BOUNDARY_MISMATCH,        /**< Data boundary mismatch - referencing invalid memory */
+	MP4_PARSE_ERROR_INVALID_INPUT,                 /**< Invalid input to parse function;  nullptr or zero length */
+	MP4_PARSE_ERROR_INVALID_KID,                   /**< Invalid (huge) kidCount  */
+	MP4_PARSE_ERROR_INVALID_ENTRY_COUNT,           /**< Entry count is zero */
+	MP4_PARSE_ERROR_VARIABLE_LENGTH_OVERFLOW       /**< Value encoded using octets exceed 32 bits */
 };
 
 /**
@@ -143,6 +145,10 @@ class Mp4Demux
 	// Parser state
 	const uint8_t *moofPtr;                       /**< Base address for sample data */
 	const uint8_t *ptr;                           /**< Current parser position */
+	const uint8_t *endPtr;                        /**< Absolute end boundary of the current parse buffer */
+	// MDAT range tracking (for sample data validation)
+	const uint8_t *mdatStart;                     /**< Pointer to the first byte of the payload of the current or most recently parsed 'mdat' box. Set when an 'mdat' box is parsed and used as the lower bound when validating that computed sample data offsets and sizes remain within the available payload range. */
+	const uint8_t *mdatEnd;                        /**< Pointer to one past the last byte of the payload of the current or most recently parsed 'mdat' box. Set together with mdatStart during 'mdat' parsing and used as the upper bound during sample data bounds checking to ensure no read extends beyond the validated 'mdat' payload. */
 
 	// Box header fields
 	uint8_t version;                              /**< Box version */
@@ -167,6 +173,12 @@ class Mp4Demux
 	MediaCodecInfo codecInfo;                     /**< Codec information */
 	Mp4ParseError parseError;                     /**< Current parse error state */
 
+	/**
+	 * @brief log human readable parse error and update state
+	 * @param parseError one of Mp4ParseError
+	 */
+	void setParseError( Mp4ParseError );
+
 	/**
 	 * @brief Read n bytes from current position in big-endian format
 	 * @param n Number of bytes to read
@@ -321,7 +333,7 @@ class Mp4Demux
 	 * @brief Read length field with variable encoding
 	 * @return Length value
 	 */
-	int ReadLen();
+	uint32_t ReadLen();
 
 	/**
 	 * @brief Parse codec configuration helper
@@ -383,6 +395,14 @@ class Mp4Demux
 	 * @brief Parse MP4 data
 	 * @param ptr Pointer to MP4 data
 	 * @param len Length of data
+	 *
+	 * @note Internally, an end-of-buffer pointer (endPtr) is computed as
+	 *       &((const uint8_t*)ptr)[len]. All parsing routines use this
+	 *       endPtr for uniform bounds checking, ensuring that every read
+	 *       and pointer increment is validated against the end of the
+	 *       input buffer to prevent buffer overruns and out-of-bounds
+	 *       memory access.
+	 *
 	 * @return true if parsing succeeded, false on error
 	 */
 	bool Parse(const void *ptr, size_t len);
@@ -428,4 +448,4 @@ class Mp4Demux
 	std::vector<AampMediaSample> GetSamples();
 };
 
-#endif /* __MP4_DEMUX_H__ */
+#endif /* __MP4_DEMUX_H__ */