[WIP] WIP address feedback on mp4demux hardening from PR #830 review

mp4demux/MP4Demux.h

@@ -79,18 +79,20 @@ enum mp4LogLevel
  */
 enum Mp4ParseError
 {
-	MP4_PARSE_OK = 0,                              /**< No error */
-	MP4_PARSE_ERROR_INVALID_BOX,                   /**< Invalid box encountered */
-	MP4_PARSE_ERROR_INVALID_CONSTANT_IV_SIZE,      /**< Invalid constant IV size */
-	MP4_PARSE_ERROR_SAMPLE_COUNT_MISMATCH,         /**< Sample count mismatch */
-	MP4_PARSE_ERROR_UNSUPPORTED_ENCRYPTION_SCHEME, /**< Invalid auxiliary info type */
-	MP4_PARSE_ERROR_MISSING_DATA_OFFSET,           /**< Missing data offset in TRUN */
-	MP4_PARSE_ERROR_INVALID_PADDING,               /**< Invalid padding value */
-	MP4_PARSE_ERROR_UNSUPPORTED_SAMPLE_ENTRY_COUNT,/**< Unsupported sample entry count */
+	MP4_PARSE_OK,                                  /**< No error */
+	MP4_PARSE_ERROR_INVALID_BOX,                   /**< Invalid box header size */
+	MP4_PARSE_ERROR_INVALID_IV_SIZE,               /**< Invalid  IV size (expected 8 or 16) */
+	MP4_PARSE_ERROR_SAMPLE_COUNT_MISMATCH,         /**< Explicit sample count doesn't match implicit sample count */
+	MP4_PARSE_ERROR_UNSUPPORTED_ENCRYPTION_SCHEME, /**< Expected cenc or cbcs */
+	MP4_PARSE_ERROR_INVALID_PADDING,               /**< Unexpected Video Padding field (should be 0xffff) */
+	MP4_PARSE_ERROR_UNSUPPORTED_SAMPLE_ENTRY_COUNT,/**< Zero sample entry count */
 	MP4_PARSE_ERROR_UNSUPPORTED_STREAM_FORMAT,     /**< Unsupported stream format */
 	MP4_PARSE_ERROR_INVALID_ESDS_TAG,              /**< Invalid ESDS tag */
-	MP4_PARSE_ERROR_DATA_BOUNDARY_MISMATCH,        /**< Data boundary mismatch */
-	MP4_PARSE_ERROR_INVALID_INPUT                 /**< Invalid input to parse function */
+	MP4_PARSE_ERROR_DATA_BOUNDARY_MISMATCH,        /**< Data boundary mismatch - referencing invalid memory */
+	MP4_PARSE_ERROR_INVALID_INPUT,                 /**< Invalid input to parse function;  nullptr or zero length */
+	MP4_PARSE_ERROR_INVALID_KID,                   /**< Invalid (huge) kidCount  */
+	MP4_PARSE_ERROR_INVALID_ENTRY_COUNT,           /**< Entry count is zero */
+	MP4_PARSE_ERROR_VARIABLE_LENGTH_OVERFLOW       /**< Value encoded using octets exceed 32 bits */
 };
 
 /**
@@ -143,6 +145,10 @@ class Mp4Demux
 	// Parser state
 	const uint8_t *moofPtr;                       /**< Base address for sample data */
 	const uint8_t *ptr;                           /**< Current parser position */
+	const uint8_t* endPtr;                        /**< Absolute end of current parse buffer */
+	// MDAT range tracking (for sample data validation)
+	const uint8_t *mdatStart;                     /**< Start of current/last mdat payload */
+	const uint8_t *mdatEnd;                       /**< End of current/last mdat payload */
 
 	// Box header fields
 	uint8_t version;                              /**< Box version */
@@ -167,6 +173,12 @@ class Mp4Demux
 	MediaCodecInfo codecInfo;                     /**< Codec information */
 	Mp4ParseError parseError;                     /**< Current parse error state */
 
+	/**
+	 * @brief log human readable parse error and update state
+	 * @param parseError one of Mp4ParseError
+	 */
+	void setParseError( Mp4ParseError );
+
 	/**
 	 * @brief Read n bytes from current position in big-endian format
 	 * @param n Number of bytes to read
@@ -321,7 +333,7 @@ class Mp4Demux
 	 * @brief Read length field with variable encoding
 	 * @return Length value
 	 */
-	int ReadLen();
+	uint32_t ReadLen();
 
 	/**
 	 * @brief Parse codec configuration helper
@@ -383,6 +395,7 @@ class Mp4Demux
 	 * @brief Parse MP4 data
 	 * @param ptr Pointer to MP4 data
 	 * @param len Length of data
+	 * @note endPtr is set to &((const uint8_t*)ptr)[len] for uniform bounds checking
 	 * @return true if parsing succeeded, false on error
 	 */
 	bool Parse(const void *ptr, size_t len);
@@ -428,4 +441,4 @@ class Mp4Demux
 	std::vector<AampMediaSample> GetSamples();
 };
 
-#endif /* __MP4_DEMUX_H__ */
+#endif /* __MP4_DEMUX_H__ */