Overlapped auto-generated subtitles

[VahidN]

There's a serious usability issue with current produced .srt files. Suppose this is one the Youtube's auto-generated transcripts:

<transcript>
  <text start="0.06" dur="4.44">next come to use seven warm-up period is</text>
  <text start="2.34" dur="4.68">not necessarily 30 minutes and I promise</text>
</transcript>

Which has a lot of overlapped items. Item 2 will start before finishing the item 1 and … this is really annoying and makes it hard to use and follow. It's nearly unusable! You are watching a video and subtitle items are cutting, covering and overlapping each other constantly!

Here is a solution for this problem which can be used as an idea. If the next subtitle item overlaps the previous one, it will set the EndTs of the current subtitle to StartTs of the next subtitle-10 milliseconds.

using System.Text;
using System.Web;
using System.Xml.Linq;

public static class GoogleXmlSubtitleConverter
{
    public static string ConvertXmlToSrt(string xmlContent)
    {
        var transcript = LoadTranscript(xmlContent);
        if (transcript.Root is null)
        {
            throw new InvalidOperationException($"couldn't load received data: {xmlContent}");
        }

        var srtBuf = new StringBuilder();
        var lineNumber = 0;

        var xElements = transcript.Root.Elements("text").ToList();
        for (var currentIndex = 0; currentIndex < xElements.Count; currentIndex++)
        {
            var currentItem = xElements[currentIndex];
            var text = DecodeText(currentItem);
            if (string.IsNullOrWhiteSpace(text))
            {
                // skip over empty items
                continue;
            }
            
            lineNumber++;
            srtBuf.Append(lineNumber.ToString(CultureInfo.InvariantCulture));

            var (currentStartTs, currentEndTs) = GetTimeLine(currentItem);
            currentEndTs = GetNonOverlappedEndTs(xElements, currentIndex, currentEndTs);
            srtBuf.Append(CultureInfo.InvariantCulture,
                          $"{Environment.NewLine}{TimeSpanToString(currentStartTs)} --> {TimeSpanToString(currentEndTs)}{Environment.NewLine}");

            srtBuf.Append(text).AppendLine(Environment.NewLine);
        }

        return srtBuf.ToString();
    }

    private static TimeSpan GetNonOverlappedEndTs(IReadOnlyList<XElement> xElements,
                                                  int currentIndex,
                                                  TimeSpan currentEndTs)
    {
        var nextIndex = currentIndex + 1;
        while (nextIndex < xElements.Count)
        {
            var nextItem = xElements[nextIndex];
            nextIndex++;

            var nextItemText = DecodeText(nextItem);
            if (string.IsNullOrWhiteSpace(nextItemText))
            {
                // skip over empty items
                continue;
            }

            var (nextStartTs, _) = GetTimeLine(nextItem);
            // If the next subtitle overlaps the previous one
            if (currentEndTs > nextStartTs)
            {
                // Set the EndTs of the current subtitle to StartTs of the next subtitle-10 milliseconds if they overlap time-wise.
                currentEndTs = nextStartTs - TimeSpan.FromMilliseconds(10);
            }

            break;
        }

        return currentEndTs;
    }

    private static XDocument LoadTranscript(string content)
    {
        var transcript = XDocument.Parse(content);

        if (transcript.Root == null)
        {
            throw new InvalidOperationException("couldn't load received data.");
        }

        return transcript;
    }

    private static string DecodeText(XElement text)
    {
        var textValue = text.Value.Split('\n');
        var msg = textValue.Aggregate(
                                      string.Empty,
                                      (current, line) =>
                                          current + HttpUtility.HtmlDecode(line.Trim()) + Environment.NewLine
                                     );
        return msg.Trim();
    }

    private static int FractionalPart(string data)
    {
        var idx = data.IndexOf(".", StringComparison.Ordinal);
        if (idx == -1)
        {
            return 0;
        }

        var fractionalPart = data.Substring(idx + 1);
        return fractionalPart.Length switch
               {
                   //3.4 --> 3.400
                   1 => int.Parse($"{fractionalPart}00", CultureInfo.InvariantCulture),
                   //3.04 --> 3.040
                   2 => int.Parse($"{fractionalPart}0", CultureInfo.InvariantCulture),
                   _ => int.Parse(fractionalPart.AsSpan(0, 3), CultureInfo.InvariantCulture),
               };
    }

    private static (TimeSpan StartTs, TimeSpan EndTs) GetTimeLine(XElement text)
    {
        var startTs = GetStartTs(text);
        var endTs = GetEndTs(text, startTs);
        return (startTs, endTs);
    }

    private static TimeSpan GetStartTs(XElement text)
    {
        var startData = text.Attribute("start");
        if (startData == null)
        {
            throw new InvalidOperationException("This is not a valid subtitle file.");
        }

        var start = startData.Value;
        var startTs = new TimeSpan(0,
                                   0,
                                   0,
                                   (int)Math.Truncate(double.Parse(start, CultureInfo.InvariantCulture)),
                                   FractionalPart(start));
        return startTs;
    }

    private static TimeSpan GetEndTs(XElement text, TimeSpan startTs)
    {
        TimeSpan endTs;
        var durData = text.Attribute("dur");
        if (durData == null)
        {
            endTs = startTs + new TimeSpan(0, 0, 0, 3, 0);
        }
        else
        {
            var dur = durData.Value;
            var durTs = new TimeSpan(0,
                                     0,
                                     0,
                                     (int)Math.Truncate(double.Parse(dur, CultureInfo.InvariantCulture)),
                                     FractionalPart(dur));
            endTs = startTs + durTs;
        }

        return endTs;
    }

    private static string TimeSpanToString(TimeSpan lineTs) =>
        Invariant($"{lineTs.Hours:D2}:{lineTs.Minutes:D2}:{lineTs.Seconds:D2},{lineTs.Milliseconds:D3}");
}

[Tyrrrz]

The produced SRT is 1:1 mapping from YouTube's own subtitles, which is by design. You can get the structured closed caption data using Youtube.Videos.ClosedCaptions.GetAsync(...) and then write it to SRT yourself with necessary adjustments. The format is pretty simple so it shouldn't take too much effort.

Note that video players tend to handle overlapped subtitles well, either by actually overlapping them, or by eliminating the overlap as you described. At least that's what I concluded when testing with VLC.