Sort tokens to workaround Ripper out-of-order tokenization

Fixes #1313

Author: lsegal

lib/yard/parser/ruby/ruby_parser.rb

@@ -684,6 +684,7 @@ def add_comment(line, node = nil, before_node = nil, into = false)
         end
 
         def freeze_tree(node = nil)
+          @tokens = @tokens.sort_by {|t| t.last }
           nodes = [node || root]
           until nodes.empty?
             p_node = nodes.shift

spec/parser/ruby/ruby_parser_spec.rb

@@ -317,6 +317,19 @@ class X
       expect(symbols).to eq %w(:'' :bar :BAR :"B+z" :if)
     end
 
+    # @bug gh-1313
+    it "tokenizes comments in-order" do
+      src = <<-eof
+        def method
+          # Method comment not docstring
+        end
+      eof
+      
+      tokens = tokenize(src.gsub(/^ +/, ''))
+      expect(tokens).to eq(tokens.sort_by {|t| t.last })
+      expect(tokens.map {|t| t.first }).to eq %i(kw sp ident nl comment kw nl)
+    end
+
     it "parses %w() array in constant declaration" do
       s = stmt(<<-eof)
         class Foo