Merge remote-tracking branch 'mastodon/main' into hota/master

akane-blue · Jul 5, 2024 · 01a7fa0 · 01a7fa0
2 parents 76fc520 + 8f5694d
commit 01a7fa0
Show file tree

Hide file tree

Showing 7 changed files with 103 additions and 11 deletions.
diff --git a/app/javascript/mastodon/features/status/components/card.jsx b/app/javascript/mastodon/features/status/components/card.jsx
@@ -141,7 +141,7 @@ export default class Card extends PureComponent {
     const showAuthor  = !!card.getIn(['authors', 0, 'accountId']);
 
     const description = (
-      <div className='status-card__content'>
+      <div className='status-card__content' dir='auto'>
         <span className='status-card__host'>
           <span lang={language}>{provider}</span>
           {card.get('published_at') && <> · <RelativeTimestamp timestamp={card.get('published_at')} /></>}

diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
@@ -156,11 +156,11 @@ def height
   end
 
   def title
-    html_entities.decode(structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first).strip
+    html_entities_decode(structured_data&.headline || opengraph_tag('og:title') || document.xpath('//title').map(&:content).first)&.strip
   end
 
   def description
-    html_entities.decode(structured_data&.description || opengraph_tag('og:description') || meta_tag('description'))
+    html_entities_decode(structured_data&.description || opengraph_tag('og:description') || meta_tag('description'))
   end
 
   def published_at
@@ -180,15 +180,15 @@ def canonical_url
   end
 
   def provider_name
-    html_entities.decode(structured_data&.publisher_name || opengraph_tag('og:site_name'))
+    html_entities_decode(structured_data&.publisher_name || opengraph_tag('og:site_name'))
   end
 
   def provider_url
     valid_url_or_nil(host_to_url(opengraph_tag('og:site')))
   end
 
   def author_name
-    html_entities.decode(structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username'))
+    html_entities_decode(structured_data&.author_name || opengraph_tag('og:author') || opengraph_tag('og:author:username'))
   end
 
   def author_url
@@ -257,7 +257,7 @@ def structured_data
 
       next if json_ld.blank?
 
-      structured_data = StructuredData.new(html_entities.decode(json_ld))
+      structured_data = StructuredData.new(html_entities_decode(json_ld))
 
       next unless structured_data.valid?
 
@@ -273,10 +273,11 @@ def document
   end
 
   def detect_encoding_and_parse_document
-    [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
+    [detect_encoding, nil, @html_charset].uniq.each do |encoding|
       document = Nokogiri::HTML(@html, nil, encoding)
       return document if document.to_s.valid_encoding?
     end
+    Nokogiri::HTML(@html, nil, 'UTF-8')
   end
 
   def detect_encoding
@@ -290,6 +291,15 @@ def detector
     end
   end
 
+  def html_entities_decode(string)
+    return if string.nil?
+
+    unicode_string = string.encode('UTF-8')
+    raise EncodingError, 'cannot convert string to valid UTF-8' unless unicode_string.valid_encoding?
+
+    html_entities.decode(unicode_string)
+  end
+
   def html_entities
     @html_entities ||= HTMLEntities.new(:expanded)
   end

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
@@ -32,7 +32,7 @@ def call(status)
     end
 
     attach_card if @card&.persisted?
-  rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Encoding::UndefinedConversionError => e
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, EncodingError => e
     Rails.logger.debug { "Error fetching link #{@original_url}: #{e}" }
     nil
   end

diff --git a/spec/fixtures/requests/latin1_posing_as_utf8_broken.txt b/spec/fixtures/requests/latin1_posing_as_utf8_broken.txt
@@ -0,0 +1,17 @@
+HTTP/1.1 200 OK
+server: nginx
+date: Thu, 13 Jun 2024 14:33:13 GMT
+content-type: text/html; charset=utf-8
+content-length: 158
+accept-ranges: bytes
+
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Tofu � l'orange</title>
+</head>
+<body>
+  <h2>Tofu � l'orange</h2>
+</body>
+</html>
diff --git a/spec/fixtures/requests/latin1_posing_as_utf8_recoverable.txt b/spec/fixtures/requests/latin1_posing_as_utf8_recoverable.txt
@@ -0,0 +1,17 @@
+HTTP/1.1 200 OK
+server: nginx
+date: Thu, 13 Jun 2024 14:33:13 GMT
+content-type: text/html; charset=utf-8
+content-length: 158
+accept-ranges: bytes
+
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Tofu with orange sauce</title>
+</head>
+<body>
+  <h2>Tofu � l'orange</h2>
+</body>
+</html>
diff --git a/spec/fixtures/requests/page_without_title.txt b/spec/fixtures/requests/page_without_title.txt
@@ -0,0 +1,17 @@
+HTTP/1.1 200 OK
+server: nginx
+date: Thu, 13 Jun 2024 14:33:13 GMT
+content-type: text/html; charset=utf-8
+content-length: 171
+accept-ranges: bytes
+
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+</head>
+<body>
+  <h2>I am not a valid page</h2>
+  <p>Thankfully, browsers do not care</p>
+</body>
+</html>
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
@@ -27,7 +27,10 @@
     stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
     stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
     stub_request(:get, 'http://example.com/low_confidence_latin1').to_return(request_fixture('low_confidence_latin1.txt'))
+    stub_request(:get, 'http://example.com/latin1_posing_as_utf8_broken').to_return(request_fixture('latin1_posing_as_utf8_broken.txt'))
+    stub_request(:get, 'http://example.com/latin1_posing_as_utf8_recoverable').to_return(request_fixture('latin1_posing_as_utf8_recoverable.txt'))
     stub_request(:get, 'http://example.com/aergerliche-umlaute').to_return(request_fixture('redirect_with_utf8_url.txt'))
+    stub_request(:get, 'http://example.com/page_without_title').to_return(request_fixture('page_without_title.txt'))
 
     Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache
 
@@ -110,6 +113,14 @@
       end
     end
 
+    context 'with a page that has no title' do
+      let(:status) { Fabricate(:status, text: 'http://example.com/page_without_title') }
+
+      it 'does not create a preview card' do
+        expect(status.preview_card).to be_nil
+      end
+    end
+
     context 'with a 404 URL' do
       let(:status) { Fabricate(:status, text: 'http://example.com/not-found') }
 
@@ -159,10 +170,30 @@
     end
 
     context 'with a URL of a page in ISO-8859-1 encoding, that charlock_holmes cannot detect' do
-      let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') }
+      context 'when encoding in http header is correct' do
+        let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') }
 
-      it 'decodes the HTML' do
-        expect(status.preview_card.title).to eq("Tofu á l'orange")
+        it 'decodes the HTML' do
+          expect(status.preview_card.title).to eq("Tofu á l'orange")
+        end
+      end
+
+      context 'when encoding in http header is incorrect' do
+        context 'when encoding problems appear in unrelated tags' do
+          let(:status) { Fabricate(:status, text: 'Check out http://example.com/latin1_posing_as_utf8_recoverable') }
+
+          it 'decodes the HTML' do
+            expect(status.preview_card.title).to eq('Tofu with orange sauce')
+          end
+        end
+
+        context 'when encoding problems appear in title tag' do
+          let(:status) { Fabricate(:status, text: 'Check out http://example.com/latin1_posing_as_utf8_broken') }
+
+          it 'does not create a preview card' do
+            expect(status.preview_card).to be_nil
+          end
+        end
       end
     end