]> cat aescling's git repositories - mastodon.git/commitdiff
cat aescling's git repositories / mastodon.git / commitdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | patch | inline | side by side (parent: 4ede517)
Ignore low-confidence CharlockHolmes guesses when parsing link cards (#9510)
authorThibG <thib@sitedethib.com>
Mon, 17 Dec 2018 18:19:45 +0000 (19:19 +0100)
committerEugen Rochko <eugen@zeonfederated.com>
Mon, 17 Dec 2018 18:19:45 +0000 (19:19 +0100)
* Add failing test for windows-1251 link cards

* Ignore low-confidence CharlockHolmes guesses

Fixes #9466

* Fix no method error when charlock holmes cannot detect charset

app/services/fetch_link_card_service.rb patch | blob | history
spec/fixtures/requests/windows-1251.txt [new file with mode: 0644] patch | blob
spec/services/fetch_link_card_service_spec.rb patch | blob | history

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 38c578de29265e87d135286426799f93bd36ddac..7979c312e5d771c84efbca492e495c4e54ccb019 100644 (file)
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService
     detector.strip_tags = true
 
     guess      = detector.detect(@html, @html_charset)
-    page       = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
+    encoding   = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+    page       = Nokogiri::HTML(@html, nil, encoding)
     player_url = meta_property(page, 'twitter:player')
 
     if player_url && !bad_url?(Addressable::URI.parse(player_url))
diff --git a/spec/fixtures/requests/windows-1251.txt b/spec/fixtures/requests/windows-1251.txt
new file mode 100644 (file)
index 0000000..f573e28
--- /dev/null
+++ b/spec/fixtures/requests/windows-1251.txt
@@ -0,0 +1,17 @@
+HTTP/1.1 200 OK\r
+server: nginx\r
+date: Wed, 12 Dec 2018 13:14:03 GMT\r
+content-type: text/html\r
+content-length: 190\r
+accept-ranges: bytes\r
+\r
+<!DOCTYPE html>\r
+<html>\r
+<head>\r
+  <meta http-equiv="Content-Type" content="text/html; charset=windows-1251" />\r
+  <title>����� �����</title>\r
+</head>\r
+<body>\r
+  <p>����� �����</p>\r
+</body>\r
+</html>\r
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 88c5339db44929207a7e831408bcf65da6a3515a..50c60aafd1d6dec13d8b5cd06482da34ca14bd4a 100644 (file)
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -17,6 +17,8 @@ RSpec.describe FetchLinkCardService, type: :service do
     stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
     stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
+    stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
+    stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
 
     subject.call(status)
   end
@@ -57,6 +59,15 @@ RSpec.describe FetchLinkCardService, type: :service do
       end
     end
 
+    context do
+      let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') }
+
+      it 'works with windows-1251' do
+        expect(a_request(:get, 'http://example.com/windows-1251')).to have_been_made.at_least_once
+        expect(status.preview_cards.first.title).to eq('сэмпл текст')
+      end
+    end
+
     context do
       let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }
 
source code fur Kibicat Mastodon