Improve RTL detection (#3682)

- Use plaintext
- Strip out URLs
- Strip out mentions
- Strip out hashtags
- Strip out whitespace from "overall" count
- Consistent between JS and Ruby

diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb
index 275762c871ddfd36632b66a3129366084adf5374..a17b021281f1c071a096511e7c2a223ded2b322e 100644 (file)
--- a/app/helpers/stream_entries_helper.rb
+++ b/app/helpers/stream_entries_helper.rb
@@ -47,11 +47,16 @@ module StreamEntriesHelper
     end
   end
 
+  def rtl_status?(status)
+    status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text))
+  end
+
   def rtl?(text)
+    text = simplified_text(text)
     rtl_characters = /[\p{Hebrew}|\p{Arabic}|\p{Syriac}|\p{Thaana}|\p{Nko}]+/m.match(text)
 
     if rtl_characters.present?
-      total_size = text.strip.size.to_f
+      total_size = text.size.to_f
       rtl_size(rtl_characters.to_a) / total_size > 0.3
     else
       false
@@ -60,6 +65,18 @@ module StreamEntriesHelper
 
   private
 
+  def simplified_text(text)
+    text.dup.tap do |new_text|
+      URI.extract(new_text).each do |url|
+        new_text.gsub!(url, '')
+      end
+
+      new_text.gsub!(Account::MENTION_RE, '')
+      new_text.gsub!(Tag::HASHTAG_RE, '')
+      new_text.gsub!(/\s+/, '')
+    end
+  end
+
   def rtl_size(characters)
     characters.reduce(0) { |acc, elem| acc + elem.size }.to_f
   end

diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js
index 06e25b36a66f529ca661cd41b4ece6dc2d2fa054..d2285428841165460070f1fc9b219a6441df6cbe 100644 (file)
--- a/app/javascript/mastodon/components/status_content.js
+++ b/app/javascript/mastodon/components/status_content.js
@@ -100,7 +100,7 @@ class StatusContent extends React.PureComponent {
     const spoilerContent = { __html: emojify(escapeTextContentForBrowser(status.get('spoiler_text', ''))) };
     const directionStyle = { direction: 'ltr' };
 
-    if (isRtl(status.get('content'))) {
+    if (isRtl(status.get('search_index'))) {
       directionStyle.direction = 'rtl';
     }
 

diff --git a/app/javascript/mastodon/rtl.js b/app/javascript/mastodon/rtl.js
index 8f14bb33823e224e9529dacea84f9f3609e003e9..00870a15d677b48498a70bd12f22fc6d745494b3 100644 (file)
--- a/app/javascript/mastodon/rtl.js
+++ b/app/javascript/mastodon/rtl.js
@@ -17,11 +17,15 @@ export function isRtl(text) {
     return false;
   }
 
+  text = text.replace(/(?:^|[^\/\w])@([a-z0-9_]+(@[a-z0-9\.\-]+)?)/ig, '');
+  text = text.replace(/(?:^|[^\/\w])#([\S]+)/ig, '');
+  text = text.replace(/\s+/g, '');
+
   const matches = text.match(rtlChars);
 
   if (!matches) {
     return false;
   }
 
-  return matches.length / text.trim().length > 0.3;
+  return matches.length / text.length > 0.3;
 };

diff --git a/app/views/stream_entries/_detailed_status.html.haml b/app/views/stream_entries/_detailed_status.html.haml
index a3e2f766eec8cbcb5a3d112e782880bcd8773f5e..ef60b9925a3cfaf9d94647017fdefa2787f0bfa7 100644 (file)
--- a/app/views/stream_entries/_detailed_status.html.haml
+++ b/app/views/stream_entries/_detailed_status.html.haml
@@ -12,7 +12,7 @@
       %p{ style: 'margin-bottom: 0' }<
         %span.p-summary> #{status.spoiler_text}&nbsp;
         %a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more')
-    .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl?(status.content) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
+    .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
 
   - unless status.media_attachments.empty?
     - if status.media_attachments.first.video?

diff --git a/app/views/stream_entries/_simple_status.html.haml b/app/views/stream_entries/_simple_status.html.haml
index 3fa347f74da9e9c8994320f9e81656084e65c7a0..db4e30fda47231cda5242cd450f31c69b67a3605 100644 (file)
--- a/app/views/stream_entries/_simple_status.html.haml
+++ b/app/views/stream_entries/_simple_status.html.haml
@@ -18,7 +18,7 @@
       %p{ style: 'margin-bottom: 0' }<
         %span.p-summary> #{status.spoiler_text}&nbsp;
         %a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more')
-    .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl?(status.content) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
+    .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
 
   - unless status.media_attachments.empty?
     .status__attachments