Remove usernames and hashtags from language detection (#3503)

* Add failing specs for hashtag and username extraction in language detector

* Remove usernames and hashtags from text before language detection

* Handle multiple instances of special case, and reduce whitespace

diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index 1c22a9ccc54e3faaf34521a566f6ff2dd3b32127..0d087f8d1e3552aa56c5bd91576ba9ac62c9ab2c 100644 (file)
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -13,6 +13,10 @@ class LanguageDetector
     detected_language_code || default_locale.to_sym
   end
 
+  def prepared_text
+    simplified_text.strip
+  end
+
   private
 
   def detected_language_code
@@ -20,18 +24,21 @@ class LanguageDetector
   end
 
   def result
-    @result ||= @identifier.find_language(text_without_urls)
+    @result ||= @identifier.find_language(prepared_text)
   end
 
   def detected_language_reliable?
     result.reliable?
   end
 
-  def text_without_urls
+  def simplified_text
     text.dup.tap do |new_text|
       URI.extract(new_text).each do |url|
         new_text.gsub!(url, '')
       end
+      new_text.gsub!(Account::MENTION_RE, '')
+      new_text.gsub!(Tag::HASHTAG_RE, '')
+      new_text.gsub!(/\s+/, ' ')
     end
   end
 

diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb
index e543edd49694c6bcf26f6ea8672d5080989c6cf1..ace7a326af165d7f2cc49c6a44b2d9d72f7ee031 100644 (file)
--- a/spec/lib/language_detector_spec.rb
+++ b/spec/lib/language_detector_spec.rb
@@ -1,7 +1,45 @@
 # frozen_string_literal: true
+
 require 'rails_helper'
 
 describe LanguageDetector do
+  describe 'prepared_text' do
+    it 'returns unmodified string without special cases' do
+      string = 'just a regular string'
+      result = described_class.new(string).prepared_text
+
+      expect(result).to eq string
+    end
+
+    it 'collapses spacing in strings' do
+      string = 'The formatting   in    this is very        odd'
+
+      result = described_class.new(string).prepared_text
+      expect(result).to eq 'The formatting in this is very odd'
+    end
+
+    it 'strips usernames from strings before detection' do
+      string = '@username Yeah, very surreal...! also @friend'
+
+      result = described_class.new(string).prepared_text
+      expect(result).to eq 'Yeah, very surreal...! also'
+    end
+
+    it 'strips URLs from strings before detection' do
+      string = 'Our website is https://example.com and also http://localhost.dev'
+
+      result = described_class.new(string).prepared_text
+      expect(result).to eq 'Our website is and also'
+    end
+
+    it 'strips #hashtags from strings before detection' do
+      string = 'Hey look at all the #animals and #fish'
+
+      result = described_class.new(string).prepared_text
+      expect(result).to eq 'Hey look at all the and'
+    end
+  end
+
   describe 'to_iso_s' do
     it 'detects english language for basic strings' do
       strings = [