Change language detection to include hashtags as words (#11341)

diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index 1e90af42d7abfd5b6664a45577eb8e8046de2f3c..6f9511a54192a70cbb646ec371f6d7e2102e36f2 100644 (file)
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -69,7 +69,7 @@ class LanguageDetector
     new_text = remove_html(text)
     new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
     new_text.gsub!(Account::MENTION_RE, '')
-    new_text.gsub!(Tag::HASHTAG_RE, '')
+    new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
     new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')
     new_text.gsub!(/\s+/, ' ')
     new_text

diff --git a/spec/lib/language_detector_spec.rb b/spec/lib/language_detector_spec.rb
index 0cb70605ad411536b59c366cb4b8c7f25784c89a..b7ba0f6c4f4ac9e924176b40991d8f73e7739668 100644 (file)
--- a/spec/lib/language_detector_spec.rb
+++ b/spec/lib/language_detector_spec.rb
@@ -32,11 +32,11 @@ describe LanguageDetector do
       expect(result).to eq 'Our website is and also'
     end
 
-    it 'strips #hashtags from strings before detection' do
-      string = 'Hey look at all the #animals and #fish'
+    it 'converts #hashtags back to normal text before detection' do
+      string = 'Hey look at all the #animals and #FishAndChips'
 
       result = described_class.instance.send(:prepare_text, string)
-      expect(result).to eq 'Hey look at all the and'
+      expect(result).to eq 'Hey look at all the animals and fish and chips'
     end
   end