Allow keywords to match either substrings or whole words.

Word-boundary matching only works as intended in English and languages
that use similar word-breaking characters; it doesn't work so well in
(say) Japanese, Chinese, or Thai.  It's unacceptable to have a feature
that doesn't work as intended for some languages.  (Moreso especially
considering that it's likely that the largest contingent on the Mastodon
bit of the fediverse speaks Japanese.)

There are rules specified in Unicode TR29[1] for word-breaking across
all languages supported by Unicode, but the rules deliberately do not
cover all cases.  In fact, TR29 states

    For example, reliable detection of word boundaries in languages such
    as Thai, Lao, Chinese, or Japanese requires the use of dictionary
    lookup, analogous to English hyphenation.

So we aren't going to be able to make word detection work with regexes
within Mastodon (or glitchsoc).  However, for a first pass (even if it's
kind of punting) we can allow the user to choose whether they want word
or substring detection and warn about the limitations of this
implementation in, say, docs.

[1]: https://unicode.org/reports/tr29/
     https://web.archive.org/web/20171001005125/https://unicode.org/reports/tr29/

diff --git a/app/models/keyword_mute.rb b/app/models/keyword_mute.rb
index 8b54ad696eb9c4f0a5b931a8afa6492a5ec1b0c1..b0229923d8744528a9c7cec947dc7768837b768b 100644 (file)
--- a/app/models/keyword_mute.rb
+++ b/app/models/keyword_mute.rb
@@ -6,6 +6,7 @@
 #  id         :integer          not null, primary key
 #  account_id :integer          not null
 #  keyword    :string           not null
+#  whole_word :boolean          default(TRUE), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
@@ -32,12 +33,13 @@ class KeywordMute < ApplicationRecord
 
     def initialize(account_id)
       re = [].tap do |arr|
-        KeywordMute.where(account_id: account_id).select(:keyword, :id).find_each do |m|
-          arr << Regexp.escape(m.keyword.strip)
+        KeywordMute.where(account_id: account_id).select(:keyword, :id, :whole_word).find_each do |m|
+          boundary = m.whole_word ? '\b' : ''
+          arr << "#{boundary}#{Regexp.escape(m.keyword.strip)}#{boundary}"
         end
       end.join('|')
 
-      @regex = /\b(?:#{re})\b/i unless re.empty?
+      @regex = /#{re}/i unless re.empty?
     end
 
     def =~(str)

diff --git a/db/migrate/20171009222537_create_keyword_mutes.rb b/db/migrate/20171009222537_create_keyword_mutes.rb
index ee690e799a82a8d00ea8b23ce798def92b60058f..ec0c756fbf3a140a7128abed705776ffd97486ba 100644 (file)
--- a/db/migrate/20171009222537_create_keyword_mutes.rb
+++ b/db/migrate/20171009222537_create_keyword_mutes.rb
@@ -3,6 +3,7 @@ class CreateKeywordMutes < ActiveRecord::Migration[5.1]
     create_table :keyword_mutes do |t|
       t.references :account, null: false
       t.string :keyword, null: false
+      t.boolean :whole_word, null: false, default: true
       t.timestamps
     end
 

diff --git a/db/schema.rb b/db/schema.rb
index 420bb0d2ee2bc4c7aacf24b847dc66737ff5e48c..c0704b13e310098c9e33c545f985c01228a644d9 100644 (file)
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -170,6 +170,7 @@ ActiveRecord::Schema.define(version: 20171010025614) do
   create_table "keyword_mutes", force: :cascade do |t|
     t.bigint "account_id", null: false
     t.string "keyword", null: false
+    t.boolean "whole_word", default: true, null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["account_id"], name: "index_keyword_mutes_on_account_id"

diff --git a/spec/models/keyword_mute_spec.rb b/spec/models/keyword_mute_spec.rb
index de5d32bb42712768f4d2613b1155fc32aecf89c1..c745051886124331a382f5ff671b3069127b2a9b 100644 (file)
--- a/spec/models/keyword_mute_spec.rb
+++ b/spec/models/keyword_mute_spec.rb
@@ -30,10 +30,16 @@ RSpec.describe KeywordMute, type: :model do
         expect(matcher =~ 'This is a hot take').to be_falsy
       end
 
-      it 'does not match substrings matching keywords' do
-        KeywordMute.create!(account: alice, keyword: 'take')
+      it 'considers word boundaries when matching' do
+        KeywordMute.create!(account: alice, keyword: 'bob', whole_word: true)
+
+        expect(matcher =~ 'bobcats').to be_falsy
+      end
+
+      it 'matches substrings if whole_word is false' do
+        KeywordMute.create!(account: alice, keyword: 'take', whole_word: false)
 
-        expect(matcher =~ 'This is a shiitake mushroom').to be_falsy
+        expect(matcher =~ 'This is a shiitake mushroom').to be_truthy
       end
 
       it 'matches keywords at the beginning of the text' do