Fix #4149, fix #1199 - Store emojis as unicode (#4189)

- Use unicode when selecting emoji through picker
- Convert shortcodes to unicode when storing text input server-side
- Do not convert shortcodes in JS anymore

diff --git a/Gemfile b/Gemfile
index 988b4d6b981562e1a33788ab5e8fe3e11a73f0a3..531d01ae059fa587987cebe7ea9978ac2ed1c1a2 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -28,6 +28,7 @@ gem 'devise', '~> 4.2'
 gem 'devise-two-factor', '~> 3.0'
 gem 'doorkeeper', '~> 4.2'
 gem 'fast_blank', '~> 1.0'
+gem 'gemoji', '~> 3.0'
 gem 'goldfinger', '~> 1.2'
 gem 'hiredis', '~> 0.6'
 gem 'redis-namespace', '~> 1.5'

diff --git a/Gemfile.lock b/Gemfile.lock
index 5599e1db1601ade4c9d2bc1477a1e6881846a06f..83202189d0fc0483723203f873eaf475fd9c65b3 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -106,9 +106,9 @@ GEM
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       xpath (~> 2.0)
-    charlock_holmes (0.7.3)
     case_transform (0.2)
       activesupport
+    charlock_holmes (0.7.3)
     chunky_png (1.3.8)
     cld3 (3.1.3)
       ffi (>= 1.1.0, < 1.10.0)
@@ -163,6 +163,7 @@ GEM
     fuubar (2.2.0)
       rspec-core (~> 3.0)
       ruby-progressbar (~> 1.4)
+    gemoji (3.0.0)
     globalid (0.4.0)
       activesupport (>= 4.2.0)
     goldfinger (1.2.0)
@@ -518,6 +519,7 @@ DEPENDENCIES
   faker (~> 1.7)
   fast_blank (~> 1.0)
   fuubar (~> 2.2)
+  gemoji (~> 3.0)
   goldfinger (~> 1.2)
   hamlit-rails (~> 0.2)
   hiredis (~> 0.6)

diff --git a/app/helpers/emoji_helper.rb b/app/helpers/emoji_helper.rb
new file mode 100644 (file)
index 0000000..c159585
--- /dev/null
+++ b/app/helpers/emoji_helper.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module EmojiHelper
+  EMOJI_PATTERN = /(?<=[^[:alnum:]:]|\n|^):([\w+-]+):(?=[^[:alnum:]:]|$)/x
+
+  def emojify(text)
+    return text if text.blank?
+
+    text.gsub(EMOJI_PATTERN) do |match|
+      emoji = Emoji.find_by_alias($1) # rubocop:disable Rails/DynamicFindBy,Style/PerlBackrefs
+
+      if emoji
+        emoji.raw
+      else
+        match
+      end
+    end
+  end
+end

diff --git a/app/javascript/mastodon/actions/compose.js b/app/javascript/mastodon/actions/compose.js
index 647a52b9325454dce84f1530316e3c7e4cf93c75..9f05a53e965f122964b4c47a41c99933622a491a 100644 (file)
--- a/app/javascript/mastodon/actions/compose.js
+++ b/app/javascript/mastodon/actions/compose.js
@@ -2,8 +2,6 @@ import api from '../api';
 
 import { updateTimeline } from './timelines';
 
-import * as emojione from 'emojione';
-
 export const COMPOSE_CHANGE          = 'COMPOSE_CHANGE';
 export const COMPOSE_SUBMIT_REQUEST  = 'COMPOSE_SUBMIT_REQUEST';
 export const COMPOSE_SUBMIT_SUCCESS  = 'COMPOSE_SUBMIT_SUCCESS';
@@ -73,11 +71,14 @@ export function mentionCompose(account, router) {
 
 export function submitCompose() {
   return function (dispatch, getState) {
-    const status = emojione.shortnameToUnicode(getState().getIn(['compose', 'text'], ''));
+    const status = getState().getIn(['compose', 'text'], '');
+
     if (!status || !status.length) {
       return;
     }
+
     dispatch(submitComposeRequest());
+
     api(getState).post('/api/v1/statuses', {
       status,
       in_reply_to_id: getState().getIn(['compose', 'in_reply_to'], null),

diff --git a/app/javascript/mastodon/emoji.js b/app/javascript/mastodon/emoji.js
index 7043d5f3a35b03e8ad5969df0aa9743bb41758e8..ed2180cd1de5274d47799d92157c1ed47f0ceadb 100644 (file)
--- a/app/javascript/mastodon/emoji.js
+++ b/app/javascript/mastodon/emoji.js
@@ -6,36 +6,18 @@ const trie = new Trie(Object.keys(emojione.jsEscapeMap));
 
 function emojify(str) {
   // This walks through the string from start to end, ignoring any tags (<p>, <br>, etc.)
-  // and replacing valid shortnames like :smile: and :wink: as well as unicode strings
+  // and replacing valid unicode strings
   // that _aren't_ within tags with an <img> version.
-  // The goal is to be the same as an emojione.regShortNames/regUnicode replacement, but faster.
+  // The goal is to be the same as an emojione.regUnicode replacement, but faster.
   let i = -1;
   let insideTag = false;
-  let insideShortname = false;
-  let shortnameStartIndex = -1;
   let match;
   while (++i < str.length) {
     const char = str.charAt(i);
-    if (insideShortname && char === ':') {
-      const shortname = str.substring(shortnameStartIndex, i + 1);
-      if (shortname in emojione.emojioneList) {
-        const unicode = emojione.emojioneList[shortname].unicode[emojione.emojioneList[shortname].unicode.length - 1];
-        const alt = emojione.convert(unicode.toUpperCase());
-        const replacement = `<img draggable="false" class="emojione" alt="${alt}" title="${shortname}" src="/emoji/${unicode}.svg" />`;
-        str = str.substring(0, shortnameStartIndex) + replacement + str.substring(i + 1);
-        i += (replacement.length - shortname.length - 1); // jump ahead the length we've added to the string
-      } else {
-        i--; // stray colon, try again
-      }
-      insideShortname = false;
-    } else if (insideTag && char === '>') {
+    if (insideTag && char === '>') {
       insideTag = false;
     } else if (char === '<') {
       insideTag = true;
-      insideShortname = false;
-    } else if (!insideTag && char === ':') {
-      insideShortname = true;
-      shortnameStartIndex = i;
     } else if (!insideTag && (match = trie.search(str.substring(i)))) {
       const unicodeStr = match;
       if (unicodeStr in emojione.jsEscapeMap) {

diff --git a/app/javascript/mastodon/features/compose/components/compose_form.js b/app/javascript/mastodon/features/compose/components/compose_form.js
index f7eeedc69f70c71bed581e5127e9148d464ef6e9..f0755294794c76a7f6e2a8963276bc76093f80b2 100644 (file)
--- a/app/javascript/mastodon/features/compose/components/compose_form.js
+++ b/app/javascript/mastodon/features/compose/components/compose_form.js
@@ -136,7 +136,8 @@ export default class ComposeForm extends ImmutablePureComponent {
 
   handleEmojiPick = (data) => {
     const position     = this.autosuggestTextarea.textarea.selectionStart;
-    this._restoreCaret = position + data.shortname.length + 1;
+    const emojiChar    = String.fromCodePoint(parseInt(data.unicode, 16));
+    this._restoreCaret = position + emojiChar.length + 1;
     this.props.onPickEmoji(position, data);
   }
 

diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
index 83c66a5d59488b599658d21f3298b4976f59da52..acc584f20cfdf51778f5cb34496d634db64d857b 100644 (file)
--- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
+++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js
@@ -109,11 +109,12 @@ export default class EmojiPickerDropdown extends React.PureComponent {
       <Dropdown ref={this.setRef} className='emoji-picker__dropdown' onShow={this.onShowDropdown} onHide={this.onHideDropdown}>
         <DropdownTrigger className='emoji-button' title={intl.formatMessage(messages.emoji)}>
           <img
-            draggable='false'
             className={`emojione ${active && loading ? 'pulse-loading' : ''}`}
-            alt='🙂' src='/emoji/1f602.svg'
+            alt='🙂'
+            src='/emoji/1f602.svg'
           />
         </DropdownTrigger>
+
         <DropdownContent className='dropdown__left'>
           {
             this.state.active && !this.state.loading &&

diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js
index a92b5aa236bd224f331e618c79e0a8af10b3ece0..ea3b78b67a1ec90ee48b7e2c75f38fd0715d2206 100644 (file)
--- a/app/javascript/mastodon/reducers/compose.js
+++ b/app/javascript/mastodon/reducers/compose.js
@@ -118,7 +118,7 @@ const insertSuggestion = (state, position, token, completion) => {
 };
 
 const insertEmoji = (state, position, emojiData) => {
-  const emoji = emojiData.shortname;
+  const emoji = String.fromCodePoint(parseInt(emojiData.unicode, 16));
 
   return state.withMutations(map => {
     map.update('text', oldText => `${oldText.slice(0, position)}${emoji} ${oldText.slice(position)}`);

diff --git a/app/javascript/styles/components.scss b/app/javascript/styles/components.scss
index 0face646dde7b0239ccd984ec1a808f248d1185c..0420a2bedc6343c9e4e353e50dc253fe2a3c8d9f 100644 (file)
--- a/app/javascript/styles/components.scss
+++ b/app/javascript/styles/components.scss
@@ -2708,6 +2708,7 @@ button.icon-button.active i.fa-retweet {
   margin-left: 2px;
   width: 24px;
   outline: 0;
+  cursor: pointer;
 
   &:active,
   &:focus {

diff --git a/app/models/account.rb b/app/models/account.rb
index 2b54cee5fc923f168a986c118a6b56c536399285..7243cb1a58fbac0a9f828f45c686f1265478c966 100644 (file)
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -47,6 +47,7 @@ class Account < ApplicationRecord
   include AccountInteractions
   include Attachmentable
   include Remotable
+  include EmojiHelper
 
   # Local users
   has_one :user, inverse_of: :account
@@ -240,9 +241,18 @@ class Account < ApplicationRecord
 
   before_create :generate_keys
   before_validation :normalize_domain
+  before_validation :prepare_contents, if: :local?
 
   private
 
+  def prepare_contents
+    display_name&.strip!
+    note&.strip!
+
+    self.display_name = emojify(display_name)
+    self.note         = emojify(note)
+  end
+
   def generate_keys
     return unless local?
 

diff --git a/app/models/status.rb b/app/models/status.rb
index 65db7579afc0c8c990e4d24530eb43159d9bb168..24eaf7071f7ed9be9f15c7b4a6cd2ed1f92e9164 100644 (file)
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -29,6 +29,7 @@ class Status < ApplicationRecord
   include Streamable
   include Cacheable
   include StatusThreadingConcern
+  include EmojiHelper
 
   enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility
 
@@ -120,7 +121,7 @@ class Status < ApplicationRecord
     !sensitive? && media_attachments.any?
   end
 
-  before_validation :prepare_contents
+  before_validation :prepare_contents, if: :local?
   before_validation :set_reblog
   before_validation :set_visibility
   before_validation :set_conversation
@@ -241,6 +242,9 @@ class Status < ApplicationRecord
   def prepare_contents
     text&.strip!
     spoiler_text&.strip!
+
+    self.text         = emojify(text)
+    self.spoiler_text = emojify(spoiler_text)
   end
 
   def set_reblog

diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 2e6fbb5c382fb8cf9a7dbb5674629c37aaa5f31a..951a38e1957b26bd525681ef03128c0d18d3aa56 100644 (file)
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -21,6 +21,7 @@ class PostStatusService < BaseService
 
     media  = validate_media!(options[:media_ids])
     status = nil
+
     ApplicationRecord.transaction do
       status = account.statuses.create!(text: text,
                                         thread: in_reply_to,
@@ -31,6 +32,7 @@ class PostStatusService < BaseService
                                         application: options[:application])
       attach_media(status, media)
     end
+
     process_mentions_service.call(status)
     process_hashtags_service.call(status)
 

diff --git a/spec/helpers/emoji_helper_spec.rb b/spec/helpers/emoji_helper_spec.rb
new file mode 100644 (file)
index 0000000..1eedfb7
--- /dev/null
+++ b/spec/helpers/emoji_helper_spec.rb
@@ -0,0 +1,15 @@
+require 'rails_helper'
+
+RSpec.describe EmojiHelper, type: :helper do
+  describe '#emojify' do
+    it 'converts shortcodes to unicode' do
+      text = ':book: Book'
+      expect(emojify(text)).to eq '📖 Book'
+    end
+
+    it 'does not convert shortcodes that are part of a string into unicode' do
+      text = ':see_no_evil::hear_no_evil::speak_no_evil:'
+      expect(emojify(text)).to eq text
+    end
+  end
+end

diff --git a/spec/helpers/routing_helper.rb b/spec/helpers/routing_helper.rb
deleted file mode 100644 (file)
index 3cd3973..0000000
--- a/spec/helpers/routing_helper.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-require 'rails_helper'
-
-RSpec.describe RoutingHelper, type: :helper do
-
-end

diff --git a/spec/javascript/components/emojify.test.js b/spec/javascript/components/emojify.test.js
index 3e8b25af93541f2a6f0e1c8e9e0a7a1c6f8a57ac..e165b4519fc3419e29d1dd587f5eeb603a9c6af4 100644 (file)
--- a/spec/javascript/components/emojify.test.js
+++ b/spec/javascript/components/emojify.test.js
@@ -2,32 +2,6 @@ import { expect } from 'chai';
 import emojify from '../../../app/javascript/mastodon/emoji';
 
 describe('emojify', () => {
-  it('does a basic emojify', () => {
-    expect(emojify(':smile:')).to.equal(
-      '<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" />');
-  });
-
-  it('does a double emojify', () => {
-    expect(emojify(':smile: and :wink:')).to.equal(
-      '<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /> and <img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
-  });
-
-  it('works with random colons', () => {
-    expect(emojify(':smile: : :wink:')).to.equal(
-      '<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /> : <img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
-    expect(emojify(':smile::::wink:')).to.equal(
-      '<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" />::<img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
-    expect(emojify(':smile:::::wink:')).to.equal(
-      '<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" />:::<img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
-  });
-
-  it('works with tags', () => {
-    expect(emojify('<p>:smile:</p>')).to.equal(
-      '<p><img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /></p>');
-    expect(emojify('<p>:smile:</p> and <p>:wink:</p>')).to.equal(
-      '<p><img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /></p> and <p><img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" /></p>');
-  });
-
   it('ignores unknown shortcodes', () => {
     expect(emojify(':foobarbazfake:')).to.equal(':foobarbazfake:');
   });
@@ -46,11 +20,6 @@ describe('emojify', () => {
     expect(emojify(':smile')).to.equal(':smile');
   });
 
-  it('does two emoji next to each other', () => {
-    expect(emojify(':smile::wink:')).to.equal(
-      '<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /><img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
-  });
-
   it('does unicode', () => {
     expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).to.equal(
       '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":family_wwbb:" src="/emoji/1f469-1f469-1f466-1f466.svg" />');
@@ -72,12 +41,7 @@ describe('emojify', () => {
       'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/0023-20e3.svg" /> bar');
   });
 
-  it('does mixed unicode and shortnames', () => {
-    expect(emojify(':smile:#\uFE0F\u20E3:wink:\u2757')).to.equal('<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/0023-20e3.svg" /><img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" /><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
-  });
-
   it('ignores unicode inside of tags', () => {
     expect(emojify('<p data-foo="\uD83D\uDC69\uD83D\uDC69\uD83D\uDC66"></p>')).to.equal('<p data-foo="\uD83D\uDC69\uD83D\uDC69\uD83D\uDC66"></p>');
   });
-
 });