Optimize uri normalization (#4212)

* Add dependency on idn-ruby to speed up URI normalization

* Use normalized_host instead of normalize.host when applicable

When we are only interested in the normalized host, calling normalized_host
avoids normalizing the other components of the URI as well as creating a
new object

diff --git a/Gemfile b/Gemfile
index 5d5ddfae109ccae5ef8ca189f710c33999210bbc..a6c2b2d65c6b5851214138c96f2c4b44ffe86b82 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -36,6 +36,7 @@ gem 'htmlentities', '~> 4.3'
 gem 'http', '~> 2.2'
 gem 'http_accept_language', '~> 2.1'
 gem 'httplog', '~> 0.99'
+gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.0'
 gem 'link_header', '~> 0.0'
 gem 'mime-types', '~> 3.1'

diff --git a/Gemfile.lock b/Gemfile.lock
index daef3e1ad869816009c78a6650d80255c64348e4..f637c9bbe4748feedb2fae4d1345cea8ab522d8f 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -208,6 +208,7 @@ GEM
       parser (>= 2.2.3.0)
       rainbow (~> 2.2)
       terminal-table (>= 1.5.1)
+    idn-ruby (0.1.0)
     jmespath (1.3.1)
     json (2.1.0)
     jsonapi-renderer (0.1.2)
@@ -528,6 +529,7 @@ DEPENDENCIES
   http_accept_language (~> 2.1)
   httplog (~> 0.99)
   i18n-tasks (~> 0.9)
+  idn-ruby
   kaminari (~> 1.0)
   letter_opener (~> 1.4)
   letter_opener_web (~> 1.3)

diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb
index f1a2234dc44aaf5b73866a2040456859c8e24c83..5f87a2a48dcad0a3a855e0868d936f3ed6060f4d 100644 (file)
--- a/app/lib/tag_manager.rb
+++ b/app/lib/tag_manager.rb
@@ -70,7 +70,7 @@ class TagManager
 
     uri = Addressable::URI.new
     uri.host = domain.gsub(/[\/]/, '')
-    uri.normalize.host
+    uri.normalized_host
   end
 
   def same_acct?(canonical, needle)

diff --git a/app/services/concerns/author_extractor.rb b/app/services/concerns/author_extractor.rb
index 00fe1c66393a0b88b1c1fedc129dc12ae1f05b7d..867d6dc25974b960357079e939cd19d531f44eec 100644 (file)
--- a/app/services/concerns/author_extractor.rb
+++ b/app/services/concerns/author_extractor.rb
@@ -14,7 +14,7 @@ module AuthorExtractor
 
       return nil if username.blank? || uri.blank?
 
-      domain = Addressable::URI.parse(uri).normalize.host
+      domain = Addressable::URI.parse(uri).normalized_host
       acct   = "#{username}@#{domain}"
     end
 

diff --git a/app/services/fetch_remote_status_service.rb b/app/services/fetch_remote_status_service.rb
index 4cfd33d90a0b90a8eab249db538eac2d65716946..6ac31e4d8b5baba1aff71dfa0cee96f051327591 100644 (file)
--- a/app/services/fetch_remote_status_service.rb
+++ b/app/services/fetch_remote_status_service.rb
@@ -24,7 +24,7 @@ class FetchRemoteStatusService < BaseService
     xml.encoding = 'utf-8'
 
     account = author_from_xml(xml.at_xpath('/xmlns:entry', xmlns: TagManager::XMLNS))
-    domain  = Addressable::URI.parse(url).normalize.host
+    domain  = Addressable::URI.parse(url).normalized_host
 
     return nil unless !account.nil? && confirmed_domain?(domain, account)
 
@@ -39,6 +39,6 @@ class FetchRemoteStatusService < BaseService
   end
 
   def confirmed_domain?(domain, account)
-    account.domain.nil? || domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url).normalize.host).zero?
+    account.domain.nil? || domain.casecmp(account.domain).zero? || domain.casecmp(Addressable::URI.parse(account.remote_url).normalized_host).zero?
   end
 end

diff --git a/app/workers/pubsubhubbub/delivery_worker.rb b/app/workers/pubsubhubbub/delivery_worker.rb
index 05d160cf7c7e50898309877a56594c92c15dc306..2e1101b93ccdae9b62ab0e3b3cb5170274237445 100644 (file)
--- a/app/workers/pubsubhubbub/delivery_worker.rb
+++ b/app/workers/pubsubhubbub/delivery_worker.rb
@@ -43,7 +43,7 @@ class Pubsubhubbub::DeliveryWorker
   end
 
   def host
-    Addressable::URI.parse(subscription.callback_url).normalize.host
+    Addressable::URI.parse(subscription.callback_url).normalized_host
   end
 
   def headers