app/services/fetch_link_card_service.rb

   1 # frozen_string_literal: true
   2
   3 class FetchLinkCardService < BaseService
   4   URL_PATTERN = %r{
   5     (                                                                                                 #   $1 URL
   6       (https?:\/\/)                                                                                   #   $2 Protocol (required)
   7       (#{Twitter::Regex[:valid_domain]})                                                              #   $3 Domain(s)
   8       (?::(#{Twitter::Regex[:valid_port_number]}))?                                                   #   $4 Port number (optional)
   9       (/#{Twitter::Regex[:valid_url_path]}*)?                                                         #   $5 URL Path and anchor
  10       (\?#{Twitter::Regex[:valid_url_query_chars]}*#{Twitter::Regex[:valid_url_query_ending_chars]})? #   $6 Query String
  11     )
  12   }iox
  13
  14   def call(status)
  15     @status = status
  16     @url    = parse_urls
  17
  18     return if @url.nil? || @status.preview_cards.any?
  19
  20     @mentions = status.mentions
  21     @url      = @url.to_s
  22
  23     RedisLock.acquire(lock_options) do |lock|
  24       if lock.acquired?
  25         @card = PreviewCard.find_by(url: @url)
  26         process_url if @card.nil? || @card.updated_at <= 2.weeks.ago
  27       else
  28         raise Mastodon::RaceConditionError
  29       end
  30     end
  31
  32     attach_card if @card&.persisted?
  33   rescue HTTP::Error, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
  34     Rails.logger.debug "Error fetching link #{@url}: #{e}"
  35     nil
  36   end
  37
  38   private
  39
  40   def process_url
  41     @card ||= PreviewCard.new(url: @url)
  42
  43     failed = Request.new(:head, @url).perform do |res|
  44       res.code != 405 && res.code != 501 && (res.code != 200 || res.mime_type != 'text/html')
  45     end
  46
  47     return if failed
  48
  49     Request.new(:get, @url).perform do |res|
  50       if res.code == 200 && res.mime_type == 'text/html'
  51         @html = res.body_with_limit
  52         @html_charset = res.charset
  53       else
  54         @html = nil
  55         @html_charset = nil
  56       end
  57     end
  58
  59     return if @html.nil?
  60
  61     attempt_oembed || attempt_opengraph
  62   end
  63
  64   def attach_card
  65     @status.preview_cards << @card
  66     Rails.cache.delete(@status)
  67   end
  68
  69   def parse_urls
  70     if @status.local?
  71       urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[0]).normalize }
  72     else
  73       html  = Nokogiri::HTML(@status.text)
  74       links = html.css('a')
  75       urls  = links.map { |a| Addressable::URI.parse(a['href']).normalize unless skip_link?(a) }.compact
  76     end
  77
  78     urls.reject { |uri| bad_url?(uri) }.first
  79   end
  80
  81   def bad_url?(uri)
  82     # Avoid local instance URLs and invalid URLs
  83     uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
  84   end
  85
  86   def mention_link?(a)
  87     return false if @mentions.nil?
  88     @mentions.any? do |mention|
  89       a['href'] == TagManager.instance.url_for(mention.target)
  90     end
  91   end
  92
  93   def skip_link?(a)
  94     # Avoid links for hashtags and mentions (microformats)
  95     a['rel']&.include?('tag') || a['class']&.include?('u-url') || mention_link?(a)
  96   end
  97
  98   def attempt_oembed
  99     service = FetchOEmbedService.new
 100     embed   = service.call(@url, html: @html)
 101     url     = Addressable::URI.parse(service.endpoint_url)
 102
 103     return false if embed.nil?
 104
 105     @card.type          = embed[:type]
 106     @card.title         = embed[:title]         || ''
 107     @card.author_name   = embed[:author_name]   || ''
 108     @card.author_url    = embed[:author_url].present? ? (url + embed[:author_url]).to_s : ''
 109     @card.provider_name = embed[:provider_name] || ''
 110     @card.provider_url  = embed[:provider_url].present? ? (url + embed[:provider_url]).to_s : ''
 111     @card.width         = 0
 112     @card.height        = 0
 113
 114     case @card.type
 115     when 'link'
 116       @card.image_remote_url = (url + embed[:thumbnail_url]).to_s if embed[:thumbnail_url].present?
 117     when 'photo'
 118       return false if embed[:url].blank?
 119
 120       @card.embed_url        = (url + embed[:url]).to_s
 121       @card.image_remote_url = (url + embed[:url]).to_s
 122       @card.width            = embed[:width].presence  || 0
 123       @card.height           = embed[:height].presence || 0
 124     when 'video'
 125       @card.width            = embed[:width].presence  || 0
 126       @card.height           = embed[:height].presence || 0
 127       @card.html             = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED)
 128       @card.image_remote_url = (url + embed[:thumbnail_url]).to_s if embed[:thumbnail_url].present?
 129     when 'rich'
 130       # Most providers rely on <script> tags, which is a no-no
 131       return false
 132     end
 133
 134     @card.save_with_optional_image!
 135   end
 136
 137   def attempt_opengraph
 138     detector = CharlockHolmes::EncodingDetector.new
 139     detector.strip_tags = true
 140
 141     guess = detector.detect(@html, @html_charset)
 142     page  = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
 143
 144     if meta_property(page, 'twitter:player')
 145       @card.type   = :video
 146       @card.width  = meta_property(page, 'twitter:player:width') || 0
 147       @card.height = meta_property(page, 'twitter:player:height') || 0
 148       @card.html   = content_tag(:iframe, nil, src: meta_property(page, 'twitter:player'),
 149                                                width: @card.width,
 150                                                height: @card.height,
 151                                                allowtransparency: 'true',
 152                                                scrolling: 'no',
 153                                                frameborder: '0')
 154     else
 155       @card.type = :link
 156     end
 157
 158     @card.title            = meta_property(page, 'og:title').presence || page.at_xpath('//title')&.content || ''
 159     @card.description      = meta_property(page, 'og:description').presence || meta_property(page, 'description') || ''
 160     @card.image_remote_url = (Addressable::URI.parse(@url) + meta_property(page, 'og:image')).to_s if meta_property(page, 'og:image')
 161
 162     return if @card.title.blank? && @card.html.blank?
 163
 164     @card.save_with_optional_image!
 165   end
 166
 167   def meta_property(page, property)
 168     page.at_xpath("//meta[@property=\"#{property}\"]")&.attribute('content')&.value || page.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
 169   end
 170
 171   def lock_options
 172     { redis: Redis.current, key: "fetch:#{@url}" }
 173   end
 174 end