Fix language filter codes (#4841)

* Fix language filter codes

CLD3 returns BCP-47 language identifier, filter settings expect
identifiers in the ISO 639-1 format. Convert between formats,
and exclude duplicate languages from filter choices (zh-CN->zh)

* Fix zh name

diff --git a/Gemfile b/Gemfile
index 486e72cc4ad96a9770a83eb777439a5462ce076c..637bf53eebe21276039fe6baabcb517cb9cd2c0a 100644 (file)
--- a/Gemfile
+++ b/Gemfile
@@ -24,6 +24,7 @@ gem 'addressable', '~> 2.5'
 gem 'bootsnap'
 gem 'browser'
 gem 'charlock_holmes', '~> 0.7.5'
+gem 'iso-639'
 gem 'cld3', '~> 3.1'
 gem 'devise', '~> 4.2'
 gem 'devise-two-factor', '~> 3.0'

diff --git a/Gemfile.lock b/Gemfile.lock
index ef99e0d7b3c69b4503983c1a5306a52e1303fe5d..ddb97dd940adad3e19c9e351a2329ec555858879 100644 (file)
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -225,6 +225,7 @@ GEM
       terminal-table (>= 1.5.1)
     idn-ruby (0.1.0)
     ipaddress (0.8.3)
+    iso-639 (0.2.8)
     jmespath (1.3.1)
     json (2.1.0)
     json-ld (2.1.5)
@@ -560,6 +561,7 @@ DEPENDENCIES
   httplog (~> 0.99)
   i18n-tasks (~> 0.9)
   idn-ruby
+  iso-639
   json-ld-preloaded (~> 2.2.1)
   kaminari (~> 1.0)
   letter_opener (~> 1.4)

diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index af950aa634f27d71e32881d96d2d35aa110f9538..369a456809943e48314287ff62c22804e307c28d 100644 (file)
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -30,6 +30,7 @@ module SettingsHelper
     th: 'ภาษาไทย',
     tr: 'Türkçe',
     uk: 'Українська',
+    zh: '中文',
     'zh-CN': '简体中文',
     'zh-HK': '繁體中文（香港）',
     'zh-TW': '繁體中文（臺灣）',
@@ -39,6 +40,10 @@ module SettingsHelper
     HUMAN_LOCALES[locale]
   end
 
+  def filterable_languages
+    I18n.available_locales.map { |locale| locale.to_s.split('-').first.to_sym }.uniq
+  end
+
   def hash_to_object(hash)
     HashObject.new(hash)
   end

diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb
index cc7509fdc8e9f9eace0644417c88d4389398182c..1d9932b528055e099d9a90da89b179c61ea06014 100644 (file)
--- a/app/lib/language_detector.rb
+++ b/app/lib/language_detector.rb
@@ -20,7 +20,16 @@ class LanguageDetector
   private
 
   def detected_language_code
-    result.language.to_sym if detected_language_reliable?
+    iso6391(result.language).to_sym if detected_language_reliable?
+  end
+
+  def iso6391(bcp47)
+    iso639 = bcp47.split('-').first
+
+    # CLD3 returns grandfathered language code for Hebrew
+    return 'he' if iso639 == 'iw'
+
+    ISO_639.find(iso639).alpha2
   end
 
   def result

diff --git a/app/views/settings/preferences/show.html.haml b/app/views/settings/preferences/show.html.haml
index fae6090c81b31f057a4f7bf1a3b945d578ee108d..f42f92508075effedcb2cb2cc9bf2ee680edc0ca 100644 (file)
--- a/app/views/settings/preferences/show.html.haml
+++ b/app/views/settings/preferences/show.html.haml
@@ -13,7 +13,7 @@
       selected: I18n.locale
 
     = f.input :filtered_languages,
-      collection: I18n.available_locales,
+      collection: filterable_languages,
       wrapper: :with_block_label,
       include_blank: false,
       label_method: lambda { |locale| human_locale(locale) },

diff --git a/spec/helpers/settings_helper_spec.rb b/spec/helpers/settings_helper_spec.rb
index 5a51e0ef1f190b70c945c745f5ff8d38b87818a2..092c37583699f0186a2f03f6670678ba52ab7f51 100644 (file)
--- a/spec/helpers/settings_helper_spec.rb
+++ b/spec/helpers/settings_helper_spec.rb
@@ -4,10 +4,10 @@ require 'rails_helper'
 
 describe SettingsHelper do
   describe 'the HUMAN_LOCALES constant' do
-    it 'has the same number of keys as I18n locales exist' do
+    it 'includes all I18n locales' do
       options = I18n.available_locales
 
-      expect(described_class::HUMAN_LOCALES.keys).to eq(options)
+      expect(described_class::HUMAN_LOCALES.keys).to include(*options)
     end
   end