utf8 key names (#133) and other minor fixes

CHANGES.md

@@ -1,6 +1,7 @@
 ## 0.7.13 (unreleased)
 
 * Fix relative keys when controller name consists of more than one word by [Yuji Nakayama](https://github.com/yujinakayama) [#132](https://github.com/glebm/i18n-tasks/pull/132).
+* Support keys with UTF8 word characters in the name. [#133](https://github.com/glebm/i18n-tasks/issues/133).
 
 ## 0.7.12
 

Gemfile

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development do
-  gem 'byebug', platform: :mri_21, require: false
+  gem 'byebug', platforms: [:mri_21,:mri_22], require: false
   gem 'rubinius-debugger', platform: :rbx, require: false
 end
 

lib/i18n/tasks/command/commands/tree.rb

@@ -43,12 +43,11 @@ def tree_filter(opt = {})
         cmd :tree_rename_key,
             args: '<key> <name> [tree]',
             desc: t('i18n_tasks.cmd.desc.tree_rename_key'),
-            opt:  [
-                      cmd_opt(:pattern).merge(short: :k, long: :key=, desc: proc {
-                        t('i18n_tasks.cmd.args.desc.key_pattern_to_rename') }),
-                      cmd_opt(:pattern).merge(short: :n, long: :name=, desc: proc {
-                        t('i18n_tasks.cmd.args.desc.new_key_name') })
-                  ] + cmd_opts(:data_format)
+            opt:  [cmd_opt(:pattern).merge(short: :k, long: :key=,
+                                           desc:  t('i18n_tasks.cmd.args.desc.key_pattern_to_rename')),
+                   cmd_opt(:pattern).merge(short: :n, long: :name=,
+                                           desc:  t('i18n_tasks.cmd.args.desc.new_key_name')),
+                   *cmd_opts(:data_format)]
 
         def tree_rename_key(opt = {})
           key    = opt_or_arg! :key, opt

lib/i18n/tasks/command/options/locales.rb

@@ -43,10 +43,10 @@ def parse_locale(opt, key = :locale)
           opt[key]
         end
 
-        VALID_LOCALE_RE = /\A\w[\w\-\.]*\z/i
-
         def validate_locale!(locale)
-          raise CommandError.new(I18n.t('i18n_tasks.cmd.errors.invalid_locale', invalid: locale)) if VALID_LOCALE_RE !~ locale
+          if Common::VALID_LOCALE_RE !~ locale
+            raise CommandError.new(I18n.t('i18n_tasks.cmd.errors.invalid_locale', invalid: locale))
+          end
         end
       end
     end

lib/i18n/tasks/configuration.rb

@@ -13,7 +13,7 @@ def config
 
   def file_config
     file = CONFIG_FILES.detect { |f| File.exist?(f) }
-    config = file && YAML.load(Erubis::Eruby.new(File.read(file)).result)
+    config = file && YAML.load(Erubis::Eruby.new(File.read(file, encoding: 'UTF-8')).result)
     if config.present?
       config.with_indifferent_access.tap do |c|
         if c[:relative_roots]

lib/i18n/tasks/data/file_formats.rb

@@ -34,7 +34,7 @@ def read_config(format)
         end
 
         def load_file(path)
-          adapter_parse ::File.read(path), self.class.adapter_name_for_path(path)
+          adapter_parse ::File.read(path, encoding: 'UTF-8'), self.class.adapter_name_for_path(path)
         end
 
         def write_tree(path, tree)

lib/i18n/tasks/reports/spreadsheet.rb

@@ -11,7 +11,6 @@ def save_report(path, opts)
       add_missing_sheet p.workbook
       add_unused_sheet p.workbook
       add_eq_base_sheet p.workbook
-      p.use_shared_strings = true
       FileUtils.mkpath(File.dirname(path))
       p.serialize(path)
       $stderr.puts Term::ANSIColor.green "Saved to #{path}"

lib/i18n/tasks/scanners/base_scanner.rb

@@ -60,7 +60,7 @@ def keys(opts = {})
 
     def read_file(path)
       result = nil
-      File.open(path, 'rb') { |f| result = f.read }
+      File.open(path, 'rb', encoding: 'UTF-8') { |f| result = f.read }
       result
     end
 
@@ -127,7 +127,7 @@ def strip_literal(literal)
       key
     end
 
-    VALID_KEY_CHARS = /[-\w.?!;]/
+    VALID_KEY_CHARS = /(?:[[:word:]]|[-.?!;À-ž])/
     VALID_KEY_RE_STRICT = /^#{VALID_KEY_CHARS}+$/
     VALID_KEY_RE = /^(#{VALID_KEY_CHARS}|[:\#{@}\[\]])+$/
 

lib/i18n/tasks/scanners/pattern_scanner.rb

@@ -60,7 +60,7 @@ def controller_file?(path)
     end
 
     def closest_method(location)
-      method = File.readlines(location[:src_path]).first(location[:line_num] - 1).reverse_each.find { |x| x=~ /\bdef\b/ }
+      method = File.readlines(location[:src_path], encoding: 'UTF-8').first(location[:line_num] - 1).reverse_each.find { |x| x=~ /\bdef\b/ }
       method &&= method.strip.sub(/^def\s*/, '').sub(/[\(\s;].*$/, '')
       method
     end

lib/i18n/tasks/used_keys.rb

@@ -11,11 +11,10 @@ module UsedKeys
     # @return [Array<String>]
     def used_tree(opts = {})
       return scanner.with_key_filter(opts[:key_filter]) { used_tree(opts.except(:key_filter)) } if opts[:key_filter]
-      key_attrs = scanner.keys(opts.slice(:strict))
       Data::Tree::Node.new(
           key: 'used',
           data: {key_filter: scanner.key_filter},
-          children: Data::Tree::Siblings.from_key_attr(key_attrs)
+          children: Data::Tree::Siblings.from_key_attr(scanner.keys(opts.slice(:strict)))
       ).to_siblings
     end
 

spec/file_system_data_spec.rb

@@ -95,7 +95,7 @@
         data[:en] = data[:en].merge!('en' => locale_data)
         files     = %w(pizza.en.json sushi.en.json)
         expect(Dir['*.json'].sort).to eq(files.sort)
-        files.each { |f| expect(JSON.parse(File.read f)['en']).to eq({File.basename(f, '.en.json') => keys}) }
+        files.each { |f| expect(JSON.parse(File.read(f, encoding: 'UTF-8'))['en']).to eq({File.basename(f, '.en.json') => keys}) }
       }
     end
   end

spec/fixtures/app/views/index.html.slim

@@ -3,6 +3,7 @@
 / t(:fp_comment)
 / i18n-tasks-use t(:fn_comment)
 = t 'only_in_es'
+#x = t 'not_a_comment'
 p #{t('ca.a')} #{t 'ca.b'} #{t "ca.c"}
 p #{t 'ca.d'} #{t 'ca.f', i: 'world'} #{t 'ca.e', i: 'world'}
 p #{t 'missing_in_es.a'} #{t 'same_in_es.a'} #{t 'blank_in_es.a'}
@@ -25,3 +26,4 @@ p = t 'devise.a'
 p = t :missing_symbol_key
 p #{t :"missing_symbol.key_two"} #{t :'missing_symbol.key_three'}
 = t 'present_in_es_but_not_en.a'
+= t 'latin_extra.çüéö'

spec/i18n_tasks_spec.rb

@@ -198,7 +198,7 @@
 
 
   # --- setup ---
-  BENCH_KEYS = 10
+  BENCH_KEYS = ENV['BENCH_KEYS'].to_i
   before(:each) do
     gen_data = ->(v) {
       v_num = v.chars.map(&:ord).join('').to_i
@@ -230,10 +230,14 @@
           'devise'                 => {'a' => v},
           'scoped'                 => {'x' => v},
           'very'                   => {'scoped' => {'x' => v}},
-          'used'                   => {'a' => v}
+          'used'                   => {'a' => v},
+          'latin_extra'            => {'çüéö' => v},
+          'not_a_comment'          => v
       }.tap { |r|
-        gen = r["bench"] = {}
-        BENCH_KEYS.times { |i| gen["key#{i}"] = v }
+        if BENCH_KEYS > 0
+          gen = r['bench'] = {}
+          BENCH_KEYS.times { |i| gen["key#{i}"] = v }
+        end
       }
     }
 

spec/support/i18n_tasks_output_matcher.rb

@@ -15,7 +15,6 @@ def extract_keys(actual)
     locale_col = 0
     key_col = 1
     actual.map { |row|
-      key =
       key = "#{row[locale_col]}.#{row[key_col]}"
       key = key[0..-2] if key.end_with?(':')
       key