[WIP] Update collation

Rakefile

@@ -7,6 +7,7 @@ require 'rubygems' unless ENV['NO_RUBYGEMS']
 
 require 'bundler'
 require 'digest'
+require 'fileutils'
 
 require 'rspec/core/rake_task'
 require 'rubygems/package_task'
@@ -55,6 +56,7 @@ task :update do
     [
       "update:tailoring_data",  # per locale
       "update:collation_tries", # per locale, must come after update:tailoring_data
+      "update:tailoring_tests", # per locale, if available in CLDR v21, must come after tailoring_data and collation_tries
       "update:rbnf_tests",      # per locale
     ]
   else
@@ -85,6 +87,13 @@ end
 
 # TODO: 'add_locale' task that creates a new directory and runs all necessary 'update' tasks (+ suggests to run those that depend on JRuby)
 
+task :clean_vendored do
+  Dir.glob('./vendor/*') do |file_or_dir|
+    puts "Removing #{file_or_dir}"
+    FileUtils.rm_rf(file_or_dir)
+  end
+end
+
 namespace :update do
   ICU_JAR = './vendor/icu4j.jar'
 
@@ -112,6 +121,15 @@ namespace :update do
     ).import(TwitterCldr.supported_locales)
   end
 
+  desc 'Import tailoring tests from CLDR data (should be executed using JRuby 1.7 in 1.9 mode)'
+  task :tailoring_tests, :cldr_path, :icu4j_jar_path do |_, args|
+    TwitterCldr::Resources::TailoringTestsImporter.new(
+      args[:cldr_path] || './vendor/cldr_for_collation',
+      './spec/collation/tailoring_tests',
+      args[:icu4j_jar_path] || ICU_JAR
+    ).import(TwitterCldr.supported_locales)
+  end
+
   desc 'Import Unicode data resources'
   task :unicode_data, :unicode_data_path do |_, args|
     TwitterCldr::Resources::UnicodeDataImporter.new(

lib/twitter_cldr/resources.rb

@@ -19,6 +19,7 @@ module Resources
     autoload :PhoneCodesImporter,                  'twitter_cldr/resources/phone_codes_importer'
     autoload :PostalCodesImporter,                 'twitter_cldr/resources/postal_codes_importer'
     autoload :TailoringImporter,                   'twitter_cldr/resources/tailoring_importer'
+    autoload :TailoringTestsImporter,              'twitter_cldr/resources/tailoring_tests_importer'
     autoload :UnicodeDataImporter,                 'twitter_cldr/resources/unicode_data_importer'
     autoload :UnicodePropertiesImporter,           'twitter_cldr/resources/unicode_properties_importer'
     autoload :UnicodeScriptsImporter,              'twitter_cldr/resources/unicode_scripts_importer'

lib/twitter_cldr/resources/download.rb

@@ -6,8 +6,16 @@
 module TwitterCldr
   module Resources
 
-    CLDR_URL = 'http://unicode.org/Public/cldr/26/core.zip'
-    ICU4J_URL = 'http://download.icu-project.org/files/icu4j/54.1/icu4j-54_1.jar'
+    # CLDR_VERSION    = '26'
+    # ICU_VERSION     = '54.1.1'
+    UNICODE_VERSION = '6.1.0'
+
+    CLDR_VERSION    = '24'
+    ICU_VERSION     = '52.1'
+
+    CLDR_URL    = "http://unicode.org/Public/cldr/#{CLDR_VERSION}/core.zip"
+    ICU4J_URL   = "http://download.icu-project.org/files/icu4j/#{ICU_VERSION}/icu4j-#{ICU_VERSION.gsub('.', '_')}.jar"
+    UNICODE_URL = "ftp://ftp.unicode.org/Public/#{UNICODE_VERSION}"
 
     # Use these instead to update collation and tailoring data
     # CLDR_URL = 'http://unicode.org/Public/cldr/23.1/core.zip'

lib/twitter_cldr/resources/tailoring_importer.rb

@@ -137,38 +137,69 @@ def get_default_collation_type(collations)
       end
 
       def parse_tailorings(data, locale)
-        rules = data && data.at_xpath('rules')
-
-        return '' unless rules
+        if data && cr = data.at_xpath('cr')
+          rules = cr.text.strip.split("\n").map(&:strip)
+        else
+          return ''
+        end
 
         collator = Java::ComIbmIcuText::Collator.get_instance(Java::JavaUtil::Locale.new(locale.to_s))
 
-        rules.children.map do |child|
-          validate_tailoring_rule(child)
-
-          if child.name =~ LEVEL_RULE_REGEXP
-            if $2.empty?
-              table_entry_for_rule(collator, child.text)
+        rules.map do |rule_text|
+          rules = parse_rule(rule_text)
+
+          rules.each do |rule|
+            level, text = rule
+            # validate_tailoring_rule(rule)
+
+            # if child.name =~ LEVEL_RULE_REGEXP
+            if level =~ LEVEL_RULE_REGEXP
+              if $2.empty?
+                # table_entry_for_rule(collator, child.text)
+                table_entry_for_rule(collator, text)
+              else
+                # child.text.chars.map { |char| table_entry_for_rule(collator, char) }
+                text.chars.map { |char| table_entry_for_rule(collator, char) }
+              end
+            # elsif child.name == 'x'
+            #   context = ''
+            #   child.children.inject([]) do |memo, c|
+            #     if SIMPLE_RULES.include?(c.name)
+            #       memo << table_entry_for_rule(collator, context + c.text)
+            #     elsif c.name == 'context'
+            #       context = c.text
+            #     elsif c.name != 'extend'
+            #       raise ImportError, "Rule '#{c.name}' inside <x></x> is not supported."
+            #     end
+
+            #     memo
+            #   end
             else
-              child.text.chars.map { |char| table_entry_for_rule(collator, char) }
+              raise ImportError, "Tag '#{level}' is not supported." unless IGNORED_TAGS.include?(level)
+              # raise ImportError, "Tag '#{child.name}' is not supported." unless IGNORED_TAGS.include?(child.name)
             end
-          elsif child.name == 'x'
-            context = ''
-            child.children.inject([]) do |memo, c|
-              if SIMPLE_RULES.include?(c.name)
-                memo << table_entry_for_rule(collator, context + c.text)
-              elsif c.name == 'context'
-                context = c.text
-              elsif c.name != 'extend'
-                raise ImportError, "Rule '#{c.name}' inside <x></x> is not supported."
-              end
+          end
+        end.flatten.compact.join("\n")
+      end
 
-              memo
+      def parse_rule(rule_text)
+        puts rule_text
+        tokens = rule_text.split(/([&=]|[<]+)/).reject(&:empty?)
+        level = nil
+
+        [].tap do |rules|
+          tokens.each do |token|
+            case token
+              when '<' then level = 'p'
+              when '<<' then level = 's'
+              when '<<<' then level = 't'
+              when '=' then level = 'i'
+              when '&'
+              else
+                rules << [level, token] if level
             end
-          else
-            raise ImportError, "Tag '#{child.name}' is not supported." unless IGNORED_TAGS.include?(child.name)
           end
-        end.flatten.compact.join("\n")
+        end
       end
 
       def table_entry_for_rule(collator, tailored_value)

lib/twitter_cldr/resources/tailoring_tests_importer.rb

@@ -0,0 +1,84 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'cgi'
+require 'java'
+require 'nokogiri'
+require 'twitter_cldr/resources/download'
+
+module TwitterCldr
+  module Resources
+
+    # This class should be used with JRuby 1.7 in 1.9 mode, ICU4J version >= 49.1
+    class TailoringTestsImporter < IcuBasedImporter
+      CLDR_URL = "http://unicode.org/cldr/trac/export/12161/tags/release-21-d02/test"
+
+      # Arguments:
+      #
+      #   input_path  - path to a directory containing CLDR data
+      #   output_path - output directory for imported YAML files
+      #   icu4j_path  - path to ICU4J jar file
+      #
+      def initialize(input_path, output_path, icu4j_path)
+        require_icu4j(icu4j_path)
+
+        @input_path  = input_path
+        @output_path = output_path
+      end
+
+      def import(locales)
+        locales.each { |locale| import_locale(locale) }
+      end
+
+      private
+
+      def import_locale(locale)
+        download_test_for_locale_if_necessary(locale)
+
+        if tailoring_test_present?(locale)
+          doc = Nokogiri::XML(File.read(input_file_path(locale)))
+          results = doc.xpath('//collation/result').flat_map do |result|
+            CGI.unescapeHTML(result.text).split("\n")
+          end
+
+          results = results.uniq.reject(&:empty?)
+          results = sort_results(locale, results)
+
+          File.open(output_file_path(locale), 'w+') do |f|
+            f.write(results.join("\n"))
+          end
+        end
+      end
+
+      def sort_results(locale, results)
+        collator = Java::ComIbmIcuText::Collator.get_instance(
+          Java::JavaUtil::Locale.new(locale.to_s)
+        )
+
+        results.sort { |a, b| collator.compare(a, b) }
+      end
+
+      def download_test_for_locale_if_necessary(locale)
+        TwitterCldr::Resources.download_if_necessary(
+          input_file_path(locale), "#{CLDR_URL}/#{locale}.xml"
+        )
+      end
+
+      def tailoring_test_present?(locale)
+        File.file?(input_file_path(locale))
+      end
+
+      def input_file_path(locale)
+        File.join(@input_path, "#{locale}.xml")
+      end
+
+      def output_file_path(locale)
+        File.join(@output_path, "#{locale}.txt")
+      end
+
+    end
+
+  end
+end