Issue #2259 - Support Unicode extensions with multiple value subtags in BCP47 language tags

Based-on: m-c 1321789

3 files changed, 168 insertions, 89 deletions

diff --git a/js/src/builtin/intl/Collator.cpp b/js/src/builtin/intl/Collator.cpp
index ab031dde52..ea04b4ee3a 100644
--- a/js/src/builtin/intl/Collator.cpp
+++ b/js/src/builtin/intl/Collator.cpp
@@ -227,6 +227,15 @@ js::intl_availableCollations(JSContext* cx, unsigned argc, Value* vp)
         return false;

 

     uint32_t index = 0;

+

+    // The first element of the collations array must be |null| per

+    // ES2017 Intl, 10.2.3 Internal Slots.

+    if (!DefineElement(cx, collations, index++, NullHandleValue))

+        return false;

+

+    RootedString jscollation(cx);

+    RootedValue element(cx);

+

     for (uint32_t i = 0; i < count; i++) {

         const char* collation = uenum_next(values, nullptr, &status);

         if (U_FAILURE(status)) {

@@ -241,21 +250,12 @@ js::intl_availableCollations(JSContext* cx, unsigned argc, Value* vp)
         if (StringsAreEqual(collation, "standard") || StringsAreEqual(collation, "search"))

             continue;

 

-        // ICU returns old-style keyword values; map them to BCP 47 equivalents

-        // (see http://bugs.icu-project.org/trac/ticket/9620).

-        if (StringsAreEqual(collation, "dictionary"))

-            collation = "dict";

-        else if (StringsAreEqual(collation, "gb2312han"))

-            collation = "gb2312";

-        else if (StringsAreEqual(collation, "phonebook"))

-            collation = "phonebk";

-        else if (StringsAreEqual(collation, "traditional"))

-            collation = "trad";

-

-        RootedString jscollation(cx, JS_NewStringCopyZ(cx, collation));

+        // ICU returns old-style keyword values; map them to BCP 47 equivalents.

+        jscollation = JS_NewStringCopyZ(cx, uloc_toUnicodeLocaleType("co", collation));

+

         if (!jscollation)

             return false;

-        RootedValue element(cx, StringValue(jscollation));

+        element = StringValue(jscollation);

         if (!DefineElement(cx, collations, index++, element))

             return false;

     }

diff --git a/js/src/builtin/intl/CommonFunctions.js b/js/src/builtin/intl/CommonFunctions.js
index c1999f001e..59fbeaea3b 100644
--- a/js/src/builtin/intl/CommonFunctions.js
+++ b/js/src/builtin/intl/CommonFunctions.js
@@ -799,10 +799,7 @@ function LookupMatcher(availableLocales, requestedLocales) {
         if (locale !== noExtensionsLocale) {
             var unicodeLocaleExtensionSequenceRE = getUnicodeLocaleExtensionSequenceRE();
             var extensionMatch = regexp_exec_no_statics(unicodeLocaleExtensionSequenceRE, locale);
-            var extension = extensionMatch[0];
-            var extensionIndex = extensionMatch.index;
-            result.extension = extension;
-            result.extensionIndex = extensionIndex;
+            result.extension = extensionMatch[0];
         }
     } else {
         result.locale = DefaultLocale();
@@ -825,6 +822,77 @@ function BestFitMatcher(availableLocales, requestedLocales) {
     return LookupMatcher(availableLocales, requestedLocales);
 }
 
+/**
+ * Returns the Unicode extension value subtags for the requested key subtag.
+ *
+ * NOTE: PR to add UnicodeExtensionValue to ECMA-402 isn't yet written.
+ */
+function UnicodeExtensionValue(extension, key) {
+    assert(typeof extension === "string", "extension is a string value");
+    assert(function() {
+        var unicodeLocaleExtensionSequenceRE = getUnicodeLocaleExtensionSequenceRE();
+        var extensionMatch = regexp_exec_no_statics(unicodeLocaleExtensionSequenceRE, extension);
+        return extensionMatch !== null && extensionMatch[0] === extension;
+    }(), "extension is a Unicode extension subtag");
+    assert(typeof key === "string", "key is a string value");
+    assert(key.length === 2, "key is a Unicode extension key subtag");
+
+    // Step 1.
+    var size = extension.length;
+
+    // Step 2.
+    var searchValue = "-" + key + "-";
+
+    // Step 3.
+    var pos = callFunction(std_String_indexOf, extension, searchValue);
+
+    // Step 4.
+    if (pos !== -1) {
+        // Step 4.a.
+        var start = pos + 4;
+
+        // Step 4.b.
+        var end = start;
+
+        // Step 4.c.
+        var k = start;
+
+        // Steps 4.d-e.
+        while (true) {
+            // Step 4.e.i.
+            var e = callFunction(std_String_indexOf, extension, "-", k);
+
+            // Step 4.e.ii.
+            var len = e === -1 ? size - k : e - k;
+
+            // Step 4.e.iii.
+            if (len === 2)
+                break;
+
+            // Step 4.e.iv.
+            if (e === -1) {
+                end = size;
+                break;
+            }
+
+            // Step 4.e.v.
+            end = e;
+            k = e + 1;
+        }
+
+        // Step 4.f.
+        return callFunction(String_substring, extension, start, end);
+    }
+
+    // Step 5.
+    searchValue = "-" + key;
+
+    // Steps 6-7.
+    if (callFunction(std_String_endsWith, extension, searchValue))
+        return "";
+
+    // Step 8 (implicit).
+}
 
 /**
  * Compares a BCP 47 language priority list against availableLocales and
@@ -847,19 +915,8 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte
     // Step 4.
     var foundLocale = r.locale;
 
-    // Step 5.a.
+    // Step 5 (Not applicable in this implementation).
     var extension = r.extension;
-    var extensionIndex, extensionSubtags, extensionSubtagsLength;
-
-    // Step 5.
-    if (extension !== undefined) {
-        // Step 5.b.
-        extensionIndex = r.extensionIndex;
-
-        // Steps 5.d-e.
-        extensionSubtags = StringSplitString(ToString(extension), "-");
-        extensionSubtagsLength = extensionSubtags.length;
-    }
 
     // Steps 6-7.
     var result = new Record();
@@ -868,11 +925,11 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte
     // Step 8.
     var supportedExtension = "-u";
 
-    // Steps 9-11.
+    // Steps 9-12.
     var i = 0;
     var len = relevantExtensionKeys.length;
     while (i < len) {
-        // Steps 11.a-c.
+        // Steps 12.a-c.
         var key = relevantExtensionKeys[i];
 
         // In this implementation, localeData is a function, not an object.
@@ -880,51 +937,41 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte
         var keyLocaleData = foundLocaleData[key];
 
         // Locale data provides default value.
-        // Step 11.d.
+        // Step 12.d.
         var value = keyLocaleData[0];
+        assert(typeof value === "string" || value === null, "unexpected locale data value");
 
         // Locale tag may override.
 
-        // Step 11.e.
+        // Step 12.e.
         var supportedExtensionAddition = "";
 
-        // Step 11.f is implemented by Utilities.js.
-
-        var valuePos;
-
-        // Step 11.g.
-        if (extensionSubtags !== undefined) {
-            // Step 11.g.i.
-            var keyPos = callFunction(ArrayIndexOf, extensionSubtags, key);
-
-            // Step 11.g.ii.
-            if (keyPos !== -1) {
-                // Step 11.g.ii.1.
-                if (keyPos + 1 < extensionSubtagsLength &&
-                    extensionSubtags[keyPos + 1].length > 2)
-                {
-                    // Step 11.g.ii.1.a.
-                    var requestedValue = extensionSubtags[keyPos + 1];
-
-                    // Step 11.g.ii.1.b.
-                    valuePos = callFunction(ArrayIndexOf, keyLocaleData, requestedValue);
-
-                    // Step 11.g.ii.1.c.
-                    if (valuePos !== -1) {
+        // Step 12.f.
+        if (extension !== undefined) {
+            // NB: The step annotations don't yet match the ES2017 Intl draft,
+            // 94045d234762ad107a3d09bb6f7381a65f1a2f9b, because the PR to add
+            // the new UnicodeExtensionValue abstract operation still needs to
+            // be written.
+
+            // Step 12.f.i.
+            var requestedValue = UnicodeExtensionValue(extension, key);
+
+            // Step 12.f.ii.
+            if (requestedValue !== undefined) {
+                // Step 12.f.ii.1.
+                if (requestedValue !== "") {
+                    // Step 12.f.ii.1.a.
+                    if (callFunction(ArrayIndexOf, keyLocaleData, requestedValue) !== -1) {
                         value = requestedValue;
                         supportedExtensionAddition = "-" + key + "-" + value;
                     }
                 } else {
-                    // Step 11.g.ii.2.
+                    // Step 12.f.ii.2.
 
                     // According to the LDML spec, if there's no type value,
                     // and true is an allowed value, it's used.
 
-                    // Step 11.g.ii.2.a.
-                    valuePos = callFunction(ArrayIndexOf, keyLocaleData, "true");
-
-                    // Step 11.g.ii.2.b.
-                    if (valuePos !== -1)
+                    if (callFunction(ArrayIndexOf, keyLocaleData, "true") !== -1)
                         value = "true";
                 }
             }
@@ -932,35 +979,54 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte
 
         // Options override all.
 
-        // Step 11.h.i.
+        // Step 12.g.i.
         var optionsValue = options[key];
 
-        // Step 11.h, 11.h.ii.
+        // Step 12.g, 12.gg.ii.
         if (optionsValue !== undefined &&
+            optionsValue !== value &&
             callFunction(ArrayIndexOf, keyLocaleData, optionsValue) !== -1)
         {
-            // Step 11.h.ii.1.
-            if (optionsValue !== value) {
-                value = optionsValue;
-                supportedExtensionAddition = "";
-            }
+            value = optionsValue;
+            supportedExtensionAddition = "";
         }
 
-        // Steps 11.i-k.
+        // Steps 12.h-j.
         result[key] = value;
         supportedExtension += supportedExtensionAddition;
         i++;
     }
 
-    // Step 12.
+    // Step 13.
     if (supportedExtension.length > 2) {
-        var preExtension = callFunction(String_substring, foundLocale, 0, extensionIndex);
-        var postExtension = callFunction(String_substring, foundLocale, extensionIndex);
-        foundLocale = preExtension + supportedExtension + postExtension;
+        assert(!callFunction(std_String_startsWith, foundLocale, "x-"),
+               "unexpected privateuse-only locale returned from ICU");
+
+        // Step 13.a.
+        var privateIndex = callFunction(std_String_indexOf, foundLocale, "-x-");
+
+        // Steps 13.b-c.
+        if (privateIndex === -1) {
+            foundLocale += supportedExtension;
+        } else {
+            var preExtension = callFunction(String_substring, foundLocale, 0, privateIndex);
+            var postExtension = callFunction(String_substring, foundLocale, privateIndex);
+            foundLocale = preExtension + supportedExtension + postExtension;
+        }
+
+        // Step 13.d.
+        assert(IsStructurallyValidLanguageTag(foundLocale), "invalid locale after concatenation");
+
+        // Step 13.e (Not required in this implementation, because we don't
+        // canonicalize Unicode extension subtags).
+        assert(foundLocale === CanonicalizeLanguageTag(foundLocale), "same locale with extension");
+
     }
 
-    // Steps 13-14.
+    // Step 14.
     result.locale = foundLocale;
+
+    // Step 15.
     return result;
 }
 
diff --git a/js/src/builtin/intl/DateTimeFormat.cpp b/js/src/builtin/intl/DateTimeFormat.cpp
index 3e643061c6..c649c3c416 100644
--- a/js/src/builtin/intl/DateTimeFormat.cpp
+++ b/js/src/builtin/intl/DateTimeFormat.cpp
@@ -241,19 +241,16 @@ js::intl_DateTimeFormat_availableLocales(JSContext* cx, unsigned argc, Value* vp
     return true;

 }

 

-// ICU returns old-style keyword values; map them to BCP 47 equivalents

-// (see http://bugs.icu-project.org/trac/ticket/9620).

-static const char*

-bcp47CalendarName(const char* icuName)

+struct CalendarAlias

 {

-    if (StringsAreEqual(icuName, "ethiopic-amete-alem"))

-        return "ethioaa";

-    if (StringsAreEqual(icuName, "gregorian"))

-        return "gregory";

-    if (StringsAreEqual(icuName, "islamic-civil"))

-        return "islamicc";

-    return icuName;

-}

+    const char* const calendar;

+    const char* const alias;

+};

+

+const CalendarAlias calendarAliases[] = {

+    { "islamic-civil", "islamicc" },

+    { "ethioaa", "ethiopic-amete-alem" }

+};

 

 bool

 js::intl_availableCalendars(JSContext* cx, unsigned argc, Value* vp)

@@ -286,7 +283,8 @@ js::intl_availableCalendars(JSContext* cx, unsigned argc, Value* vp)
             return false;

         }

 

-        jscalendar = JS_NewStringCopyZ(cx, bcp47CalendarName(calendar));

+        // ICU returns old-style keyword values; map them to BCP 47 equivalents

+        jscalendar = JS_NewStringCopyZ(cx, uloc_toUnicodeLocaleType("ca", calendar));

         if (!jscalendar)

             return false;

     }

@@ -316,12 +314,27 @@ js::intl_availableCalendars(JSContext* cx, unsigned argc, Value* vp)
             return false;

         }

 

-        jscalendar = JS_NewStringCopyZ(cx, bcp47CalendarName(calendar));

+        // ICU returns old-style keyword values; map them to BCP 47 equivalents

+        calendar = uloc_toUnicodeLocaleType("ca", calendar);

+

+        jscalendar = JS_NewStringCopyZ(cx, calendar);

         if (!jscalendar)

             return false;

         element = StringValue(jscalendar);

         if (!DefineElement(cx, calendars, index++, element))

             return false;

+

+        // ICU doesn't return calendar aliases, append them here.

+        for (const auto& calendarAlias : calendarAliases) {

+            if (StringsAreEqual(calendar, calendarAlias.calendar)) {

+                jscalendar = JS_NewStringCopyZ(cx, calendarAlias.alias);

+                if (!jscalendar)

+                    return false;

+                element = StringValue(jscalendar);

+                if (!DefineElement(cx, calendars, index++, element))

+                    return false;

+            }

+        }

     }

 

     args.rval().setObject(*calendars);