Fix date time format in ICU4J, ICU4C, ICU4X (#297)

* Basic lint applied

* Remove unneeded include

* Updating to use Z as timezone in generator and in Java

* ICU4J date time format - add calendar, fixing many issues

* Fix date/time generator to use instant. Remove 'und' locale

* Fix ICU4C DateTime parsing of 'Z', resolving many test failures

* Fix Rust date/time to accept new UTC instant data. Still not handling timezones correctly

* formatted .rs file

* Removing debug printing from datatime_fmt.cpp

* Add tz_offset_secs to date/time test data so ICU4X can use timezone

* Trying to fix timezone setting in ICU4X datetime

* Fixing about 1/2 of the ICU4X test failures

* Update based on feedback

* Moving code lines as suggested

Author: sven-oly

executors/cpp/datetime_fmt.cpp

@@ -48,7 +48,9 @@ const string TestDatetimeFmt(json_object *json_in) {
   string label_string = json_object_get_string(label_obj);
 
   Calendar *cal = nullptr;
-  TimeZone *tz = nullptr;
+
+  UnicodeString u_tz_utc("UTC");
+  TimeZone *tz = nullptr;  // TimeZone::createTimeZone(u_tz_utc);
 
   // The locale for formatted output
   json_object *locale_label_obj = json_object_object_get(json_in, "locale");
@@ -65,46 +67,55 @@ const string TestDatetimeFmt(json_object *json_in) {
   json_object *return_json = json_object_new_object();
   json_object_object_add(return_json, "label", label_obj);
 
-  string calendar_str;
+  string calendar_str = "gregory";
 
   // Get fields out of the options if present
   json_object* options_obj = json_object_object_get(json_in, "options");
 
   if (options_obj) {
-    json_object* cal_item = json_object_object_get(options_obj, "calendar");
+    // Check for timezone and calendar
+    json_object* option_item =
+        json_object_object_get(options_obj, "timeZone");
+    if (option_item) {
+      string timezone_str = json_object_get_string(option_item);
+      UnicodeString u_tz(timezone_str.c_str());
+      tz = TimeZone::createTimeZone(u_tz);
+    }
+
+    json_object* cal_item =
+        json_object_object_get(options_obj, "calendar");
     if (cal_item) {
       calendar_str = json_object_get_string(cal_item);
-
-      // Add '@calendar=' + calendar_string to locale
-      locale_string = locale_string + "@calendar=" + calendar_str;
-      display_locale = locale_string.c_str();
-
-      if (tz) {
-        cal = Calendar::createInstance(*tz, display_locale, status);
-      } else {
-        cal = Calendar::createInstance(display_locale, status);
-      }
-      if (U_FAILURE(status)) {
-        json_object_object_add(
-            return_json,
-            "error",
-            json_object_new_string("Error in createInstance for calendar"));
-        return json_object_to_json_string(return_json);
-      }
     }
   }
 
+  // Add '@calendar=' + calendar_string to locale
+  locale_string = locale_string + "@calendar=" + calendar_str;
+  display_locale = locale_string.c_str();
+
+  if (tz) {
+    cal = Calendar::createInstance(tz, display_locale, status);
+  } else {
+    cal = Calendar::createInstance(display_locale, status);
+  }
+  if (U_FAILURE(status)) {
+    json_object_object_add(
+        return_json,
+        "error",
+        json_object_new_string("Error in createInstance for calendar"));
+    return json_object_to_json_string(return_json);
+  }
+
   DateFormat* df;
 
 
   // Get the input data as a date object.
   // Types of input:
-  //   "input_string" parsable ISO formatted string such as
-  //       "2020-03-02 10:15:17 -08:00"
+  //   "input_string" parsable ISO formatted string of an instant
+  //       "2020-03-02 10:15:17Z
 
   string dateStyle_str;
   string timeStyle_str;
-  string timezone_str;
 
   // Expected values if neither dateStyle nor timeStyle is given explicitly.
   icu::DateFormat::EStyle date_style = icu::DateFormat::EStyle::kNone;
@@ -126,17 +137,6 @@ const string TestDatetimeFmt(json_object *json_in) {
       timeStyle_str = json_object_get_string(option_item);
       time_style = StringToEStyle(timeStyle_str);
     }
-
-    option_item = json_object_object_get(options_obj, "timeZone");
-    if (option_item) {
-      timezone_str = json_object_get_string(option_item);
-      UnicodeString u_tz(timezone_str.c_str());
-      tz = TimeZone::createTimeZone(u_tz);
-    } else {
-      // Default is UTC
-      UnicodeString u_tz("UTC");
-      tz = TimeZone::createTimeZone(u_tz);
-    }
   }
 
   json_object *date_skeleton_obj =
@@ -183,10 +183,9 @@ const string TestDatetimeFmt(json_object *json_in) {
   }
 
   // !!! IS OFFSET ALREADY CONSIDERED?
-  // if (tz) {
-  //   df->setTimeZone(*tz);
-  // }
-
+  if (tz) {
+    df->setTimeZone(*tz);
+  }
 
   // Use ISO string form of the date/time.
   json_object *input_string_obj =
@@ -219,7 +218,7 @@ const string TestDatetimeFmt(json_object *json_in) {
     UnicodeString date_ustring(input_date_string.c_str());
 
     // TODO:  handles the offset +/-
-    SimpleDateFormat iso_date_fmt(u"y-M-d'T'h:m:s", und_locale, status);
+    SimpleDateFormat iso_date_fmt(u"y-M-d'T'h:m:sZ", und_locale, status);
     if (U_FAILURE(status)) {
       string error_name = u_errorName(status);
       string error_message =

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterDateStyle.java

@@ -4,9 +4,10 @@ public enum DateTimeFormatterDateStyle {
   FULL,
   LONG,
   MEDIUM,
-  SHORT;
+  SHORT,
+  UNDEFINED;
 
-  public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle DEFAULT = MEDIUM;
+  public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle DEFAULT = UNDEFINED;
 
   public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle getFromString(
       String s) {

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterInputJson.java

@@ -1,7 +1,12 @@
 package org.unicode.conformance.testtype.datetimeformatter;
 
+import java.time.Instant;
+
 import java.util.Date;
 import com.ibm.icu.util.Calendar;
+import com.ibm.icu.util.TimeZone;
+
+import java.util.Locale;
 
 import org.unicode.conformance.testtype.ITestTypeInputJson;
 
@@ -11,11 +16,13 @@ public class DateTimeFormatterInputJson implements ITestTypeInputJson {
 
   public String label;
 
-  public String locale;
+  public String locale_string;
+  public Locale locale_with_calendar;
 
-  // UTC formatted time
+  // UTC formatted instant in time
   public String inputString;
 
+  public Instant time_instant;
   public Date myDate;
 
   public String skeleton;
@@ -25,13 +32,13 @@ public class DateTimeFormatterInputJson implements ITestTypeInputJson {
   public DateTimeFormatterTimeStyle timeStyle;
 
   // TODO!!!
-  public String calendarString;
-  // Set calendar from calendarString!
+  public String calendar_string;
+  // Set calendar from calendar_string!
   public Calendar calendar;
 
   public String numberingSystem;
 
-  public String timeZone;
+  public TimeZone timeZone;
 
   public String timeZoneName;
 }

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTester.java

@@ -1,26 +1,26 @@
 package org.unicode.conformance.testtype.datetimeformatter;
 
-import java.text.ParseException;
-import java.time.LocalDateTime;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-
 import java.util.Date;
+import java.time.Instant;
+
+import java.util.Locale;
+import java.util.Locale.Builder;
 
 import com.ibm.icu.util.Calendar;
 import com.ibm.icu.text.DateFormat;
+import com.ibm.icu.util.TimeZone;
 import com.ibm.icu.util.ULocale;
 
 import io.lacuna.bifurcan.IMap;
 import io.lacuna.bifurcan.Map;
 
-
 import org.unicode.conformance.ExecutorUtils;
 import org.unicode.conformance.testtype.ITestType;
 import org.unicode.conformance.testtype.ITestTypeInputJson;
 import org.unicode.conformance.testtype.ITestTypeOutputJson;
 
 public class DateTimeFormatterTester implements ITestType {
+  private static final int UNDEFINED_DATETIME_STYLE = -1;
 
   public static DateTimeFormatterTester INSTANCE = new DateTimeFormatterTester();
 
@@ -29,32 +29,27 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
     DateTimeFormatterInputJson result = new DateTimeFormatterInputJson();
 
     result.label = (String) inputMapData.get("label", null);
-    result.locale = (String) inputMapData.get("locale", null);
+    result.locale_string = (String) inputMapData.get("locale", null);
     result.skeleton = (String) inputMapData.get("skeleton", null);
 
+    // The instant in UTC time.
     result.inputString = (String) inputMapData.get("input_string", null);
 
     java.util.Map<String, Object> inputOptions =
         (java.util.Map<String, Object>) inputMapData.get("options", null);
 
-    result.timeZone = (String) inputOptions.get("timeZone");
-    ZoneId thisZoneId;
-    if (result.timeZone == null) {
-      thisZoneId = ZoneId.systemDefault();
+    result.timeZoneName = (String) inputOptions.get("timeZone");
+    if (result.timeZoneName == null) {
+      result.timeZone = TimeZone.GMT_ZONE;
     } else {
-      thisZoneId = ZoneId.of(result.timeZone);
+      result.timeZone = TimeZone.getTimeZone(result.timeZoneName);
     }
 
     // Extract ISO part of the input string to parse.
-    String inputStringDateTime = result.inputString.substring(0, 25);
+    result.time_instant = Instant.parse(result.inputString);
+    result.myDate = Date.from(result.time_instant);
 
     // For parsing the input string and converting to java.util.date
-    LocalDateTime parsedLocalDateTime =
-        LocalDateTime.parse(inputStringDateTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME);
-    result.myDate =
-        java.util.Date.from(parsedLocalDateTime.atZone(thisZoneId)
-            .toInstant());
-
     result.dateStyle = DateTimeFormatterDateStyle.getFromString(
         "" + inputOptions.get("dateStyle")
     );
@@ -63,15 +58,15 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
         "" + inputOptions.get("timeStyle")
     );
 
-    result.calendarString = (String) inputOptions.get("calendar");
+    result.calendar_string = (String) inputOptions.get("calendar");
 
-    // TODO!!! Get calendar object. Depends on timezone and locale.
-    // Just a placeholder for now.
-    result.calendar = Calendar.getInstance();
+    result.locale_with_calendar = new Builder().setLanguageTag(result.locale_string)
+        .setUnicodeLocaleKeyword("ca", result.calendar_string)
+        .build();
 
-    result.numberingSystem = (String) inputOptions.get("numberingSystem");
+    result.calendar = Calendar.getInstance(result.locale_with_calendar);
 
-    result.timeZoneName = (String) inputOptions.get("timeZoneName");
+    result.numberingSystem = (String) inputOptions.get("numberingSystem");
 
     return result;
   }
@@ -114,40 +109,44 @@ public String formatOutputJson(ITestTypeOutputJson outputJson) {
 
   public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input) {
 
-    ULocale locale = ULocale.forLanguageTag(input.locale);
+    ULocale locale = ULocale.forLanguageTag(input.locale_string);
 
-    int dateStyle;
+    int dateStyle = UNDEFINED_DATETIME_STYLE;
     switch (input.dateStyle) {
       case FULL:
         dateStyle = DateFormat.FULL;
         break;
       case LONG:
         dateStyle = DateFormat.LONG;
         break;
-      default:
       case MEDIUM:
         dateStyle = DateFormat.MEDIUM;
         break;
       case SHORT:
         dateStyle = DateFormat.SHORT;
         break;
+      default:
+        dateStyle = UNDEFINED_DATETIME_STYLE;  // Undefined
+        break;
     }
 
-    int timeStyle;
+    int timeStyle = UNDEFINED_DATETIME_STYLE;
     switch (input.timeStyle) {
       case FULL:
         timeStyle = DateFormat.FULL;
         break;
       case LONG:
         timeStyle = DateFormat.LONG;
         break;
-      default:
       case MEDIUM:
         timeStyle = DateFormat.MEDIUM;
         break;
       case SHORT:
         timeStyle = DateFormat.SHORT;
         break;
+      default:
+        timeStyle = UNDEFINED_DATETIME_STYLE;  // Undefined
+        break;
 
     }
 
@@ -156,10 +155,22 @@ public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input)
 
     DateFormat dtf;
     if (input.skeleton != null) {
-      dtf = DateFormat.getInstanceForSkeleton(cal, input.skeleton, locale);
+      dtf = DateFormat.getInstanceForSkeleton(cal, input.skeleton, input.locale_with_calendar);
     } else {
-      dtf = DateFormat.getDateTimeInstance(cal, dateStyle, timeStyle, locale);
+      if (dateStyle != UNDEFINED_DATETIME_STYLE && timeStyle != UNDEFINED_DATETIME_STYLE) {
+        dtf = DateFormat.getDateTimeInstance(cal, dateStyle, timeStyle, input.locale_with_calendar);
+      } else
+        if (dateStyle != UNDEFINED_DATETIME_STYLE) {
+          dtf = DateFormat.getDateInstance(cal, dateStyle,input.locale_with_calendar);
+        } else
+          if (timeStyle != UNDEFINED_DATETIME_STYLE) {
+            dtf = DateFormat.getTimeInstance(cal, timeStyle, input.locale_with_calendar);
+          } else {
+            dtf = DateFormat.getInstance(cal, input.locale_with_calendar);
+          }
     }
+    dtf.setCalendar(input.calendar);
+    dtf.setTimeZone(input.timeZone);
 
     return dtf.format(input.myDate);
   }

executors/icu4j/74/executor-icu4j/src/main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTimeStyle.java

@@ -4,9 +4,10 @@ public enum DateTimeFormatterTimeStyle {
   FULL,
   LONG,
   MEDIUM,
-  SHORT;
+  SHORT,
+  UNDEFINED;
 
-  public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle DEFAULT = MEDIUM;
+  public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle DEFAULT = UNDEFINED;
 
   public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle getFromString(
       String s) {