Skip to content

Commit

Permalink
Fix date time format in ICU4J, ICU4C, ICU4X (#297)
Browse files Browse the repository at this point in the history 
* Basic lint applied

* Remove unneeded include

* Updating to use Z as timezone in generator and in Java

* ICU4J date time format - add calendar, fixing many issues

* Fix date/time generator to use instant. Remove 'und' locale

* Fix ICU4C DateTime parsing of 'Z', resolving many test failures

* Fix Rust date/time to accept new UTC instant data. Still not handling timezones correctly

* formatted .rs file

* Removing debug printing from datatime_fmt.cpp

* Add tz_offset_secs to date/time test data so ICU4X can use timezone

* Trying to fix timezone setting in ICU4X datetime

* Fixing about 1/2 of the ICU4X test failures

* Update based on feedback

* Moving code lines as suggested
  • Loading branch information
@sven-oly
sven-oly authored Sep 19, 2024
1 parent f89ea86 commit 919ee65
Show file tree
Hide file tree
Showing 12 changed files with 305 additions and 192 deletions.
77 changes: 38 additions & 39 deletions executors/cpp/datetime_fmt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ const string TestDatetimeFmt(json_object *json_in) {
string label_string = json_object_get_string(label_obj);

Calendar *cal = nullptr;
TimeZone *tz = nullptr;

UnicodeString u_tz_utc("UTC");
TimeZone *tz = nullptr; // TimeZone::createTimeZone(u_tz_utc);

// The locale for formatted output
json_object *locale_label_obj = json_object_object_get(json_in, "locale");
Expand All @@ -65,46 +67,55 @@ const string TestDatetimeFmt(json_object *json_in) {
json_object *return_json = json_object_new_object();
json_object_object_add(return_json, "label", label_obj);

string calendar_str;
string calendar_str = "gregory";

// Get fields out of the options if present
json_object* options_obj = json_object_object_get(json_in, "options");

if (options_obj) {
json_object* cal_item = json_object_object_get(options_obj, "calendar");
// Check for timezone and calendar
json_object* option_item =
json_object_object_get(options_obj, "timeZone");
if (option_item) {
string timezone_str = json_object_get_string(option_item);
UnicodeString u_tz(timezone_str.c_str());
tz = TimeZone::createTimeZone(u_tz);
}

json_object* cal_item =
json_object_object_get(options_obj, "calendar");
if (cal_item) {
calendar_str = json_object_get_string(cal_item);

// Add '@calendar=' + calendar_string to locale
locale_string = locale_string + "@calendar=" + calendar_str;
display_locale = locale_string.c_str();

if (tz) {
cal = Calendar::createInstance(*tz, display_locale, status);
} else {
cal = Calendar::createInstance(display_locale, status);
}
if (U_FAILURE(status)) {
json_object_object_add(
return_json,
"error",
json_object_new_string("Error in createInstance for calendar"));
return json_object_to_json_string(return_json);
}
}
}

// Add '@calendar=' + calendar_string to locale
locale_string = locale_string + "@calendar=" + calendar_str;
display_locale = locale_string.c_str();

if (tz) {
cal = Calendar::createInstance(tz, display_locale, status);
} else {
cal = Calendar::createInstance(display_locale, status);
}
if (U_FAILURE(status)) {
json_object_object_add(
return_json,
"error",
json_object_new_string("Error in createInstance for calendar"));
return json_object_to_json_string(return_json);
}

DateFormat* df;


// Get the input data as a date object.
// Types of input:
// "input_string" parsable ISO formatted string such as
// "2020-03-02 10:15:17 -08:00"
// "input_string" parsable ISO formatted string of an instant
// "2020-03-02 10:15:17Z

string dateStyle_str;
string timeStyle_str;
string timezone_str;

// Expected values if neither dateStyle nor timeStyle is given explicitly.
icu::DateFormat::EStyle date_style = icu::DateFormat::EStyle::kNone;
Expand All @@ -126,17 +137,6 @@ const string TestDatetimeFmt(json_object *json_in) {
timeStyle_str = json_object_get_string(option_item);
time_style = StringToEStyle(timeStyle_str);
}

option_item = json_object_object_get(options_obj, "timeZone");
if (option_item) {
timezone_str = json_object_get_string(option_item);
UnicodeString u_tz(timezone_str.c_str());
tz = TimeZone::createTimeZone(u_tz);
} else {
// Default is UTC
UnicodeString u_tz("UTC");
tz = TimeZone::createTimeZone(u_tz);
}
}

json_object *date_skeleton_obj =
Expand Down Expand Up @@ -183,10 +183,9 @@ const string TestDatetimeFmt(json_object *json_in) {
}

// !!! IS OFFSET ALREADY CONSIDERED?
// if (tz) {
// df->setTimeZone(*tz);
// }

if (tz) {
df->setTimeZone(*tz);
}

// Use ISO string form of the date/time.
json_object *input_string_obj =
Expand Down Expand Up @@ -219,7 +218,7 @@ const string TestDatetimeFmt(json_object *json_in) {
UnicodeString date_ustring(input_date_string.c_str());

// TODO: handles the offset +/-
SimpleDateFormat iso_date_fmt(u"y-M-d'T'h:m:s", und_locale, status);
SimpleDateFormat iso_date_fmt(u"y-M-d'T'h:m:sZ", und_locale, status);
if (U_FAILURE(status)) {
string error_name = u_errorName(status);
string error_message =
Expand Down
5 changes: 3 additions & 2 deletions ...n/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterDateStyle.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ public enum DateTimeFormatterDateStyle {
FULL,
LONG,
MEDIUM,
SHORT;
SHORT,
UNDEFINED;

public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle DEFAULT = MEDIUM;
public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle DEFAULT = UNDEFINED;

public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterDateStyle getFromString(
String s) {
Expand Down
17 changes: 12 additions & 5 deletions ...n/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterInputJson.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
package org.unicode.conformance.testtype.datetimeformatter;

import java.time.Instant;

import java.util.Date;
import com.ibm.icu.util.Calendar;
import com.ibm.icu.util.TimeZone;

import java.util.Locale;

import org.unicode.conformance.testtype.ITestTypeInputJson;

Expand All @@ -11,11 +16,13 @@ public class DateTimeFormatterInputJson implements ITestTypeInputJson {

public String label;

public String locale;
public String locale_string;
public Locale locale_with_calendar;

// UTC formatted time
// UTC formatted instant in time
public String inputString;

public Instant time_instant;
public Date myDate;

public String skeleton;
Expand All @@ -25,13 +32,13 @@ public class DateTimeFormatterInputJson implements ITestTypeInputJson {
public DateTimeFormatterTimeStyle timeStyle;

// TODO!!!
public String calendarString;
// Set calendar from calendarString!
public String calendar_string;
// Set calendar from calendar_string!
public Calendar calendar;

public String numberingSystem;

public String timeZone;
public TimeZone timeZone;

public String timeZoneName;
}
75 changes: 43 additions & 32 deletions ...main/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTester.java
View file
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
package org.unicode.conformance.testtype.datetimeformatter;

import java.text.ParseException;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;

import java.util.Date;
import java.time.Instant;

import java.util.Locale;
import java.util.Locale.Builder;

import com.ibm.icu.util.Calendar;
import com.ibm.icu.text.DateFormat;
import com.ibm.icu.util.TimeZone;
import com.ibm.icu.util.ULocale;

import io.lacuna.bifurcan.IMap;
import io.lacuna.bifurcan.Map;


import org.unicode.conformance.ExecutorUtils;
import org.unicode.conformance.testtype.ITestType;
import org.unicode.conformance.testtype.ITestTypeInputJson;
import org.unicode.conformance.testtype.ITestTypeOutputJson;

public class DateTimeFormatterTester implements ITestType {
private static final int UNDEFINED_DATETIME_STYLE = -1;

public static DateTimeFormatterTester INSTANCE = new DateTimeFormatterTester();

Expand All @@ -29,32 +29,27 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
DateTimeFormatterInputJson result = new DateTimeFormatterInputJson();

result.label = (String) inputMapData.get("label", null);
result.locale = (String) inputMapData.get("locale", null);
result.locale_string = (String) inputMapData.get("locale", null);
result.skeleton = (String) inputMapData.get("skeleton", null);

// The instant in UTC time.
result.inputString = (String) inputMapData.get("input_string", null);

java.util.Map<String, Object> inputOptions =
(java.util.Map<String, Object>) inputMapData.get("options", null);

result.timeZone = (String) inputOptions.get("timeZone");
ZoneId thisZoneId;
if (result.timeZone == null) {
thisZoneId = ZoneId.systemDefault();
result.timeZoneName = (String) inputOptions.get("timeZone");
if (result.timeZoneName == null) {
result.timeZone = TimeZone.GMT_ZONE;
} else {
thisZoneId = ZoneId.of(result.timeZone);
result.timeZone = TimeZone.getTimeZone(result.timeZoneName);
}

// Extract ISO part of the input string to parse.
String inputStringDateTime = result.inputString.substring(0, 25);
result.time_instant = Instant.parse(result.inputString);
result.myDate = Date.from(result.time_instant);

// For parsing the input string and converting to java.util.date
LocalDateTime parsedLocalDateTime =
LocalDateTime.parse(inputStringDateTime, DateTimeFormatter.ISO_OFFSET_DATE_TIME);
result.myDate =
java.util.Date.from(parsedLocalDateTime.atZone(thisZoneId)
.toInstant());

result.dateStyle = DateTimeFormatterDateStyle.getFromString(
"" + inputOptions.get("dateStyle")
);
Expand All @@ -63,15 +58,15 @@ public ITestTypeInputJson inputMapToJson(Map<String, Object> inputMapData) {
"" + inputOptions.get("timeStyle")
);

result.calendarString = (String) inputOptions.get("calendar");
result.calendar_string = (String) inputOptions.get("calendar");

// TODO!!! Get calendar object. Depends on timezone and locale.
// Just a placeholder for now.
result.calendar = Calendar.getInstance();
result.locale_with_calendar = new Builder().setLanguageTag(result.locale_string)
.setUnicodeLocaleKeyword("ca", result.calendar_string)
.build();

result.numberingSystem = (String) inputOptions.get("numberingSystem");
result.calendar = Calendar.getInstance(result.locale_with_calendar);

result.timeZoneName = (String) inputOptions.get("timeZoneName");
result.numberingSystem = (String) inputOptions.get("numberingSystem");

return result;
}
Expand Down Expand Up @@ -114,40 +109,44 @@ public String formatOutputJson(ITestTypeOutputJson outputJson) {

public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input) {

ULocale locale = ULocale.forLanguageTag(input.locale);
ULocale locale = ULocale.forLanguageTag(input.locale_string);

int dateStyle;
int dateStyle = UNDEFINED_DATETIME_STYLE;
switch (input.dateStyle) {
case FULL:
dateStyle = DateFormat.FULL;
break;
case LONG:
dateStyle = DateFormat.LONG;
break;
default:
case MEDIUM:
dateStyle = DateFormat.MEDIUM;
break;
case SHORT:
dateStyle = DateFormat.SHORT;
break;
default:
dateStyle = UNDEFINED_DATETIME_STYLE; // Undefined
break;
}

int timeStyle;
int timeStyle = UNDEFINED_DATETIME_STYLE;
switch (input.timeStyle) {
case FULL:
timeStyle = DateFormat.FULL;
break;
case LONG:
timeStyle = DateFormat.LONG;
break;
default:
case MEDIUM:
timeStyle = DateFormat.MEDIUM;
break;
case SHORT:
timeStyle = DateFormat.SHORT;
break;
default:
timeStyle = UNDEFINED_DATETIME_STYLE; // Undefined
break;

}

Expand All @@ -156,10 +155,22 @@ public String getDateTimeFormatterResultString(DateTimeFormatterInputJson input)

DateFormat dtf;
if (input.skeleton != null) {
dtf = DateFormat.getInstanceForSkeleton(cal, input.skeleton, locale);
dtf = DateFormat.getInstanceForSkeleton(cal, input.skeleton, input.locale_with_calendar);
} else {
dtf = DateFormat.getDateTimeInstance(cal, dateStyle, timeStyle, locale);
if (dateStyle != UNDEFINED_DATETIME_STYLE && timeStyle != UNDEFINED_DATETIME_STYLE) {
dtf = DateFormat.getDateTimeInstance(cal, dateStyle, timeStyle, input.locale_with_calendar);
} else
if (dateStyle != UNDEFINED_DATETIME_STYLE) {
dtf = DateFormat.getDateInstance(cal, dateStyle,input.locale_with_calendar);
} else
if (timeStyle != UNDEFINED_DATETIME_STYLE) {
dtf = DateFormat.getTimeInstance(cal, timeStyle, input.locale_with_calendar);
} else {
dtf = DateFormat.getInstance(cal, input.locale_with_calendar);
}
}
dtf.setCalendar(input.calendar);
dtf.setTimeZone(input.timeZone);

return dtf.format(input.myDate);
}
Expand Down
5 changes: 3 additions & 2 deletions ...n/java/org/unicode/conformance/testtype/datetimeformatter/DateTimeFormatterTimeStyle.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ public enum DateTimeFormatterTimeStyle {
FULL,
LONG,
MEDIUM,
SHORT;
SHORT,
UNDEFINED;

public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle DEFAULT = MEDIUM;
public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle DEFAULT = UNDEFINED;

public static org.unicode.conformance.testtype.datetimeformatter.DateTimeFormatterTimeStyle getFromString(
String s) {
Expand Down
Loading

0 comments on commit 919ee65

Please sign in to comment.