Skip to content

Commit

Permalink
ICU4C: fixing some date/time format problems (#223)
Browse files Browse the repository at this point in the history 
* Fix skeleton parsing in CPP date/time format. Update sampling, too

* Fixing some CPP date time failures

* Updating tools to understand failures
  • Loading branch information
@sven-oly
sven-oly authored Apr 19, 2024
1 parent 6a94dcf commit ed68b40
Show file tree
Hide file tree
Showing 9 changed files with 178 additions and 24 deletions.
18 changes: 12 additions & 6 deletions executors/cpp/datetime_fmt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,23 +132,26 @@ const string test_datetime_fmt(json_object *json_in) {
timezone_str = json_object_get_string(option_item);
UnicodeString u_tz(timezone_str.c_str());
tz = TimeZone::createTimeZone(u_tz);
cout << "# Created timezone " << tz << " for " << timezone_str << endl;
// cout << "# Created timezone " << tz << " for " << timezone_str << endl;
}
}

json_object *date_skeleton_obj =
json_object_object_get(json_in, "datetime_skeleton");
json_object_object_get(json_in, "skeleton");
if (date_skeleton_obj) {
// Data specifies a date time skeleton. Make a formatter based on this.
string skeleton_string = json_object_get_string(date_skeleton_obj);

cout << "# Skeleton = " << skeleton_string << endl;
UnicodeString u_skeleton(skeleton_string.c_str());
if (cal) {
cout << " Cal defined " << endl;
df = DateFormat::createInstanceForSkeleton(cal,
u_skeleton,
displayLocale,
status);
} else {
cout << " NO CAL DEFINED " << endl;
df = DateFormat::createInstanceForSkeleton(u_skeleton,
displayLocale,
status);
Expand All @@ -163,6 +166,12 @@ const string test_datetime_fmt(json_object *json_in) {
}
} else {
// Create default formatter
cout << "# createDateTimeInstance: dstyle " <<
date_style << ", tstyle " <<
time_style << " locale " <<
locale_string <<
endl;

df = DateFormat::createDateTimeInstance(
date_style,
time_style,
Expand All @@ -180,10 +189,7 @@ const string test_datetime_fmt(json_object *json_in) {
}

if (tz) {
cout <<
"# Setting TZ " <<
tz <<
endl;
// cout << "# Setting TZ " << tz << endl;
df->setTimeZone(*tz);
}

Expand Down
15 changes: 15 additions & 0 deletions executors/test_strings.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,3 +313,18 @@
{"test_type": "rdt_fmt", "label": "100", "unit":"day" "day", "count": 1, "locale": "en-US", "options": {}}
{"test_type": "rdt_fmt", "label": "100", "unit":"day", "count": 1, "locale": "es", "options": {}}



// Testing skeleton options
{"test_type": "datetime_fmt", "label":"10","input_string":"2001-09-09T01:46:40.000Z","locale":"en-US","options":{"calendar":"gregory","numberingSystem":"latn"}}
{"test_type": "datetime_fmt", "label":"10","input_string":"2001-09-09T01:46:40.000Z","locale":"en-US","options":{"calendar":"gregory","numberingSystem":"latn"}}

{"test_type": "datetime_fmt", "label":"101","input_string":"2000-01-01T21:00:00.000Z","skeleton":"hh","locale":"en-US","options":{"hour":"2-digit","calendar":"gregory","timeZone":"America/Los_Angeles","numberingSystem":"latn"}}
{"test_type": "datetime_fmt", "label":"101","input_string":"2000-01-01T21:00:00.000Z","skeleton":"h","locale":"en-US","options":{"hour":"2-digit","calendar":"gregory","timeZone":"America/Los_Angeles","numberingSystem":"latn"}}
{"test_type": "datetime_fmt", "label":"101","input_string":"2000-01-01T21:00:00.000Z","skeleton":"H","locale":"en-US","options":{"hour":"2-digit","calendar":"gregory","timeZone":"America/Los_Angeles","numberingSystem":"latn"}}
{"test_type": "datetime_fmt", "label":"101","input_string":"2000-01-01T21:00:00.000Z","skeleton":"hhh","locale":"en-US","options":{"hour":"2-digit","calendar":"gregory","timeZone":"America/Los_Angeles","numberingSystem":"latn"}}
{"test_type": "datetime_fmt", "label":"1265","input_string":"2024-03-17T07:00:00.000Z","skeleton":"hhhm","locale":"en-US","options":{"hour":"2-digit","minute":"numeric","calendar":"chinese","timeZone":"Australia/Brisbane","numberingSystem":"latn"}}

{"test_type": "datetime_fmt", "label":"3812","input_string":"1969-07-16T07:00:00.000Z","skeleton":"HHm","locale":"en-GB","options":{"hour":"2-digit","minute":"numeric","calendar":"buddhist","timeZone":"Australia/Brisbane","numberingSystem":"latn"}}

{"test_type": "datetime_fmt", "label":"3812","input_string":"1969-07-16T07:00:00.000Z","locale":"en-GB","options":{"dateStyle": "short", "timeStyle": "short", "hour":"2-digit","minute":"numeric","calendar":"buddhist","timeZone":"Australia/Brisbane","numberingSystem":"latn"}}
3 changes: 2 additions & 1 deletion testgen/generators/datetime_fmt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ def process_test_data(self):

# Set up Node version and call the generator
nvm_version = icu_nvm_versions[self.icu_version]
generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s; node generators/datetime_gen.js' % (nvm_version, nvm_version)
generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s; node generators/datetime_gen.js %s %s' % (
nvm_version, nvm_version, '-run_limit', self.run_limit)

logging.info('Running this command: %s', generate_command)
result = result = subprocess.run(generate_command, shell=True)
Expand Down
51 changes: 40 additions & 11 deletions testgen/generators/datetime_gen.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,10 @@ const dates = [
new Date('Mar 17, 2024'),
new Date('AD 1'),
new Date('AD 0, 13:00'),
//new Date('1066, December 16, 7:17'), // Expect Sunday
new Date('1066, December 16, 7:17'),
new Date('1454, May 29, 16:47'),
// new Date('1454, May 29, 16:47'), // Expect Monday
new Date('1754, May 29, 16:47'),
new Date('BCE 753, April 21'),
new Date('1969, July 16'),
new Date(0),
Expand Down Expand Up @@ -188,7 +190,7 @@ function optionsToSkeleton(options) {
return skeleton_array.join('');
}

function generateAll() {
function generateAll(run_limit) {

let test_obj = {
'Test scenario': 'datetime_fmt',
Expand Down Expand Up @@ -256,7 +258,7 @@ function generateAll() {
try {
formatter = new Intl.DateTimeFormat(locale, all_options);
} catch (error) {
console.log(error, ' with locale ',
console.error(error, ' with locale ',
locale, ' and options: ', all_options);
continue;
}
Expand All @@ -268,7 +270,7 @@ function generateAll() {
const parts = formatter.formatToParts(d);
result = parts.map((x) => x.value).join("");
} catch (error) {
console.log('FORMATTER CREATION FAILS! ', error);
console.error('FORMATTER CREATION FAILS! ', error);
}
// format this date
// get the milliseconds
Expand Down Expand Up @@ -297,7 +299,7 @@ function generateAll() {
}

if (debug) {
console.log("TEST CASE :", test_case);
console.debug("TEST CASE :", test_case);
}
test_cases.push(test_case);

Expand All @@ -309,7 +311,7 @@ function generateAll() {
console.log(' expected = ', result);
}
} catch (error) {
console.log('!!! error ', error, ' in label ', label_num,
console.error('!!! error ', error, ' in label ', label_num,
' for date = ', d);
}
label_num ++;
Expand All @@ -318,25 +320,52 @@ function generateAll() {
}
}


console.log('Number of date/time tests generated for ',
console.log('Total date/time tests generated for ',
process.versions.icu, ': ', label_num);

test_obj['tests'] = test_cases;
test_obj['tests'] = sample_tests(test_cases, run_limit);
try {
fs.writeFileSync('datetime_fmt_test.json', JSON.stringify(test_obj, null, 2));
// file written successfully
} catch (err) {
console.error(err);
}

verify_obj['verifications'] = verify_cases;
console.log('Number of date/time sampled tests ',
process.versions.icu, ': ', test_obj['tests'] .length);

verify_obj['verifications'] = sample_tests(verify_cases, run_limit);
try {
fs.writeFileSync('datetime_fmt_verify.json', JSON.stringify(verify_obj, null, 2));
// file written successfully
} catch (err) {
console.error(err);
}
}

function sample_tests(all_tests, run_limit) {
// Gets a sampling of the data based on total and the expected number.

if (run_limit < 0 || all_tests.length <= run_limit) {
return all_tests;
}

let size_all = all_tests.length;
let increment = Math.floor(size_all / run_limit);
let samples = [];
for (let index = 0; index < size_all; index += increment) {
samples.push(all_tests[index]);
}
return samples;
}

/* Call the generator */
generateAll();
let run_limit = -1;
if (process.argv.length >= 5) {
if (process.argv[3] == '-run_limit') {
run_limit = Number(process.argv[4]);
}
}

// Call the generator
generateAll(run_limit);
2 changes: 1 addition & 1 deletion testgen/generators/relativedatetime_fmt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def process_test_data(self):
generate_command = 'source ~/.nvm/nvm.sh; nvm install %s; nvm use %s; node generators/rdt_fmt_gen.js' % (nvm_version, nvm_version)

logging.info('Running this command: %s', generate_command)
result = result = subprocess.run(generate_command, shell=True)
result = subprocess.run(generate_command, shell=True)

# Move results to the right directory
mv_command = 'mv rdt_fmt*.json %s' % self.icu_version
Expand Down
37 changes: 35 additions & 2 deletions verifier/detail_template.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,11 @@
cursor: pointer;
font-size:20px;
}


#diff_area {
font-size: 24px;
font-color: blue;
}
h1, h2, h3, p {
font-family: Arial, Helvetica, sans-serif;
}
Expand Down Expand Up @@ -350,7 +354,7 @@
} else {
output = item[key];
}
html.push('<td onclick="captureInputDataOnClick(this);">' + output +'</td>');
html.push('<td onclick="captureInputDataOnClick(this);" onmouseover="hoverDiffText(this);">' + output +'</td>');
}
}
html.push("</tr>");
Expand Down Expand Up @@ -609,7 +613,31 @@
navigator.clipboard.writeText(output);
}

// On hover, show the differen between expected and actual result.
function hoverDiffText(element) {
// First, get the row of this item.
const row = element.parentNode;
const text1 = row.children[1].innerHTML;
const text2 = row.children[2].innerHTML;
if (text1 == text2) {
return;
}
const dmp = new diff_match_patch();
const diff_text = dmp.diff_main(text1, text2);
dmp.diff_cleanupSemantic(diff_text);
// And show the difference nicely.
const ds = dmp.diff_prettyHtml(diff_text);
const diff_area = document.getElementById("diff_area");
diff_area.innerHTML = ds;
}
</script>

<!-- diff-match-patch code from
https://github.com/google/diff-match-patch/wiki/Language:-JavaScript
Intended to use for differences between expected and actual result.
-->
<script src="../../../diff_match_patch.js"></script>

</head>
<body>
<h1>Verification report: $test_type on $library_name</h1>
Expand All @@ -628,6 +656,10 @@ <h2>$platform_info</h2>
<h2>Test details</h2>
<p>$test_environment</p>
</div>
<div bp="4">
<h2>Acknowledgements</h2>
<p><a href='https://github.com/google/diff-match-patch/wiki/Language:-JavaScript'>diff-match-patch</a></p>
</div>
</div>
<br>
<div id='test_result_details' class='test_results'>
Expand Down Expand Up @@ -660,6 +692,7 @@ <h2>Test details</h2>

<details>
<summary>Failing tests <span id='fail_count'>($failing_tests)</span></summary>
<div id="diff_area"><p></p></div>
<div id="fail-pagination-container"></div>
<div id="fail-data-container"></div>
<div id='testFailuresCharacterized'>
Expand Down
Loading

0 comments on commit ed68b40

Please sign in to comment.