Skip to content

Commit ca99527

Browse files
daniel-zamoradaniel-zamora
authored
CMR-10069: Recurring temporal is displayed as date/time but the underlying search is day-of-year, leading to confusing search results (#2180)
* CMR-10069 fixes periodic temporal condition to adjust for leap years * CMR-10069 test fixes * CMR-10069 test fixes * CMR-10069 test fixes * CMR-10069 pr comments * CMR-10069 adds test
1 parent 2ad115c commit ca99527

File tree

4 files changed

+201
-93
lines changed

4 files changed

+201
-93
lines changed

‎search-app/src/cmr/search/data/complex_to_simple_converters/temporal.clj‎

Lines changed: 77 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
(ns cmr.search.data.complex-to-simple-converters.temporal
22
"Defines functions that implement the reduce-query-condition method of the ComplexQueryToSimple
33
protocol for Temporal conditions."
4-
5-
(:require [clj-time.core :as t]
6-
[cmr.common.time-keeper :as tk]
7-
[cmr.common.services.errors :as errors]
8-
[cmr.search.models.query :as qm]
9-
[cmr.common.services.search.query-model :as cqm]
10-
[cmr.elastic-utils.search.es-group-query-conditions :as gc]
11-
[cmr.elastic-utils.datetime-helper :as h]
12-
[cmr.elastic-utils.search.query-transform :as c2s]))
4+
(:require
5+
[clj-time.core :as t]
6+
[cmr.common.time-keeper :as tk]
7+
[cmr.common.services.errors :as errors]
8+
[cmr.search.models.query :as qm]
9+
[cmr.common.services.search.query-model :as cqm]
10+
[cmr.elastic-utils.search.es-group-query-conditions :as gc]
11+
[cmr.elastic-utils.datetime-helper :as h]
12+
[cmr.elastic-utils.search.query-transform :as c2s]))
1313

1414
(defn- intersect-temporal->simple-conditions
1515
"Convert a temporal condition with INTERSECT mask into a combination of simpler conditions
@@ -43,41 +43,88 @@
4343
(cqm/nested-condition :temporals and-conditions))
4444
and-conditions)))
4545

46+
(defn- is-leap-year?
47+
"Checks if year is a leap year."
48+
[year]
49+
(cond
50+
(not (zero? (mod year 4)))
51+
false
52+
53+
(not (zero? (mod year 100)))
54+
true
55+
56+
(zero? (mod year 400))
57+
true
58+
59+
:else
60+
false))
61+
62+
(defn- adjust-day
63+
"Adjusts a given julian day based on whether the current year is a leap year."
64+
[current-year day]
65+
(let [adjusted-day (if (and (is-leap-year? current-year)
66+
(>= day 59))
67+
day
68+
(dec day))]
69+
adjusted-day))
70+
4671
(defn current-end-date
4772
"Returns the current end datetime for a given year and attributes of a periodic temporal condition"
4873
[current-year end-date start-day end-day end-year]
49-
(cond
50-
(nil? end-day)
51-
(if (and (= current-year end-year) end-date)
52-
end-date
53-
(t/minus (t/date-time (inc current-year)) (t/seconds 1)))
54-
55-
;; recurring range does not cross over a year boundary - for example January 25 - May 6
56-
(>= end-day start-day)
57-
(let [current-end (t/plus (t/date-time current-year) (t/days (dec end-day)))]
58-
(if (and end-date (t/after? current-end end-date))
74+
(let [adjusted-end-day (when end-day
75+
(adjust-day current-year end-day))
76+
adjusted-date-time (when end-day
77+
(t/plus (t/date-time current-year) (t/days adjusted-end-day)))
78+
current-end (when end-day
79+
(t/date-time current-year
80+
(t/month adjusted-date-time)
81+
(t/day adjusted-date-time)
82+
(t/hour end-date)
83+
(t/minute end-date)
84+
(t/second end-date)
85+
(t/milli end-date)))]
86+
(cond
87+
(nil? end-day)
88+
(if (and (= current-year end-year) end-date)
5989
end-date
60-
current-end))
90+
(t/minus (t/date-time (inc current-year)) (t/seconds 1)))
6191

62-
;; recurring range does cross over a year boundary - for example October 1 - March 10
63-
:else
64-
(let [current-end (t/plus (t/date-time (inc current-year)) (t/days (dec end-day)))]
92+
;; recurring range does not cross over a year boundary - for example January 25 - May 6
93+
(>= end-day start-day)
6594
(if (and end-date (t/after? current-end end-date))
6695
end-date
67-
current-end))))
96+
current-end)
97+
98+
;; recurring range does cross over a year boundary - for example October 1 - March 10
99+
:else
100+
(let [current-end (t/plus current-end (t/years 1))]
101+
(if (and end-date (t/after? current-end end-date))
102+
end-date
103+
current-end)))))
68104

69105
(defn- simple-conditions-for-year
70106
"Returns simple-conditions constructed for a given year with the periodic temporal condition"
71107
[current-year start-date end-date start-day end-day end-year limit-to-granules concept-type]
72-
(let [current-start (t/plus (t/date-time current-year) (t/days (dec start-day)))
73-
current-start (if (t/before? current-start start-date) start-date current-start)
108+
(let [adjusted-start-day (adjust-day current-year start-day)
109+
adjusted-date-time (t/plus (t/date-time current-year) (t/days adjusted-start-day))
110+
current-start (t/date-time current-year
111+
(t/month adjusted-date-time)
112+
(t/day adjusted-date-time)
113+
(t/hour start-date)
114+
(t/minute start-date)
115+
(t/second start-date)
116+
(t/milli start-date))
117+
current-start (if (t/before? current-start start-date)
118+
start-date
119+
current-start)
74120
current-end (current-end-date current-year end-date start-day end-day end-year)]
75-
(when-not (t/before? current-end current-start)
76-
(intersect-temporal->simple-conditions
121+
122+
(when-not (t/before? current-end current-start)
123+
(intersect-temporal->simple-conditions
77124
(assoc
78125
(qm/map->TemporalCondition {:start-date current-start
79-
:end-date current-end
80-
:limit-to-granules limit-to-granules})
126+
:end-date current-end
127+
:limit-to-granules limit-to-granules})
81128
:concept-type
82129
concept-type)))))
83130

‎search-app/test/cmr/search/test/unit/data/query_to_elastic_converters/temporal.clj‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
end-year))
1717
"1986-10-14T04:03:27Z" 1986 "1986-10-14T04:03:27Z" 1 nil 1986
1818
"1985-12-31T23:59:59Z" 1985 "1986-10-14T04:03:27Z" 1 nil 1986
19-
"1985-12-26T00:00:00Z" 1985 "1986-10-14T04:03:27Z" 1 360 1986
19+
"1985-12-26T04:03:27Z" 1985 "1986-10-14T04:03:27Z" 1 360 1986
2020
"1986-10-14T04:03:27Z" 1986 "1986-10-14T04:03:27Z" 1 360 1986
21-
"1986-01-10T00:00:00Z" 1985 "1986-10-14T04:03:27Z" 360 10 1986
21+
"1986-01-10T04:03:27Z" 1985 "1986-10-14T04:03:27Z" 360 10 1986
2222
"1986-10-14T04:03:27Z" 1986 "1986-10-14T04:03:27Z" 360 10 1986))

‎system-int-test/test/cmr/system_int_test/search/granule/granule_periodic_temporal_search_test.clj‎

Lines changed: 73 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
(ns cmr.system-int-test.search.granule.granule-periodic-temporal-search-test
22
"Integration test for CMR granule periodic temporal search"
33
(:require
4-
[clojure.test :refer :all]
5-
[cmr.common.util :refer [are2]]
6-
[cmr.system-int-test.data2.core :as d]
7-
[cmr.system-int-test.data2.granule :as dg]
8-
[cmr.system-int-test.data2.umm-spec-collection :as data-umm-c]
9-
[cmr.system-int-test.data2.umm-spec-common :as data-umm-cmn]
10-
[cmr.system-int-test.utils.index-util :as index]
11-
[cmr.system-int-test.utils.ingest-util :as ingest]
12-
[cmr.system-int-test.utils.search-util :as search]))
4+
[clojure.test :refer :all]
5+
[cmr.common.util :refer [are3]]
6+
[cmr.system-int-test.data2.core :as d]
7+
[cmr.system-int-test.data2.granule :as dg]
8+
[cmr.system-int-test.data2.umm-spec-collection :as data-umm-c]
9+
[cmr.system-int-test.data2.umm-spec-common :as data-umm-cmn]
10+
[cmr.system-int-test.utils.index-util :as index]
11+
[cmr.system-int-test.utils.ingest-util :as ingest]
12+
[cmr.system-int-test.utils.search-util :as search]))
1313

1414
(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"}))
1515

@@ -75,7 +75,7 @@
7575
(index/wait-until-indexed)
7676

7777
(testing "Search granules with periodic temporal parameter"
78-
(are2 [grans temporal-params]
78+
(are3 [grans temporal-params]
7979
(d/refs-match? grans (search/find-refs :granule {"temporal[]" temporal-params
8080
:page_size 100}))
8181

@@ -140,28 +140,61 @@
140140
[(data-umm-cmn/temporal-extent
141141
{:beginning-date-time "1970-01-01T00:00:00Z"})]}))
142142
gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule1"
143-
:beginning-date-time "2012-01-01T00:00:00Z"
144-
:ending-date-time "2012-01-02T00:00:00Z"}))
143+
:beginning-date-time "2012-01-01T00:00:00Z"
144+
:ending-date-time "2012-01-02T00:00:00Z"}))
145145
gran2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule2"
146-
:beginning-date-time "2012-02-14T00:00:00Z"
147-
:ending-date-time "2012-02-18T00:00:00Z"}))
146+
:beginning-date-time "2012-02-14T00:00:00Z"
147+
:ending-date-time "2012-02-18T00:00:00Z"}))
148148
gran3 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule3"
149-
:beginning-date-time "2012-04-08T00:00:00Z"
150-
:ending-date-time "2012-04-09T00:00:00Z"}))
149+
:beginning-date-time "2012-04-08T00:00:00Z"
150+
:ending-date-time "2012-04-09T00:00:00Z"}))
151151
gran4 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule4"
152-
:beginning-date-time "2012-04-21T00:00:00Z"
153-
:ending-date-time "2012-04-22T00:00:00Z"}))
154-
gran5 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5"
155-
:beginning-date-time "2012-06-01T00:00:00Z"
156-
:ending-date-time "2012-06-02T00:00:00Z"}))
152+
:beginning-date-time "2012-04-21T00:00:00Z"
153+
:ending-date-time "2012-04-22T00:00:00Z"}))
154+
gran5-1-1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-1-1"
155+
:beginning-date-time "2012-05-31T00:00:00Z"
156+
:ending-date-time "2012-06-01T00:00:00Z"}))
157+
gran5-1-2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-1-2"
158+
:beginning-date-time "2012-06-01T00:00:00Z"
159+
:ending-date-time "2012-06-02T00:00:00Z"}))
160+
gran5-2-1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-2-1"
161+
:beginning-date-time "2013-05-31T00:00:00Z"
162+
:ending-date-time "2013-06-01T00:00:00Z"}))
163+
gran5-2-2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-2-2"
164+
:beginning-date-time "2016-06-01T00:00:00Z"
165+
:ending-date-time "2016-06-02T00:00:00Z"}))
166+
gran5-3-1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-3-1"
167+
:beginning-date-time "2016-05-31T00:00:00Z"
168+
:ending-date-time "2016-06-01T00:00:00Z"}))
169+
gran5-3-2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-3-2"
170+
:beginning-date-time "2016-06-01T00:00:00Z"
171+
:ending-date-time "2016-06-02T00:00:00Z"}))
172+
gran5-4-1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-4-1"
173+
:beginning-date-time "2016-02-29T00:00:00Z"
174+
:ending-date-time "2016-03-01T00:00:00Z"}))
175+
gran5-4-2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-4-2"
176+
:beginning-date-time "2016-03-01T00:00:00Z"
177+
:ending-date-time "2016-03-02T00:00:00Z"}))
178+
gran5-4-3 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-4-3"
179+
:beginning-date-time "2015-02-28T00:00:00Z"
180+
:ending-date-time "2015-03-01T00:00:00Z"}))
181+
gran5-4-4 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-4-4"
182+
:beginning-date-time "2015-03-01T00:00:00Z"
183+
:ending-date-time "2015-03-02T00:00:00Z"}))
184+
gran5-4-5 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-4-5"
185+
:beginning-date-time "2024-02-29T00:00:00Z"
186+
:ending-date-time "2024-03-01T00:00:00Z"}))
187+
gran5-4-6 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule5-4-6"
188+
:beginning-date-time "2024-03-01T00:00:00Z"
189+
:ending-date-time "2024-03-02T00:00:00Z"}))
157190
gran6 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule6"
158-
:beginning-date-time "2012-12-21T00:00:00Z"
159-
:ending-date-time "2012-12-22T00:00:00Z"}))]
191+
:beginning-date-time "2012-12-21T00:00:00Z"
192+
:ending-date-time "2012-12-22T00:00:00Z"}))]
160193
(index/wait-until-indexed)
161194

162195
(testing "periodic temporal on the start-year"
163196
(testing "start-day before end-day"
164-
(are2 [grans temporal-params]
197+
(are3 [grans temporal-params]
165198
(d/refs-match? grans (search/find-refs :granule {"temporal[]" temporal-params
166199
:page_size 100}))
167200

@@ -178,25 +211,25 @@
178211
"2012-06-15T00:00:00Z, 2015-02-01T00:00:00Z, 90, 120"))
179212

180213
(testing "start-day after end-day"
181-
(are2 [grans temporal-params]
214+
(are3 [grans temporal-params]
182215
(d/refs-match? grans (search/find-refs :granule {"temporal[]" temporal-params
183216
:page_size 100}))
184217

185218
"start-date before end-day"
186-
[gran5 gran6]
219+
[gran5-1-1 gran5-1-2 gran5-2-1 gran6]
187220
"2012-02-01T00:00:00Z, 2015-02-01T00:00:00Z, 120, 90"
188221

189222
"start-date between start-day and end-day"
190-
[gran5 gran6]
223+
[gran5-1-1 gran5-1-2 gran5-2-1 gran6]
191224
"2012-04-15T00:00:00Z, 2015-02-01T00:00:00Z, 120, 90"
192225

193226
"start-date after start-day"
194-
[gran6]
227+
[gran6 gran5-2-1]
195228
"2012-06-15T00:00:00Z, 2015-02-01T00:00:00Z, 120, 90")))
196229

197230
(testing "periodic temporal on the end-year"
198231
(testing "start-day before end-day"
199-
(are2 [grans temporal-params]
232+
(are3 [grans temporal-params]
200233
(d/refs-match? grans (search/find-refs :granule {"temporal[]" temporal-params
201234
:page_size 100}))
202235

@@ -213,7 +246,7 @@
213246
"2000-02-01T00:00:00Z, 2012-06-15T00:00:00Z, 90, 120"))
214247

215248
(testing "start-day after end-day"
216-
(are2 [grans temporal-params]
249+
(are3 [grans temporal-params]
217250
(d/refs-match? grans (search/find-refs :granule {"temporal[]" temporal-params
218251
:page_size 100}))
219252

@@ -226,11 +259,11 @@
226259
"2000-02-01T00:00:00Z, 2012-04-15T00:00:00Z, 120, 90"
227260

228261
"end-date after start-day"
229-
[gran1 gran2 gran5]
262+
[gran1 gran2 gran5-1-1 gran5-1-2]
230263
"2000-02-01T00:00:00Z, 2012-06-15T00:00:00Z, 120, 90")))
231264

232265
(testing "perodic temporal search on a single day"
233-
(are2 [grans temporal-params]
266+
(are3 [grans temporal-params]
234267
(d/refs-match? grans (search/find-refs :granule {"temporal[]" temporal-params
235268
:page_size 100}))
236269

@@ -246,6 +279,14 @@
246279
[gran2]
247280
"2000-02-01T00:00:00Z, 2012-04-15T00:00:00Z, 49, 49"
248281

282+
"match granule on day on leap year"
283+
[gran5-1-2 gran5-2-2 gran5-3-2]
284+
"2012-06-01T00:01:00Z, 2024-06-01T00:02:00Z, 152, 152"
285+
286+
"match granule on day on leap year on Mar 1st"
287+
[gran5-4-2 gran5-4-4 gran5-4-6]
288+
"2012-03-01T00:01:00Z, 2024-03-01T00:02:00Z, 60, 60"
289+
249290
"search by rolling temporal with end-date not intersect the day interval.
250291
This is a limitation of rolling temporal parameter search where searching on the end year
251292
where end-date not intersect start-day and end-day will not find anything in that year."

0 commit comments

Comments
 (0)