[b/390590500] Airflow dates params (#766)

* add date params to airflow connector, intermediate commit

* add date range support to airflow connector

* use data range parameters for one file in Airflow connector

* fix typos

* rename private method

* move comment to java doc

* update parameter description

* update params description

* update comment

* remove lookback-days param

Author: vladislav-sidorovich

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/ConnectorArguments.java

@@ -91,6 +91,8 @@ public class ConnectorArguments extends DefaultArguments {
   public static final int OPT_PORT_ORDER = 200;
   public static final String OPT_USER = "user";
   public static final String OPT_PASSWORD = "password";
+  public static final String OPT_START_DATE = "start-date";
+  public static final String OPT_END_DATE = "end-date";
 
   public static final String OPT_CLUSTER = "cluster";
   public static final String OPT_ROLE = "role";
@@ -228,6 +230,26 @@ public class ConnectorArguments extends DefaultArguments {
           .withOptionalArg()
           .describedAs("sekr1t");
 
+  private final OptionSpec<ZonedDateTime> optionStartDate =
+      parser
+          .accepts(
+              OPT_START_DATE,
+              "Inclusive start date for data to export, value will be truncated to hour")
+          .withOptionalArg()
+          .ofType(Date.class)
+          .withValuesConvertedBy(ZonedParser.withDefaultPattern(DayOffset.START_OF_DAY))
+          .describedAs("2001-01-15[ 00:00:00.[000]]");
+
+  private final OptionSpec<ZonedDateTime> optionEndDate =
+      parser
+          .accepts(
+              OPT_END_DATE,
+              "Exclusive end date for data to export, value will be truncated to hour")
+          .withOptionalArg()
+          .ofType(Date.class)
+          .withValuesConvertedBy(ZonedParser.withDefaultPattern(DayOffset.START_OF_DAY))
+          .describedAs("2001-01-15[ 00:00:00.[000]]");
+
   private final OptionSpec<String> optionCluster =
       parser
           .accepts(OPT_CLUSTER, "Cluster name to dump metadata")
@@ -309,7 +331,7 @@ public class ConnectorArguments extends DefaultArguments {
           .withOptionalArg()
           .ofType(Date.class)
           .withValuesConvertedBy(ZonedParser.withDefaultPattern(DayOffset.START_OF_DAY))
-          .describedAs("2001-01-01[ 00:00:00.[000]]");
+          .describedAs("2001-01-15[ 00:00:00.[000]]");
   private final OptionSpec<ZonedDateTime> optionQueryLogEnd =
       parser
           .accepts(
@@ -318,7 +340,7 @@ public class ConnectorArguments extends DefaultArguments {
           .withOptionalArg()
           .ofType(Date.class)
           .withValuesConvertedBy(ZonedParser.withDefaultPattern(DayOffset.END_OF_DAY))
-          .describedAs("2001-01-01[ 00:00:00.[000]]");
+          .describedAs("2001-01-15[ 00:00:00.[000]]");
 
   // This is intentionally NOT provided as a default value to the optionQueryLogEnd OptionSpec,
   // because some callers
@@ -889,6 +911,20 @@ public Integer getQueryLogDays() {
     return getOptions().valueOf(optionQueryLogDays);
   }
 
+  @CheckForNull
+  public ZonedDateTime getStartDate() {
+    return getOptions().valueOf(optionStartDate);
+  }
+
+  public ZonedDateTime getStartDate(ZonedDateTime defaultTime) {
+    return firstNonNull(getStartDate(), defaultTime);
+  }
+
+  @CheckForNull
+  public ZonedDateTime getEndDate() {
+    return getOptions().valueOf(optionEndDate);
+  }
+
   public Duration getQueryLogRotationFrequency() {
     return RotationFrequencyConverter.convert(
         getOptions().valueOf(optionQueryLogRotationFrequency));

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsArgumentQueryLogDays.java

@@ -35,5 +35,5 @@
     description = RespectsArgumentQueryLogDays.DESCRIPTION)
 public @interface RespectsArgumentQueryLogDays {
 
-  public static final String DESCRIPTION = "The number of days of query history to dump.";
+  String DESCRIPTION = "The number of days of query history to dump.";
 }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsArgumentQueryLogEnd.java

@@ -35,5 +35,5 @@
     description = RespectsArgumentQueryLogEnd.DESCRIPTION)
 public @interface RespectsArgumentQueryLogEnd {
 
-  public static final String DESCRIPTION = "End date for query history to dump";
+  String DESCRIPTION = "End date for query history to dump";
 }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/annotations/RespectsArgumentQueryLogStart.java

@@ -35,5 +35,5 @@
     description = RespectsArgumentQueryLogStart.DESCRIPTION)
 public @interface RespectsArgumentQueryLogStart {
 
-  public static final String DESCRIPTION = "Start date for query history to dump";
+  String DESCRIPTION = "Start date for query history to dump";
 }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/AbstractJdbcConnector.java

@@ -28,6 +28,9 @@
 import java.security.PrivilegedExceptionAction;
 import java.sql.Driver;
 import java.sql.SQLException;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -39,14 +42,18 @@
 
 /** @author shevek */
 public abstract class AbstractJdbcConnector extends AbstractConnector {
-
-  @SuppressWarnings("UnusedVariable")
   private static final Logger logger = LoggerFactory.getLogger(AbstractJdbcConnector.class);
+  private static final DateTimeFormatter SQL_FORMAT =
+      DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneOffset.UTC);
 
   public AbstractJdbcConnector(@Nonnull String name) {
     super(name);
   }
 
+  protected static String dateToSqlFormat(ZonedDateTime dateTime) {
+    return SQL_FORMAT.format(dateTime);
+  }
+
   @Nonnull
   private static ClassLoader newDriverParentClassLoader() throws PrivilegedActionException {
     return AccessController.doPrivileged(

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ZonedIntervalIterable.java

@@ -23,30 +23,25 @@
 import java.util.Iterator;
 import javax.annotation.Nonnull;
 import org.apache.commons.lang3.time.DurationFormatUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /** @author shevek */
 public class ZonedIntervalIterable implements Iterable<ZonedInterval> {
-
-  @SuppressWarnings("UnusedVariable")
-  private static final Logger logger = LoggerFactory.getLogger(ZonedIntervalIterable.class);
-
   private final ZonedDateTime start;
   private final ZonedDateTime end;
-  private final Duration duration;
+  private final Duration intervalDuration;
 
-  /* pp */ ZonedIntervalIterable(
+  ZonedIntervalIterable(
       @Nonnull ZonedDateTime start,
       @Nonnull ZonedDateTime end,
-      @Nonnull Duration duration,
+      @Nonnull Duration intervalDuration,
       IntervalExpander expander) {
-    this.duration = Preconditions.checkNotNull(duration, "Duration was null.");
+    this.intervalDuration =
+        Preconditions.checkNotNull(intervalDuration, "Interval duration was null.");
     Preconditions.checkNotNull(start, "Start was null.");
     Preconditions.checkNotNull(end, "End was null.");
 
     Preconditions.checkState(
-        start.isBefore(end), "Start date %s must precede end date %s", start, end);
+        start.isBefore(end), "Start date [%s] must precede end date [%s]", start, end);
 
     ZonedInterval expandedInterval = expander.apply(new ZonedInterval(start, end));
 
@@ -65,23 +60,23 @@ public ZonedDateTime getEnd() {
   }
 
   @Nonnull
-  public Duration getDuration() {
-    return duration;
+  public Duration getIntervalDuration() {
+    return intervalDuration;
   }
 
-  private class Itr extends AbstractIterator<ZonedInterval> {
+  private class DatesIterator extends AbstractIterator<ZonedInterval> {
 
     private ZonedDateTime current;
 
-    public Itr() {
+    public DatesIterator() {
       this.current = start;
     }
 
     @Override
     protected ZonedInterval computeNext() {
       if (current.isEqual(end) || current.isAfter(end)) return endOfData();
 
-      ZonedDateTime next = current.plus(duration);
+      ZonedDateTime next = current.plus(intervalDuration);
       if (next.isAfter(end)) return endOfData();
 
       ZonedInterval result = new ZonedInterval(current, next);
@@ -93,13 +88,15 @@ protected ZonedInterval computeNext() {
   @Nonnull
   @Override
   public Iterator<ZonedInterval> iterator() {
-    return new Itr();
+    return new DatesIterator();
   }
 
   @Override
   public String toString() {
     return String.format(
-        "from %s to %s every %s",
-        start, end, DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true));
+        "from [%s] to [%s] every [%s]ms",
+        start,
+        end,
+        DurationFormatUtils.formatDurationWords(intervalDuration.toMillis(), true, true));
   }
 }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/ZonedIntervalIterableGenerator.java

@@ -56,6 +56,17 @@ public class ZonedIntervalIterableGenerator {
     return forTimeUnitsUntil(ZonedDateTime.now(ZoneOffset.UTC), unitCount, duration, expander);
   }
 
+  @Nonnull
+  public static ZonedIntervalIterable forDateRangeWithIntervalDuration(
+      ZonedDateTime startDate, ZonedDateTime endDate, Duration intervalDuration)
+      throws MetadataDumperUsageException {
+    return new ZonedIntervalIterable(
+        startDate,
+        endDate,
+        intervalDuration,
+        IntervalExpander.createBasedOnDuration(intervalDuration));
+  }
+
   /**
    * Builds a ZonedIntervalIterable from connector arguments, the intervals will all be one hour
    * long ({@link ChronoUnit#HOURS}) and have an inclusive starting datetime and exclusive ending