Handle dates with year > 9999 (#122)

Author: deanmorin

tap_postgres/sync_strategies/logical_replication.py

@@ -21,6 +21,7 @@
 
 UPDATE_BOOKMARK_PERIOD = 10000
 FALLBACK_DATETIME = '9999-12-31T23:59:59.999+00:00'
+FALLBACK_DATE = '9999-12-31T00:00:00+00:00'
 
 
 class ReplicationSlotNotFoundError(Exception):
@@ -286,7 +287,15 @@ def selected_value_to_singer_value_impl(elem, og_sql_datatype, conn_info):
         if isinstance(elem, datetime.date):
             # logical replication gives us dates as strings UNLESS they from an array
             return elem.isoformat() + 'T00:00:00+00:00'
-        return parse(elem).isoformat() + "+00:00"
+        try:
+            return parse(elem).isoformat() + "+00:00"
+        except ValueError as e:
+            match = re.match(r'year (\d+) is out of range', str(e))
+            if match and int(match.group(1)) > 9999:
+                LOGGER.warning('datetimes cannot handle years past 9999, returning %s for %s',
+                               FALLBACK_DATE, elem)
+                return FALLBACK_DATE
+            raise
     if sql_datatype == 'time with time zone':
         # time with time zone values will be converted to UTC and time zone dropped
         # Replace hour=24 with hour=0

tests/test_logical_replication.py

@@ -429,6 +429,21 @@ def test_selected_value_to_singer_value_impl_with_timestamp_tz_value_as_datetime
 
         self.assertEqual('9999-12-31T23:59:59.999+00:00', output)
 
+    def test_selected_value_to_singer_value_impl_with_date_value_as_string_expect_iso_format(self):
+        output = logical_replication.selected_value_to_singer_value_impl('2021-09-07', 'date', None)
+
+        self.assertEqual('2021-09-07T00:00:00+00:00', output)
+
+    def test_selected_value_to_singer_value_impl_with_date_value_as_string_out_of_range(self):
+        """
+        Test selected_value_to_singer_value_impl with date as string where year
+        is > 9999 (which is valid in postgres) should fallback to max date
+        allowed
+        """
+        output = logical_replication.selected_value_to_singer_value_impl('10000-09-01', 'date', None)
+
+        self.assertEqual('9999-12-31T00:00:00+00:00', output)
+
     def test_row_to_singer_message(self):
         stream = {
             'stream': 'my_stream',