InvalidStreamSortException when replication key is not unique
Consider the following example:
class TestStream(Stream):
name = "test_stream"
primary_keys = ["id"]
replication_method = "INCREMENTAL"
replication_key = "updated_at"
is_timestamp_replication_key = True
is_sorted = True
schema = PropertiesList(
Property("id", IntegerType),
Property("updated_at", DateTimeType),
).to_dict()
def get_records(self, partition: Optional[dict] = None) -> Iterable[Dict[str, Any]]:
sample_data = [
{"id": 1, "updated_at": "2021-01-01T00:00.000Z"},
{"id": 2, "updated_at": "2021-01-01T01:01.000Z"},
{"id": 3, "updated_at": "2021-01-01T01:01.100Z"},
{"id": 4, "updated_at": "2021-01-01T01:01.100Z"},
{"id": 5, "updated_at": "2021-01-01T01:01.200Z"},
]
for row in sample_data:
yield row
I think ordered streams which are distinguished by one key but ordered by another key which is not necessarily unique, such as an updated_at timestamp, are a pretty common use case.
However when Stream checks for a sorted stream, it checks for > instead of >=. So rows 3 and 4, which have exactly the same timestamp despite being different rows, incorrectly throws InvalidStreamSortException.
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=tap-sample v0.0.1, Singer SDK v0.1.4)
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Skipping parse of env var settings...
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Config validation passed with 0 errors and 0 warnings.
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Beginning INCREMENTAL sync of stream 'test_stream'...
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'id' under '('properties',)...
time=2021-05-03 20:27:47 name=tap-sample level=WARNING message=Catalog entry missing for 'test_stream':'('properties',)'. Using parent value of selected=True.
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'timestamp' under '('properties',)...
{"type": "SCHEMA", "stream": "test_stream", "schema": {"properties": {"id": {"type": ["integer", "null"]}, "timestamp": {"format": "date-time", "type": ["string", "null"]}}, "type": "object"}, "key_properties": ["id"], "bookmark_properties": ["timestamp"]}
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'id' under '()'...
time=2021-05-03 20:27:47 name=tap-sample level=WARNING message=Catalog entry missing for 'test_stream':'('id',)'. Using parent value of selected=True.
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'timestamp' under '()'...
time=2021-05-03 20:27:47 name=tap-sample level=WARNING message=Catalog entry missing for 'test_stream':'('timestamp',)'. Using parent value of selected=True.
{"type": "RECORD", "stream": "test_stream", "record": {"id": 1, "timestamp": "2021-01-01T00:00.000Z"}, "time_extracted": "2021-05-04T00:27:47.867304Z"}
{"type": "STATE", "value": {"bookmarks": {"test_stream": {"replication_key": "timestamp", "replication_key_value": "2021-01-01T00:00.000Z"}}}}
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'id' under '()'...
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'timestamp' under '()'...
{"type": "RECORD", "stream": "test_stream", "record": {"id": 2, "timestamp": "2021-01-01T01:01.000Z"}, "time_extracted": "2021-05-04T00:27:47.868669Z"}
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'id' under '()'...
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'timestamp' under '()'...
{"type": "RECORD", "stream": "test_stream", "record": {"id": 3, "timestamp": "2021-01-01T01:01.100Z"}, "time_extracted": "2021-05-04T00:27:47.869016Z"}
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'id' under '()'...
time=2021-05-03 20:27:47 name=tap-sample level=INFO message=Checking 'timestamp' under '()'...
{"type": "RECORD", "stream": "test_stream", "record": {"id": 4, "timestamp": "2021-01-01T01:01.100Z"}, "time_extracted": "2021-05-04T00:27:47.869332Z"}
time=2021-05-03 20:27:47 name=tap-sample level=ERROR message=Sorting error detected on row #4. Unsorted data detected in stream. Latest value '2021-01-01T01:01.100Z' is smaller than previous max '2021-01-01T01:01.100Z'.
Traceback (most recent call last):
File "/Users/cjk/fixd/projects/tap_sample/.venv/bin/tap-sample", line 5, in <module>
cli()
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/click/core.py", line 829, in __call__
return self.main(*args, **kwargs)
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/tap_base.py", line 256, in cli
tap.sync_all()
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/tap_base.py", line 186, in sync_all
stream.sync()
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/streams/core.py", line 458, in sync
self._sync_records()
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/streams/core.py", line 440, in _sync_records
raise ex
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/streams/core.py", line 433, in _sync_records
self._increment_stream_state(row_dict, partition=partition)
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/streams/core.py", line 366, in _increment_stream_state
is_sorted=self.is_sorted,
File "/Users/cjk/fixd/projects/tap_sample/.venv/lib/python3.7/site-packages/singer_sdk/helpers/_state.py", line 206, in increment_state
f"Unsorted data detected in stream. Latest value '{new_rk_value}' is "
singer_sdk.exceptions.InvalidStreamSortException: Unsorted data detected in stream. Latest value '2021-01-01T01:01.100Z' is smaller than previous max '2021-01-01T01:01.100Z'.
```