Adding time_extracted and bookmark_properties #55
Conversation
singer/catalog.py
Outdated
| key_properties=None, schema=None, replication_key=None, | ||
| is_view=None, database=None, table=None, row_count=None, | ||
| stream_alias=None, metadata=None): | ||
| stream_alias=None, metadata=None, bookmark_properties=None): |
There was a problem hiding this comment.
You should not need to change the catalog at all. You can revert all the changes on this file.
singer/messages.py
Outdated
| ''' | ||
|
|
||
| def __init__(self, stream, record, version=None): | ||
| def __init__(self, stream, record, time_extracted=None, version=None): |
There was a problem hiding this comment.
You should add time_extracted after version in case someone is calling RecordMessage(stream, record, version).
There was a problem hiding this comment.
We should require time_extracted to be an "aware" datetime.datetime object. Maybe just add a check here to validate that it's either None or a datetime.datetime with a tzinfo property.
singer/messages.py
Outdated
| 'record': self.record, | ||
| } | ||
| if self.time_extracted is not None: | ||
| result['time_extracted'] = self.time_extracted |
There was a problem hiding this comment.
I think you'll need to change this to convert the timestamp to ISO format.
…is set to UTC for time_extracted
singer/messages.py
Outdated
| self.record = record | ||
| self.version = version | ||
| self.time_extracted = time_extracted | ||
| if time_extracted is not None: |
There was a problem hiding this comment.
You can just do if time_extracted:. You don't need the is not None. I know we have the is not None elsewhere in here but we've since learned that omitting the is not None is the preferred style.
singer/messages.py
Outdated
| self.time_extracted = time_extracted | ||
| if time_extracted is not None: | ||
| if not u.is_aware_datetime(time_extracted): | ||
| raise Exception("'time_extracted' must be an aware " |
There was a problem hiding this comment.
I'd raise ValueError here.
singer/messages.py
Outdated
| } | ||
| if self.version is not None: | ||
| result['version'] = self.version | ||
| if self.time_extracted is not None: |
singer/messages.py
Outdated
| 'schema': self.schema, | ||
| 'key_properties': self.key_properties | ||
| } | ||
| if self.bookmark_properties is not None: |
Version 0.2.0 includes the 'time_extracted' and 'bookmark_properties' that correspond to these changes