Setting property to `null` in stream map config fails to remove it
Really not a big deal for me, but I was messing around with Mappers trying to learn some things and the example https://sdk.meltano.com/en/latest/stream_maps.html?highlight=transform#:~:text=stream%20called%20%27customers%27-,%22email%22%3A%20null%2C,-//%20drop%20the%20PII
Doesn't work for me (the field still exists in my record). I debated on putting this in the https://github.com/MeltanoLabs/meltano-map-transform repo, but I'm pretty sure this is an SDK issue as other mappings work correctly like .upper()
, and .lower()
)
Replication steps
- Clone https://github.com/MeltanoLabs/meltano-map-transform
poetry install
- Create a file to test with I called mine out (see below)
- Create config.json (see example below)
cat out | poetry run meltano-map-transform --config config.json > out_mapped
- I'd expect name to be null (or not exist) instead see out_mapped below
Files to create
out
{"type": "SCHEMA", "stream": "space", "schema": {"properties": {"id": {"type": ["string"]}, "name": {"type": ["null", "string"]}, "private": {"type": ["null", "boolean"]}, "avatar": {"type": ["null", "string"]}, "statuses": {"items": {"properties": {"id": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "orderindex": {"type": ["null", "integer", "string"]}, "color": {"type": ["null", "string"]}}, "type": ["object", "null"]}, "type": ["array", "null"]}, "members": {"items": {"properties": {"user": {"properties": {"id": {"type": ["null", "integer"]}, "username": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "color": {"type": ["null", "string"]}, "profilePicture": {"type": ["null", "string"]}, "initials": {"type": ["null", "string"]}, "role": {"type": ["null", "integer"]}, "custom_role": {"type": ["null", "integer"]}, "last_active": {"type": ["null", "string"]}, "date_joined": {"type": ["null", "string"]}, "date_invited": {"type": ["null", "string"]}}, "type": "object"}, "invited_by": {"properties": {"id": {"type": ["null", "integer"]}, "username": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "color": {"type": ["null", "string"]}, "profilePicture": {"type": ["null", "string"]}, "initials": {"type": ["null", "string"]}, "role": {"type": ["null", "integer"]}, "custom_role": {"type": ["null", "integer"]}, "last_active": {"type": ["null", "string"]}, "date_joined": {"type": ["null", "string"]}, "date_invited": {"type": ["null", "string"]}}, "type": ["object", "null"]}}, "type": ["null", "object"]}, "type": "array"}, "multiple_assignees": {"type": ["null", "boolean"]}, "features": {"properties": {"due_dates": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "sprints": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "time_tracking": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "points": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "custom_items": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "time_estimates": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "check_unresolved": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "zoom": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "milestones": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "remap_dependencies": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "dependency_warning": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "multiple_assignees": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "emails": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}}, "type": "object"}, "archived": {"type": ["null", "boolean"]}}, "type": "object"}, "key_properties": ["id"]}
{"type": "RECORD", "stream": "space", "record": {"id": "54078699", "name": "Archive me", "private": false, "statuses": [{"id": "p54078699_NRY41cPD", "status": "to do", "type": "open", "orderindex": 0, "color": "#d3d3d3"}, {"id": "p54078699_o8oSalOi", "status": "complete", "type": "closed", "orderindex": 1, "color": "#6bc950"}], "multiple_assignees": true, "features": {"due_dates": {"enabled": true, "start_date": true, "remap_due_dates": true, "remap_closed_due_date": false}, "sprints": {"enabled": true}, "time_tracking": {"enabled": true, "harvest": false, "rollup": true}, "points": {"enabled": true}, "custom_items": {"enabled": false}, "priorities": {"enabled": true, "priorities": [{"id": "1", "priority": "urgent", "color": "#f50000", "orderindex": "1"}, {"id": "2", "priority": "high", "color": "#ffcc00", "orderindex": "2"}, {"id": "3", "priority": "normal", "color": "#6fddff", "orderindex": "3"}, {"id": "4", "priority": "low", "color": "#d8d8d8", "orderindex": "4"}]}, "tags": {"enabled": true}, "time_estimates": {"enabled": true, "rollup": true, "per_assignee": false}, "check_unresolved": {"enabled": true, "subtasks": null, "checklists": null, "comments": null}, "zoom": {"enabled": true}, "milestones": {"enabled": true}, "custom_fields": {"enabled": true}, "remap_dependencies": {"enabled": true}, "dependency_warning": {"enabled": true}, "multiple_assignees": {"enabled": true}, "emails": {"enabled": true}}, "archived": true}, "time_extracted": "2022-03-28T17:45:19.467386Z"}
config.json
{
"stream_maps": {
"space": {
"name": null
}
}
}
out_mapped
{"type": "SCHEMA", "stream": "space", "schema": {"properties": {"id": {"type": ["string"]}, "name": {"type": ["null", "string"]}, "private": {"type": ["null", "boolean"]}, "avatar": {"type": ["null", "string"]}, "statuses": {"items": {"properties": {"id": {"type": ["null", "string"]}, "status": {"type": ["null", "string"]}, "orderindex": {"type": ["null", "integer", "string"]}, "color": {"type": ["null", "string"]}}, "type": ["object", "null"]}, "type": ["array", "null"]}, "members": {"items": {"properties": {"user": {"properties": {"id": {"type": ["null", "integer"]}, "username": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "color": {"type": ["null", "string"]}, "profilePicture": {"type": ["null", "string"]}, "initials": {"type": ["null", "string"]}, "role": {"type": ["null", "integer"]}, "custom_role": {"type": ["null", "integer"]}, "last_active": {"type": ["null", "string"]}, "date_joined": {"type": ["null", "string"]}, "date_invited": {"type": ["null", "string"]}}, "type": "object"}, "invited_by": {"properties": {"id": {"type": ["null", "integer"]}, "username": {"type": ["null", "string"]}, "email": {"type": ["null", "string"]}, "color": {"type": ["null", "string"]}, "profilePicture": {"type": ["null", "string"]}, "initials": {"type": ["null", "string"]}, "role": {"type": ["null", "integer"]}, "custom_role": {"type": ["null", "integer"]}, "last_active": {"type": ["null", "string"]}, "date_joined": {"type": ["null", "string"]}, "date_invited": {"type": ["null", "string"]}}, "type": ["object", "null"]}}, "type": ["null", "object"]}, "type": "array"}, "multiple_assignees": {"type": ["null", "boolean"]}, "features": {"properties": {"due_dates": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "sprints": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "time_tracking": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "points": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "custom_items": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "time_estimates": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "check_unresolved": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "zoom": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "milestones": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "remap_dependencies": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "dependency_warning": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "multiple_assignees": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}, "emails": {"properties": {"enabled": {"type": ["boolean", "null"]}, "start_date": {"type": ["null", "boolean"]}, "remap_due_dates": {"type": ["null", "boolean"]}, "remap_closed_due_date": {"type": ["null", "boolean"]}, "harvest": {"type": ["null", "boolean"]}, "rollup": {"type": ["null", "boolean"]}, "per_assignee": {"type": ["null", "boolean"]}, "subtasks": {"type": ["null", "boolean"]}, "checklists": {"type": ["null", "boolean"]}}, "type": ["object", "null"]}}, "type": "object"}, "archived": {"type": ["null", "boolean"]}}, "type": "object"}, "key_properties": ["id"]}
{"type": "RECORD", "stream": "space", "record": {"id": "54078699", "name": "Archive me", "private": false, "statuses": [{"id": "p54078699_NRY41cPD", "status": "to do", "type": "open", "orderindex": 0, "color": "#d3d3d3"}, {"id": "p54078699_o8oSalOi", "status": "complete", "type": "closed", "orderindex": 1, "color": "#6bc950"}], "multiple_assignees": true, "features": {"due_dates": {"enabled": true, "start_date": true, "remap_due_dates": true, "remap_closed_due_date": false}, "sprints": {"enabled": true}, "time_tracking": {"enabled": true, "harvest": false, "rollup": true}, "points": {"enabled": true}, "custom_items": {"enabled": false}, "priorities": {"enabled": true, "priorities": [{"id": "1", "priority": "urgent", "color": "#f50000", "orderindex": "1"}, {"id": "2", "priority": "high", "color": "#ffcc00", "orderindex": "2"}, {"id": "3", "priority": "normal", "color": "#6fddff", "orderindex": "3"}, {"id": "4", "priority": "low", "color": "#d8d8d8", "orderindex": "4"}]}, "tags": {"enabled": true}, "time_estimates": {"enabled": true, "rollup": true, "per_assignee": false}, "check_unresolved": {"enabled": true, "subtasks": null, "checklists": null, "comments": null}, "zoom": {"enabled": true}, "milestones": {"enabled": true}, "custom_fields": {"enabled": true}, "remap_dependencies": {"enabled": true}, "dependency_warning": {"enabled": true}, "multiple_assignees": {"enabled": true}, "emails": {"enabled": true}}, "archived": true}, "time_extracted": "2022-03-28T18:15:07.229094Z"}
Edited by Edgar R. Mondragón