Query Text Size Over Limit

Getting an error with default batch size: snowflake.connector.errors.InterfaceError: 250003 (08001): 413 Request Entity Too Large: post https://REMOVED.us-east-1.aws.privatelink.snowflakecomputing.com:443/queries/v1/query-request?requestId=REMOVED&request_guid= REMOVED

I believe this is because the batch of 5000 inserts submitted is above the 1MB limit mentioned here: https://docs.snowflake.net/manuals/user-guide/clients-considerations.html#limits-on-query-text-size

Reducing the batch size might help, but I'm using tap-postgres to sync a large number of tables, so row-size isn't a static quantity. Even a single record could possibly be larger than 1MB.

Full error: CRITICAL (snowflake.connector.errors.InterfaceError) 250003 (08001): 413 Request Entity Too Large: post https://REMOVED.us-east-1.aws.privatelink.snowflakecomputing.com:443/queries/v1/query-request?requestId=REMOVED&request_guid=REMOVED [SQL: INSERT INTO "SF".copyforcefieldhistory (createddate, fieldname, id, newvalue, oldvalue, tablename, __loaded_at) VALUES (%(createddate)s, %(fieldname)s, %(id)s, %(newvalue)s, %(oldvalue)s, %(tablename)s, %(__loaded_at)s)] [parameters: ({REMOVED} ... displaying 10 of 5000 total bound parameter sets ... {REMOVED})] (Background on this error at: http://sqlalche.me/e/rvf5) Traceback (most recent call last): File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1224, in _execute_context cursor, statement, parameters, context File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 549, in do_executemany cursor.executemany(statement, parameters) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/cursor.py", line 706, in executemany self.execute(command) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/cursor.py", line 481, in execute _is_put_get=_is_put_get) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/cursor.py", line 369, in _execute_helper _no_results=_no_results) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/connection.py", line 807, in cmd_query _include_retry_params=True) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 313, in request timeout=timeout, _include_retry_params=_include_retry_params) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 530, in _post_request _include_retry_params=_include_retry_params) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 608, in fetch **kwargs) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 685, in _request_exec_wrapper raise e File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 648, in _request_exec_wrapper **kwargs) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 873, in _request_exec raise err File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 842, in _request_exec u'sqlstate': SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED File "/usr/local/lib/python3.7/site-packages/snowflake/connector/errors.py", line 100, in errorhandler_wrapper connection.errorhandler(connection, cursor, errorclass, errorvalue) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/errors.py", line 73, in default_errorhandler done_format_msg=errorvalue.get(u'done_format_msg')) snowflake.connector.errors.InterfaceError: 250003 (08001): 413 Request Entity Too Large: post https://REMOVED.us-east-1.aws.privatelink.snowflakecomputing.com:443/queries/v1/query-request?requestId=REMOVED&request_guid=REMOVED The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/usr/local/bin/target-snowflake", line 11, in <module> load_entry_point('target-snowflake', 'console_scripts', 'target-snowflake')() File "/target-snowflake/target_snowflake/__init__.py", line 85, in main raise exc File "/target-snowflake/target_snowflake/__init__.py", line 72, in main main_implementation() File "/target-snowflake/target_snowflake/__init__.py", line 64, in main_implementation process_input(config, input) File "/target-snowflake/target_snowflake/__init__.py", line 40, in process_input target.process_line(line) File "/target-snowflake/target_snowflake/target_snowflake.py", line 262, in process_line self.flush_records(stream) File "/target-snowflake/target_snowflake/target_snowflake.py", line 418, in flush_records self.loaders[stream].load(self.rows[stream].values()) File "/target-snowflake/target_snowflake/snowflake_loader.py", line 86, in wrapper return func(self, *args, **kwargs) File "/target-snowflake/target_snowflake/snowflake_loader.py", line 307, in load connection.execute(self.table.insert(), data) File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute return meth(self, multiparams, params) File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection return connection._execute_clauseelement(self, multiparams, params) File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement distilled_params, File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1248, in _execute_context e, statement, parameters, cursor, context File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1466, in _handle_dbapi_exception util.raise_from_cause(sqlalchemy_exception, exc_info) File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause reraise(type(exception), exception, tb=exc_tb, cause=cause) File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise raise value.with_traceback(tb) File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1224, in _execute_context cursor, statement, parameters, context File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 549, in do_executemany cursor.executemany(statement, parameters) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/cursor.py", line 706, in executemany self.execute(command) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/cursor.py", line 481, in execute _is_put_get=_is_put_get) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/cursor.py", line 369, in _execute_helper _no_results=_no_results) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/connection.py", line 807, in cmd_query _include_retry_params=True) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 313, in request timeout=timeout, _include_retry_params=_include_retry_params) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 530, in _post_request _include_retry_params=_include_retry_params) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 608, in fetch **kwargs) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 685, in _request_exec_wrapper raise e File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 648, in _request_exec_wrapper **kwargs) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 873, in _request_exec raise err File "/usr/local/lib/python3.7/site-packages/snowflake/connector/network.py", line 842, in _request_exec u'sqlstate': SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED File "/usr/local/lib/python3.7/site-packages/snowflake/connector/errors.py", line 100, in errorhandler_wrapper connection.errorhandler(connection, cursor, errorclass, errorvalue) File "/usr/local/lib/python3.7/site-packages/snowflake/connector/errors.py", line 73, in default_errorhandler done_format_msg=errorvalue.get(u'done_format_msg')) sqlalchemy.exc.InterfaceError: (snowflake.connector.errors.InterfaceError) 250003 (08001): 413 Request Entity Too Large: post https://REMOVED.us-east-1.aws.privatelink.snowflakecomputing.com:443/queries/v1/query-request?requestId= REMOVED&request_guid= REMOVED