From 9132ac32752c9a4e18c0d6473fcef537fdd0e6cd Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Sat, 10 Jun 2023 12:21:43 +0530 Subject: [PATCH 01/10] Fix queries and selected true --- tap_redshift/__init__.py | 50 ++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index 90b634e..12a763f 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -48,18 +48,17 @@ 'password', 'start_date' ] - -STRING_TYPES = {'char', 'character', 'nchar', 'bpchar', 'text', 'varchar', - 'character varying', 'nvarchar'} - BYTES_FOR_INTEGER_TYPE = { 'int2': 2, + 'smallint': 2, 'int': 4, 'int4': 4, - 'int8': 8 + 'integer': 4, + 'int8': 8, + 'bigint': 8 } -FLOAT_TYPES = {'float', 'float4', 'float8'} +FLOAT_TYPES = {'float', 'float4', 'float8', 'double precision', 'real'} DATE_TYPES = {'date'} @@ -76,38 +75,35 @@ def discover_catalog(conn, db_schema): conn, """ SELECT table_name, table_type - FROM INFORMATION_SCHEMA.Tables + FROM SVV_TABLES WHERE table_schema = '{}' """.format(db_schema)) column_specs = select_all( conn, """ - SELECT c.table_name, c.ordinal_position, c.column_name, c.udt_name, - c.is_nullable - FROM INFORMATION_SCHEMA.Tables t - JOIN INFORMATION_SCHEMA.Columns c - ON c.table_name = t.table_name AND - c.table_schema = t.table_schema - WHERE t.table_schema = '{}' - ORDER BY c.table_name, c.ordinal_position + SELECT table_name, ordinal_position, column_name, data_type, is_nullable + FROM SVV_COLUMNS + WHERE table_schema = '{}' + ORDER BY table_name, ordinal_position """.format(db_schema)) pk_specs = select_all( conn, """ - SELECT kc.table_name, kc.column_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kc - ON kc.table_name = tc.table_name AND - kc.table_schema = tc.table_schema AND - kc.constraint_name = tc.constraint_name - WHERE tc.constraint_type = 'PRIMARY KEY' AND - tc.table_schema = '{}' + SELECT + c.relname AS table_name, + a.attname AS column_name + FROM + pg_catalog.pg_constraint AS con + JOIN pg_catalog.pg_class AS c ON c.oid = con.conrelid + JOIN pg_catalog.pg_attribute AS a ON a.attrelid = c.oid AND a.attnum = ANY(con.conkey) + JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = '{}' AND contype IN ('p') ORDER BY - tc.table_schema, - tc.table_name, - kc.ordinal_position + n.nspname, + table_name, + a.attnum; """.format(db_schema)) entries = [] @@ -204,7 +200,7 @@ def create_column_metadata( db_name, cols, is_view, table_name, key_properties=[]): mdata = metadata.new() - mdata = metadata.write(mdata, (), 'selected-by-default', False) + mdata = metadata.write(mdata, (), 'selected-by-default', True) if not is_view: mdata = metadata.write( mdata, (), 'table-key-properties', key_properties) From 32672ccf836c1871ef2224518698d442e6852723 Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Sat, 10 Jun 2023 12:31:35 +0530 Subject: [PATCH 02/10] Update setup.py --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index e2dc574..c51d588 100644 --- a/setup.py +++ b/setup.py @@ -48,10 +48,10 @@ def find_version(*paths): packages=find_packages(), install_requires=[ 'attrs==18.2.0', - 'pendulum>=1.2.0<2.0.0', - 'singer-python>=5.12.2<6.0.0', - 'backoff>=1.3.2<2.0.0', - 'psycopg2>=2.9.3<3.0.0', + 'pendulum>=1.2.0,<2.0.0', + 'singer-python>=5.12.2,<6.0.0', + 'backoff>=1.3.2,<2.0.0', + 'psycopg2>=2.9.3,<3.0.0', ], setup_requires=[ 'pytest-runner>=2.11,<3.0a', From a001d6cbefe85936bebabf4ebafec68014e8c382 Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Sat, 10 Jun 2023 12:35:22 +0530 Subject: [PATCH 03/10] Update __init__.py --- tap_redshift/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index 12a763f..c302e0a 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -48,6 +48,10 @@ 'password', 'start_date' ] + +STRING_TYPES = {'char', 'character', 'nchar', 'bpchar', 'text', 'varchar', + 'character varying', 'nvarchar'} + BYTES_FOR_INTEGER_TYPE = { 'int2': 2, 'smallint': 2, From a189e51231c4aacee7ff6acfe01423e1fef47d59 Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 11:36:32 +0530 Subject: [PATCH 04/10] Add support for super data type --- tap_redshift/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index c302e0a..aa40604 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -188,6 +188,10 @@ def schema_for_column(c): result.type = 'string' result.format = 'date' + elif column_type == 'super': + result.type = 'json' + result.format = 'json' + else: result = Schema(None, inclusion='unsupported', From b5184821c845ab134a3294f9786d2d4e87add343 Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 12:06:43 +0530 Subject: [PATCH 05/10] Support for bool --- tap_redshift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index aa40604..47727f2 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -162,7 +162,7 @@ def schema_for_column(c): inclusion = 'available' result = Schema(inclusion=inclusion) - if column_type == 'bool': + if column_type IN ['bool', 'boolean']: result.type = 'boolean' elif column_type in BYTES_FOR_INTEGER_TYPE: From bc7d5c1d9f602fd8b1bd5a5f545d614b477a189c Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 14:09:10 +0530 Subject: [PATCH 06/10] Fix syntax --- tap_redshift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index 47727f2..262793b 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -162,7 +162,7 @@ def schema_for_column(c): inclusion = 'available' result = Schema(inclusion=inclusion) - if column_type IN ['bool', 'boolean']: + if column_type in ['bool', 'boolean']: result.type = 'boolean' elif column_type in BYTES_FOR_INTEGER_TYPE: From b9df04ce8d7a5c0ae847f0ec3dde78a5fc2d753d Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 14:20:38 +0530 Subject: [PATCH 07/10] Default value for nullable --- tap_redshift/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index 262793b..7798392 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -122,6 +122,7 @@ def discover_catalog(conn, db_schema): table_types = dict(table_spec) for items in table_columns: + LOGGER.debug("Processing table_name = items['name'] qualified_table_name = '{}.{}'.format(db_schema, table_name) cols = items['columns'] @@ -157,8 +158,10 @@ def do_discover(conn, db_schema): def schema_for_column(c): '''Returns the Schema object for the given Column.''' + LOGGER.debug(c] column_type = c['type'].lower() - column_nullable = c['nullable'].lower() + column_nullable = "true" if ('nullable' not in c or c['nullable'] is None) else c['nullable'].lower() + inclusion = 'available' result = Schema(inclusion=inclusion) From 4db9f923161b412eb18d9fe2e2fb44644a91ba8c Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 14:25:50 +0530 Subject: [PATCH 08/10] Update __init__.py --- tap_redshift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index 7798392..e0e3290 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -122,8 +122,8 @@ def discover_catalog(conn, db_schema): table_types = dict(table_spec) for items in table_columns: - LOGGER.debug("Processing table_name = items['name'] + LOGGER.debug("Processing" + table_name) qualified_table_name = '{}.{}'.format(db_schema, table_name) cols = items['columns'] schema = Schema(type='object', From 3e4701c3cfe325c0591079003c95912f23d078e0 Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 14:30:57 +0530 Subject: [PATCH 09/10] Update __init__.py --- tap_redshift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index e0e3290..57b791e 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -158,7 +158,7 @@ def do_discover(conn, db_schema): def schema_for_column(c): '''Returns the Schema object for the given Column.''' - LOGGER.debug(c] + LOGGER.debug(c) column_type = c['type'].lower() column_nullable = "true" if ('nullable' not in c or c['nullable'] is None) else c['nullable'].lower() From 4acdc5f11da9c87146b5d898a5b36dcb56819ba0 Mon Sep 17 00:00:00 2001 From: Shivam Kapoor Date: Mon, 12 Jun 2023 15:17:16 +0530 Subject: [PATCH 10/10] Get metadata object --- tap_redshift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_redshift/__init__.py b/tap_redshift/__init__.py index 57b791e..82382b6 100644 --- a/tap_redshift/__init__.py +++ b/tap_redshift/__init__.py @@ -320,7 +320,7 @@ def sync_table(connection, catalog_entry, state): if start_date is not None: formatted_start_date = datetime.datetime.strptime( start_date, '%Y-%m-%dT%H:%M:%SZ').astimezone() - + LOGGER.debug(catalog_entry.metadata) replication_key = metadata.to_map(catalog_entry.metadata).get( (), {}).get('replication-key') replication_key_value = None