2828VAR_PLUS_OFFSET_PATTERN = re .compile (r"^(?P<left>@[A-Za-z0-9_]+)\s*\+\s*(?P<offset>-?\d+)$" )
2929OFFSET_PLUS_VAR_PATTERN = re .compile (r"^(?P<offset>-?\d+)\s*\+\s*(?P<right>@[A-Za-z0-9_]+)$" )
3030
31+ IGNORED_OUTPUT_COLUMNS = frozenset (
32+ {
33+ "RDB_LAST_REFRESH_TIME" ,
34+ "LAB_RPT_LAST_UPDATE_DT" ,
35+ }
36+ )
37+
3138
3239@dataclass (frozen = True )
3340class DeclareEntry :
@@ -871,6 +878,65 @@ def logical_change_step(change: dict[str, object]) -> int | None:
871878 return sorted (finalized , key = lambda item : (item .schema_name .lower (), item .table_name .lower (), item .where_fields ))
872879
873880
881+ def logical_change_step (change : dict [str , object ]) -> int | None :
882+ metadata = change .get ("metadata" )
883+ if not isinstance (metadata , dict ):
884+ return None
885+ step_value = metadata .get ("step" )
886+ try :
887+ return int (step_value )
888+ except (TypeError , ValueError ):
889+ return None
890+
891+
892+ def step_numbers_from_manifest_or_changes (
893+ manifest : dict [str , object ],
894+ logical_changes : list [dict [str , object ]],
895+ ) -> list [int ]:
896+ ordered_steps : list [int ] = []
897+ seen : set [int ] = set ()
898+
899+ manifest_steps = manifest .get ("steps" )
900+ if isinstance (manifest_steps , list ):
901+ for step in manifest_steps :
902+ if not isinstance (step , dict ):
903+ continue
904+ step_value = step .get ("step" )
905+ try :
906+ step_number = int (step_value )
907+ except (TypeError , ValueError ):
908+ continue
909+ if step_number not in seen :
910+ seen .add (step_number )
911+ ordered_steps .append (step_number )
912+
913+ for change in logical_changes :
914+ step_number = logical_change_step (change )
915+ if step_number is None or step_number in seen :
916+ continue
917+ seen .add (step_number )
918+ ordered_steps .append (step_number )
919+
920+ return ordered_steps
921+
922+
923+ def build_renderable_scaffolds (
924+ logical_changes_obj : list [dict [str , object ]],
925+ declare_entries : list [DeclareEntry ],
926+ ) -> list [SelectScaffold ]:
927+ scaffolds = build_scaffolds (logical_changes_obj , declare_entries )
928+ known_lookup_keys_file = Path (__file__ ).with_name ("known_lookup_keys.json" )
929+ if known_lookup_keys_file .exists ():
930+ try :
931+ known_lookup_keys_obj = json .loads (known_lookup_keys_file .read_text (encoding = "utf-8" ))
932+ if isinstance (known_lookup_keys_obj , dict ):
933+ scaffolds = apply_known_lookup_keys (scaffolds , known_lookup_keys_obj .get ("known_tables" ))
934+ except (OSError , json .JSONDecodeError ) as error :
935+ print (f"Warning: Could not load { known_lookup_keys_file } { error } , file = sys .stderr )
936+ scaffolds = consolidate_fk_scaffolds (scaffolds )
937+ return [s for s in scaffolds if not s .table_name .lower ().startswith ("nrt_" )]
938+
939+
874940def apply_known_lookup_keys (
875941 scaffolds : list [SelectScaffold ],
876942 known_lookup_keys : dict [str , dict [str , object ]] | None ,
@@ -998,6 +1064,41 @@ def display_path(path_value: str) -> str:
9981064 return path_value .replace ("\\ " , "/" )
9991065
10001066
1067+ def write_step_query_files (
1068+ logical_output_dir : Path ,
1069+ manifest : dict [str , object ],
1070+ logical_changes_obj : list [dict [str , object ]],
1071+ declare_lines : list [str ],
1072+ declare_entries : list [DeclareEntry ],
1073+ columns_by_table : dict [tuple [str , str ], list [str ]] | None ,
1074+ primary_keys_by_table : dict [tuple [str , str ], frozenset [str ]] | None ,
1075+ foreign_keys_by_source : dict [tuple [str , str , str ], tuple [str , str , str ]] | None ,
1076+ generated_always_columns : set [tuple [str , str , str ]] | None ,
1077+ auto_datetime_defaults : set [tuple [str , str , str ]] | None ,
1078+ ) -> None :
1079+ for step_number in step_numbers_from_manifest_or_changes (manifest , logical_changes_obj ):
1080+ step_dir = logical_output_dir / f"step-{ step_number }
1081+ step_dir .mkdir (parents = True , exist_ok = True )
1082+ cumulative_changes = [
1083+ change
1084+ for change in logical_changes_obj
1085+ if (logical_change_step (change ) or 0 ) <= step_number
1086+ ]
1087+ step_scaffolds = build_renderable_scaffolds (cumulative_changes , declare_entries )
1088+ step_sql = render_sql (
1089+ manifest ,
1090+ declare_lines ,
1091+ declare_entries ,
1092+ step_scaffolds ,
1093+ columns_by_table ,
1094+ primary_keys_by_table ,
1095+ foreign_keys_by_source ,
1096+ generated_always_columns ,
1097+ auto_datetime_defaults ,
1098+ )
1099+ (step_dir / "query.sql" ).write_text (step_sql , encoding = "utf-8" )
1100+
1101+
10011102def render_sql (
10021103 manifest : dict [str , object ],
10031104 declare_lines : list [str ],
@@ -1076,8 +1177,8 @@ def render_sql(
10761177 if normalize_identifier (s ) == normalize_identifier (scaffold .schema_name )
10771178 and normalize_identifier (t ) == normalize_identifier (scaffold .table_name )
10781179 )
1079- select_excluded_columns = pk_columns | generated_excluded_for_table | auto_excluded_for_table
1080- json_excluded_columns = pk_columns | generated_excluded_for_table | auto_excluded_for_table
1180+ select_excluded_columns = pk_columns | generated_excluded_for_table | auto_excluded_for_table | IGNORED_OUTPUT_COLUMNS
1181+ json_excluded_columns = pk_columns | generated_excluded_for_table | auto_excluded_for_table | IGNORED_OUTPUT_COLUMNS
10811182 select_columns : list [str ] | None = None
10821183 if columns_by_table is not None :
10831184 all_columns = columns_by_table .get (table_key )
@@ -1116,17 +1217,7 @@ def generate_rdb_selects_from_manifest(
11161217
11171218 declare_lines = extract_declare_block (inserts_text )
11181219 declare_entries = parse_declare_entries (declare_lines )
1119- scaffolds = build_scaffolds (logical_changes_obj , declare_entries )
1120- known_lookup_keys_file = Path (__file__ ).with_name ("known_lookup_keys.json" )
1121- if known_lookup_keys_file .exists ():
1122- try :
1123- known_lookup_keys_obj = json .loads (known_lookup_keys_file .read_text (encoding = "utf-8" ))
1124- if isinstance (known_lookup_keys_obj , dict ):
1125- scaffolds = apply_known_lookup_keys (scaffolds , known_lookup_keys_obj .get ("known_tables" ))
1126- except (OSError , json .JSONDecodeError ) as error :
1127- print (f"Warning: Could not load { known_lookup_keys_file } { error } , file = sys .stderr )
1128- scaffolds = consolidate_fk_scaffolds (scaffolds )
1129- scaffolds = [s for s in scaffolds if not s .table_name .lower ().startswith ("nrt_" )]
1220+ scaffolds = build_renderable_scaffolds (logical_changes_obj , declare_entries )
11301221 logical_database = str (manifest .get ("logical_database" ) or "RDB_MODERN" )
11311222 columns_by_table , primary_keys_by_table , foreign_keys_by_source , generated_always_columns , auto_datetime_defaults = load_rdb_column_metadata (logical_database )
11321223 output_sql = render_sql (
@@ -1141,6 +1232,19 @@ def generate_rdb_selects_from_manifest(
11411232 auto_datetime_defaults ,
11421233 )
11431234
1235+ write_step_query_files (
1236+ logical_changes_path .parent ,
1237+ manifest ,
1238+ logical_changes_obj ,
1239+ declare_lines ,
1240+ declare_entries ,
1241+ columns_by_table ,
1242+ primary_keys_by_table ,
1243+ foreign_keys_by_source ,
1244+ generated_always_columns ,
1245+ auto_datetime_defaults ,
1246+ )
1247+
11441248 final_output_path = output_path if output_path is not None else manifest_path .with_name ("rdb-selects.sql" )
11451249 final_output_path .write_text (output_sql , encoding = "utf-8" )
11461250 return final_output_path , len (scaffolds )
0 commit comments