Migration commit from heyvince.co to heyvince.ca

dft/commands/load.py

@@ -0,0 +1,35 @@
+import click
+from ..base import config, files, sources
+from ..base.config import config as cfg
+from sqlalchemy import create_engine, Table, MetaData
+from sqlalchemy.sql import select, insert
+import pandas as pd
+from datetime import datetime
+
+@click.command()
+def load():
+    click.echo("Loading data from your query to your target db...")
+    #TODO: Validate init is done
+
+    # Iterate over query-paths
+    query_paths = files.get_filepaths(cfg["query-paths"])
+    for filepath in query_paths:
+        conf, query = files.process_query_file(filepath)
+
+        import pdb; pdb.set_trace()
+        
+        # Do insert data to target db
+        src = sources.get_source(conf["extract_from"], filepath)
+        rst_src = src.read_query(query)
+
+        cnt_to = sources.get_source(
+            cnt_name=cfg["target-name"],
+            filepath=filepath
+        )
+
+        # add batch_id to df
+        #df["BatchID"] = 20100
+        # Manipulate the DataFrame if necessary (e.g., filtering, adding new columns)
+        # Example: df['new_column'] = df['column1'] * 2
+
+        #df.to_sql(cnt_to.filename, con=cnt_to.engine, if_exists='append', index=False)