36 lines
1.1 KiB
Python
36 lines
1.1 KiB
Python
import click
|
|
from ..base import config, files, sources
|
|
from ..base.config import config as cfg
|
|
from sqlalchemy import create_engine, Table, MetaData
|
|
from sqlalchemy.sql import select, insert
|
|
import pandas as pd
|
|
from datetime import datetime
|
|
|
|
@click.command()
|
|
def load():
|
|
click.echo("Loading data from your query to your target db...")
|
|
#TODO: Validate init is done
|
|
|
|
# Iterate over query-paths
|
|
query_paths = files.get_filepaths(cfg["query-paths"])
|
|
for filepath in query_paths:
|
|
conf, query = files.process_query_file(filepath)
|
|
|
|
import pdb; pdb.set_trace()
|
|
|
|
# Do insert data to target db
|
|
src = sources.get_source(conf["extract_from"], filepath)
|
|
rst_src = src.read_query(query)
|
|
|
|
cnt_to = sources.get_source(
|
|
cnt_name=cfg["target-name"],
|
|
filepath=filepath
|
|
)
|
|
|
|
# add batch_id to df
|
|
#df["BatchID"] = 20100
|
|
# Manipulate the DataFrame if necessary (e.g., filtering, adding new columns)
|
|
# Example: df['new_column'] = df['column1'] * 2
|
|
|
|
#df.to_sql(cnt_to.filename, con=cnt_to.engine, if_exists='append', index=False)
|