Migration commit from heyvince.co to heyvince.ca

dft/commands/deploy.py

@@ -0,0 +1,7 @@
+import click
+
+@click.command()
+@click.option('-c', default="dft_project.yml", help='Build your flow with scheduler and deploy them to your server.')
+def deploy(config):
+    click.echo("Deploying your query files to server...")
+

dft/commands/init.py

@@ -0,0 +1,16 @@
+import click
+
+@click.command()
+def init():
+    click.echo("Initializing your dft environment...")
+    # Create folder .dft inside home
+    # Add profile.yml file inside .dft folder
+    # Init Alembic with folder migrations
+    # Move alembic.ini
+    # Create models folder inside migrations
+    # Ajouter migration/models/ base.py
+    # Create files inside folder models
+    # Generate first init.py with dftbatch + base
+    # Add to env.py from migration import models + models.Base.metadata
+    # Create folder query
+    # Add file with dft_projects.yml

dft/commands/load.py

@@ -0,0 +1,35 @@
+import click
+from ..base import config, files, sources
+from ..base.config import config as cfg
+from sqlalchemy import create_engine, Table, MetaData
+from sqlalchemy.sql import select, insert
+import pandas as pd
+from datetime import datetime
+
+@click.command()
+def load():
+    click.echo("Loading data from your query to your target db...")
+    #TODO: Validate init is done
+
+    # Iterate over query-paths
+    query_paths = files.get_filepaths(cfg["query-paths"])
+    for filepath in query_paths:
+        conf, query = files.process_query_file(filepath)
+
+        import pdb; pdb.set_trace()
+        
+        # Do insert data to target db
+        src = sources.get_source(conf["extract_from"], filepath)
+        rst_src = src.read_query(query)
+
+        cnt_to = sources.get_source(
+            cnt_name=cfg["target-name"],
+            filepath=filepath
+        )
+
+        # add batch_id to df
+        #df["BatchID"] = 20100
+        # Manipulate the DataFrame if necessary (e.g., filtering, adding new columns)
+        # Example: df['new_column'] = df['column1'] * 2
+
+        #df.to_sql(cnt_to.filename, con=cnt_to.engine, if_exists='append', index=False)

dft/commands/migrate.py

@@ -0,0 +1,32 @@
+import click
+from ..base import files, sources
+from ..base.config import config as cfg
+from ..base import migrator as mig
+from alembic.config import Config
+
+
+@click.command()
+@click.option('-m', default="", help='Description of your migration')
+def createmigration(m):
+    #TODO: Validate init is done
+    click.echo("Generating migration files... but first lets remove your models")
+    
+    # Delete files from model folder
+    files.delete_file(cfg["migration-path"]+"/models", donot=["base.py"])
+
+    # Iterate over query-paths
+    query_paths = files.get_filepaths(cfg["query-paths"])
+    for filepath in query_paths:
+        conf, query = files.process_query_file(filepath)
+        src = sources.get_source(conf["extract_from"],filepath)
+        src.generate_model(query)
+    
+    # Generate Init file
+    files.generate_init(query_paths)
+    mig.create_migration_file(script_message=str(m))
+    click.echo("Creation migration is done !")
+
+
+@click.command()
+def migrate():
+    mig.migrate()