Search bar for components + connection first step

amphi-etl/amphi/main.py

@@ -23,7 +23,7 @@ def main():
     if args.command == 'start':
         jupyter_command = [
             sys.executable, '-m', 'jupyter', 'lab', 
-            f'--notebook-dir={args.workspace}', f'--port={args.port}', f'--ip={args.ip}'
+            f'--notebook-dir={args.workspace}', f'--port={args.port}', f'--ip={args.ip} --ContentManager.allow_hidden=true'
         ]
         print(f"Running JupyterLab command: {' '.join(jupyter_command)}")
         try:

jupyterlab-amphi/packages/pipeline-components-core/src/components/index.ts

@@ -16,6 +16,8 @@ export { MySQLInput } from './inputs/databases/MySQLInput';
 export { PostgresInput } from './inputs/databases/PostgresInput';
 export { OracleInput } from './inputs/databases/OracleInput';
 export { SqlServerInput } from './inputs/databases/SqlServerInput';
+export { SnowflakeInput } from './inputs/databases/SnowflakeInput';
+
 export { RedditInput } from './inputs/cloud/RedditInput';
 
 // Transforms
@@ -59,6 +61,7 @@ export { PostgresOutput } from './outputs/databases/PostgresOutput';
 export { PineconeOutput } from './outputs/vector-stores/PineconeOutput';
 export { ChromaOutput } from './outputs/vector-stores/ChromaOutput';
 
-// Others
-export { EnvFile } from './other/EnvFile';
-export { EnvVariables } from './other/EnvVariables';
+// Settings
+export { EnvFile } from './settings/EnvFile';
+export { EnvVariables } from './settings/EnvVariables';
+export { Connection } from './settings/Connection';

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/cloud/GoogleSheetsInput.tsx

@@ -36,7 +36,7 @@ export class GoogleSheetsInput extends BaseCoreComponent {
       ],
     };
 
-    super("G. Sheets Input", "googleSheetsInput", "pandas_df_input", [], "input", googleSheetsIcon, defaultConfig, form);
+    super("G. Sheets Input", "googleSheetsInput", "pandas_df_input", [], "inputs", googleSheetsIcon, defaultConfig, form);
   }
 
   public provideImports({ config }): string[] {

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/cloud/RedditInput.tsx

@@ -44,7 +44,7 @@ export class RedditInput extends BaseCoreComponent {
       ],
     };
 
-    super("Reddit Input", "redditInput", "pandas_df_input", [], "input", redditIcon, defaultConfig, form);
+    super("Reddit Input", "redditInput", "pandas_df_input", [], "inputs", redditIcon, defaultConfig, form);
   }
 
   public provideImports({ config }): string[] {

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/cloud/RestInput.tsx

@@ -48,7 +48,7 @@ export class RestInput extends BaseCoreComponent {
       ],
     };
 
-    super("REST Input", "restInput", "pandas_df_input", [], "input", apiIcon, defaultConfig, form);
+    super("REST Input", "restInput", "pandas_df_input", [], "inputs", apiIcon, defaultConfig, form);
   }
 
   public provideImports({ config }): string[] {

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/databases/MySQLInput.tsx

@@ -12,26 +12,30 @@ export class MySQLInput extends BaseCoreComponent {
           label: "Host",
           id: "dbOptions.host",
           placeholder: "Enter database host",
+          connection: 'MySQL',
           advanced: true
         },
         {
           type: "input",
           label: "Port",
           id: "dbOptions.port",
           placeholder: "Enter database port",
+          connection: 'MySQL',
           advanced: true
         },
         {
           type: "input",
           label: "Database Name",
           id: "dbOptions.databaseName",
           placeholder: "Enter database name",
+          connection: 'MySQL'
         },
         {
           type: "input",
           label: "Username",
           id: "dbOptions.username",
           placeholder: "Enter username",
+          connection: "MySQL",
           advanced: true
         },
         {
@@ -40,6 +44,7 @@ export class MySQLInput extends BaseCoreComponent {
           id: "dbOptions.password",
           placeholder: "Enter password",
           inputType: "password",
+          connection: "MySQL",
           advanced: true
         },
         {
@@ -61,7 +66,7 @@ export class MySQLInput extends BaseCoreComponent {
       ],
     };
 
-    super("MySQL Input", "mySQLInput", "pandas_df_input", [], "input", mySQLIcon, defaultConfig, form);
+    super("MySQL Input", "mySQLInput", "pandas_df_input", [], "input.Databases", mySQLIcon, defaultConfig, form);
   }
 
     public provideImports({config}): string[] {
@@ -75,11 +80,16 @@ export class MySQLInput extends BaseCoreComponent {
       const code = `
 # Connect to the MySQL database
 ${uniqueEngineName} = sqlalchemy.create_engine("${connectionString}")
-with ${uniqueEngineName}.connect() as conn:
-    ${outputName} = pd.read_sql(
-        """${sqlQuery}""",
-        con=conn.connection
-    ).convert_dtypes()
+
+# Execute SQL statement
+try:
+    with ${uniqueEngineName}.connect() as conn:
+        ${outputName} = pd.read_sql(
+            """${sqlQuery}""",
+            con=conn.connection
+        ).convert_dtypes()
+finally:
+    ${uniqueEngineName}.dispose()
 `;
       return code;
     }

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/databases/OracleInput.tsx

@@ -60,7 +60,7 @@ export class OracleInput extends BaseCoreComponent {
       ],
     };
 
-    super("Oracle Input", "oracleInput", "pandas_df_input", [], "input", oracleIcon, defaultConfig, form);
+    super("Oracle Input", "oracleInput", "pandas_df_input", [], "inputs.Databases", oracleIcon, defaultConfig, form);
   }
 
   public provideImports({ config }): string[] {
@@ -74,11 +74,16 @@ export class OracleInput extends BaseCoreComponent {
     const code = `
 # Connect to the Oracle database
 ${uniqueEngineName} = sqlalchemy.create_engine("${connectionString}")
-with ${uniqueEngineName}.connect() as conn:
-    ${outputName} = pd.read_sql(
-        """${sqlQuery}""",
-        con=conn.connection
-    ).convert_dtypes()
+
+# Execute SQL statement
+try:
+    with ${uniqueEngineName}.connect() as conn:
+        ${outputName} = pd.read_sql(
+            """${sqlQuery}""",
+            con=conn.connection
+        ).convert_dtypes()
+finally:
+    ${uniqueEngineName}.dispose()
 `;
     return code;
   }

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/databases/PostgresInput.tsx

@@ -67,7 +67,7 @@ export class PostgresInput extends BaseCoreComponent {
       ],
     };
 
-    super("Postgres Input", "postgresInput", "pandas_df_input", [], "input", postgresIcon, defaultConfig, form);
+    super("Postgres Input", "postgresInput", "pandas_df_input", [], "inputs.Databases", postgresIcon, defaultConfig, form);
   }
 
   public provideDependencies({ config }): string[] {
@@ -95,11 +95,16 @@ export class PostgresInput extends BaseCoreComponent {
     const code = `
 # Connect to the PostgreSQL database
 ${uniqueEngineName} = sqlalchemy.create_engine("${connectionString}")
-with ${uniqueEngineName}.connect() as conn:
-    ${outputName} = pd.read_sql(
-        """${sqlQuery}""",
-        con=conn.connection
-    ).convert_dtypes()
+
+# Execute SQL statement
+try:
+    with ${uniqueEngineName}.connect() as conn:
+        ${outputName} = pd.read_sql(
+            """${sqlQuery}""",
+            con=conn.connection
+        ).convert_dtypes()
+finally:
+    ${uniqueEngineName}.dispose()
 `;
     return code;
   }

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/databases/SnowflakeInput.tsx

@@ -0,0 +1,122 @@
+
+import { snowflakeIcon } from '../../../icons';
+import { BaseCoreComponent } from '../../BaseCoreComponent';
+
+export class SnowflakeInput extends BaseCoreComponent {
+  constructor() {
+    const defaultConfig = { dbOptions: { schema: "public", tableName: "" } };
+    const form = {
+      fields: [
+        {
+          type: "input",
+          label: "Account",
+          id: "dbOptions.account",
+          placeholder: "Enter Account",
+          connection: "Snowflake",
+          advanced: true
+        },
+        {
+          type: "input",
+          label: "Database Name",
+          id: "dbOptions.databaseName",
+          connection: "Snowflake",
+          placeholder: "Enter database name",
+        },
+        {
+          type: "input",
+          label: "Username",
+          id: "dbOptions.username",
+          placeholder: "Enter username",
+          connection: "Snowflake",
+          advanced: true
+        },
+        {
+          type: "input",
+          label: "Password",
+          id: "dbOptions.password",
+          placeholder: "Enter password",
+          connection: "Snowflake",
+          inputType: "password",
+          advanced: true
+        },
+        {
+          type: "input",
+          label: "Warehouse",
+          id: "dbOptions.warehouse",
+          placeholder: "Enter warehouse name",
+          advanced: true
+        },
+        {
+          type: "input",
+          label: "Schema",
+          id: "dbOptions.schema",
+          placeholder: "Enter schema name",
+          advanced: true
+        },
+        {
+          type: "input",
+          label: "Table Name",
+          id: "dbOptions.tableName",
+          placeholder: "Enter table name",
+        },
+        {
+          type: "codeTextarea",
+          label: "SQL Query",
+          height: '50px',
+          mode: "sql",
+          placeholder: 'SELECT * FROM table_name',
+          id: "dbOptions.sqlQuery",
+          tooltip: 'Optional. By default the SQL query is: SELECT * FROM table_name_provided. If specified, the SQL Query is used.',
+          advanced: true
+        }
+      ],
+    };
+
+    super("Snowflake Input", "snowflakeInput", "pandas_df_input", [], "inputs.Databases", snowflakeIcon, defaultConfig, form);
+  }
+
+  public provideDependencies({ config }): string[] {
+    let deps: string[] = [];
+    deps.push('snowflake-sqlalchemy');
+    return deps;
+  }
+
+  public provideImports({ config }): string[] {
+    return ["import pandas as pd", "import sqlalchemy", "import urllib.parse", "from snowflake.sqlalchemy import URL"];
+  }
+
+  public generateComponentCode({ config, outputName }): string {
+    const uniqueEngineName = `${outputName}_Engine`; // Unique engine name based on the outputName
+    const tableReference = (config.dbOptions.schema && config.dbOptions.schema.toLowerCase() !== 'public')
+      ? `${config.dbOptions.schema}.${config.dbOptions.tableName}`
+      : config.dbOptions.tableName;
+
+    const sqlQuery = config.dbOptions.sqlQuery && config.dbOptions.sqlQuery.trim()
+      ? config.dbOptions.sqlQuery
+      : `SELECT * FROM ${tableReference}`;
+
+    const code = `
+# Connect to the Snowflake database
+${uniqueEngineName} = sqlalchemy.create_engine(URL(
+    account = '${config.dbOptions.account}',
+    user = '${config.dbOptions.username}',
+    password = urllib.parse.quote("${config.dbOptions.password}"),
+    database = '${config.dbOptions.database}',
+    schema = '${config.dbOptions.schema}',
+    warehouse = '${config.dbOptions.warehouse}'
+))
+
+# Execute SQL statement
+try:
+    with ${uniqueEngineName}.connect() as conn:
+        ${outputName} = pd.read_sql(
+            """${sqlQuery}""",
+            con=conn.connection
+        ).convert_dtypes()
+finally:
+    ${uniqueEngineName}.dispose()
+`;
+    return code;
+  }
+
+}