Fix r-string issues with variables + sort improvement (#84) + ODBC In…

…put (#14)

amphi-etl/amphi/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.5.95'
+__version__ = VERSION = '0.5.97'

amphi-etl/lerna.json

@@ -1,4 +1,4 @@
 {
   "npmClient": "yarn",
-  "version": "0.5.95"
+  "version": "0.5.97"
 }

amphi-etl/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@amphi/amphi-etl",
-    "version": "0.5.95",
+    "version": "0.5.97",
     "keywords": [
         "amphi",
         "etl",

amphi-etl/requirements.txt

@@ -1,4 +1,5 @@
 jupyterlab==4.2.5
-jupyterlab-amphi==0.5.95
+jupyterlab-amphi==0.5.97
+pandas==2.2.1
 # ../jupyterlab-amphi
 .

amphi-etl/setup.py

@@ -23,14 +23,14 @@ def collect_files(src_dir, dest_dir):
 
 setup(
     name='amphi-etl',
-    version='0.5.95',
+    version='0.5.97',
     description='Open-source and Python-based ETL',
     author='Thibaut Gourdel',
     author_email='tgourdel@amphi.ai',
     license='ELv2',
     install_requires=[
         'jupyterlab==4.2.5',
-        'jupyterlab-amphi==0.5.95'
+        'jupyterlab-amphi==0.5.97'
     ],
     keywords=[],  # Added an empty list for keywords to resolve the dynamic 'keywords' issue.
     packages=find_packages(include=['amphi', 'amphi.theme-light', 'amphi.ui-component', 'config', 'packages']),  # Custom package discovery.

jupyterlab-amphi/_version.py

@@ -1,2 +1,2 @@
 """source of truth for ``amphi``` version."""
-__version__ = "0.5.95"
+__version__ = "0.5.97"

jupyterlab-amphi/lerna.json

@@ -1,4 +1,4 @@
 {
   "npmClient": "yarn",
-  "version": "0.5.95"
+  "version": "0.5.97"
 }

jupyterlab-amphi/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@amphi/jupyterlab-amphi",
-    "version": "0.5.95",
+    "version": "0.5.97",
     "keywords": [
         "amphi",
         "etl",

jupyterlab-amphi/packages/pipeline-components-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-components-core",
-  "version": "0.5.95",
+  "version": "0.5.97",
   "description": "Amphi Pipeline Core Components",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {

jupyterlab-amphi/packages/pipeline-components-core/src/components/index.ts

@@ -20,6 +20,8 @@ export { OracleInput } from './inputs/databases/OracleInput';
 export { SqlServerInput } from './inputs/databases/SqlServerInput';
 export { SnowflakeInput } from './inputs/databases/SnowflakeInput';
 export { BigQueryInput } from './inputs/databases/BigQueryInput';
+export { ODBCInput } from './inputs/databases/ODBCInput';
+
 
 export { RedditInput } from './inputs/cloud/RedditInput';
 

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/databases/ODBCInput.tsx

@@ -0,0 +1,99 @@
+import { databaseIcon } from '../../../icons';
+import { BaseCoreComponent } from '../../BaseCoreComponent';
+
+export class ODBCInput extends BaseCoreComponent {
+    constructor() {
+        const defaultConfig = {
+            connectionString: "",
+            queryMethod: "table",
+            tableName: "",
+            sqlQuery: "",
+            autoCommit: true
+        };
+        const form = {
+            fields: [
+                {
+                    type: "input",
+                    label: "Connection String",
+                    id: "connectionString",
+                    placeholder: "Enter ODBC connection string",
+                    tooltip: "Provide the full ODBC connection string for your database. Reference: https://github.com/mkleehammer/pyodbc/wiki/Connecting-to-databases",
+                    connection: "ODBC",
+                    advanced: true
+                },
+                {
+                    type: "boolean",
+                    label: "Auto Commit",
+                    tooltip: "Setting autocommit True will cause the database to issue a commit after each SQL statement, otherwise database transactions will have to be explicity committed. As per the Python DB API, the default value is False (even though the ODBC default value is True). Typically, you will probably want to set autocommit True when creating a connection.",
+                    id: "autoCommit",
+                    advanced: true
+                },
+                {
+                    type: "radio",
+                    label: "Query Method",
+                    id: "queryMethod",
+                    tooltip: "Select whether you want to specify the table name to retrieve data or use a custom SQL query for greater flexibility.",
+                    options: [
+                        { value: "table", label: "Table Name" },
+                        { value: "query", label: "SQL Query" }
+                    ],
+                    advanced: true
+                },
+                {
+                    type: "input",
+                    label: "Table Name",
+                    id: "tableName",
+                    placeholder: "Enter table name",
+                    condition: { queryMethod: "table" }
+                },
+                {
+                    type: "codeTextarea",
+                    label: "SQL Query",
+                    height: '50px',
+                    mode: "sql",
+                    placeholder: 'SELECT * FROM table_name',
+                    id: "sqlQuery",
+                    tooltip: 'Optional. By default the SQL query is: SELECT * FROM table_name_provided. If specified, the SQL Query is used.',
+                    condition: { queryMethod: "query" },
+                    advanced: true
+                }
+            ],
+        };
+        const description = "Use ODBC Input to retrieve data from various databases using an ODBC connection string, along with either a table name or a custom SQL query."
+
+        super("ODBC Input", "odbcInput", description, "pandas_df_input", [], "inputs.Databases", databaseIcon, defaultConfig, form);
+    }
+
+    public provideDependencies({ config }): string[] {
+        return ['pyodbc'];
+    }
+
+    public provideImports({ config }): string[] {
+        return ["import pandas as pd", "import pyodbc"];
+    }
+
+    public generateComponentCode({ config, outputName }): string {
+        // Escape single quotes in the connection string
+        const connectionString = config.connectionString.replace(/'/g, "\\'");
+        const autoCommit = config.autoCommit;
+
+        const sqlQuery = config.queryMethod === 'query' && config.sqlQuery && config.sqlQuery.trim()
+            ? config.sqlQuery
+            : `SELECT * FROM ${config.tableName}`;
+
+        const code = `
+# Connect to the database using ODBC
+conn = pyodbc.connect(f"""${connectionString}""", autocommit=${autoCommit ? 'True' : 'False'})
+
+# Execute SQL statement
+try:
+    ${outputName} = pd.read_sql(
+        """${sqlQuery}""",
+        conn
+    ).convert_dtypes()
+finally:
+    conn.close()
+`;
+        return code;
+    }
+}

jupyterlab-amphi/packages/pipeline-components-core/src/components/transforms/Sort.tsx

@@ -1,4 +1,3 @@
-
 import { sortIcon } from '../../icons';
 import { BaseCoreComponent } from '../BaseCoreComponent';
 
@@ -9,15 +8,9 @@ export class Sort extends BaseCoreComponent {
       idPrefix: "component__form",
       fields: [
         {
-          type: "columns",
-          label: "Columns",
-          id: "by",
-          placeholder: "Select columns",
-        },
-        {
-          type: "radio",
-          label: "Order",
-          id: "order",
+          type: "keyvalueColumnsRadio",
+          label: "Columns Sorting Order",
+          id: "columnAndOrder",
           options: [
             { value: "True", label: "Asc." },
             { value: "False", label: "Desc." }
@@ -42,13 +35,13 @@ export class Sort extends BaseCoreComponent {
 
   public generateComponentCode({ config, inputName, outputName }): string {
 
-    const byColumns = `by=[${config.by.map(column => column.named ? `"${column.value}"` : column.value).join(", ")}]`;
-    const ascending = typeof config.order !== "undefined" ? `, ascending=${config.order}` : "";
+    const byColumns = `by=[${config.columnAndOrder.map(item => item.key.named ? `"${item.key.value}"` : item.key.value).join(", ")}]`;
+    const ascending = `ascending=[${config.columnAndOrder.map(item => item.value === "True" ? "True" : "False").join(", ")}]`;
     const ignoreIndex = config.ignoreIndex ? `, ignore_index=${config.ignoreIndex}` : "";
-
+    
     const code = `
 # Sort rows 
-${outputName} = ${inputName}.sort_values(${byColumns}${ascending}${ignoreIndex})
+${outputName} = ${inputName}.sort_values(${byColumns}, ${ascending}${ignoreIndex})
 `;
     return code;
   }

jupyterlab-amphi/packages/pipeline-components-core/src/icons.ts

@@ -47,6 +47,8 @@ import sumIconSvgStr from '../style/icons/sum.svg'
 import engineIconSvgStr from '../style/icons/service-16.svg'
 import s3IconSvgStr from '../style/icons/s3.svg'
 import bigQueryIconSvgStr from '../style/icons/bigquery.svg'
+import databaseIconSvgStr from '../style/icons/database-24.svg'
+
 
 export const fileTextIcon = new LabIcon({
   name: 'amphi:file-text-icon',
@@ -281,4 +283,9 @@ export const s3Icon = new LabIcon({
 export const bigQueryIcon = new LabIcon({
   name: 'amphi:bigQuery-icon',
   svgstr: bigQueryIconSvgStr
+});
+
+export const databaseIcon = new LabIcon({
+  name: 'amphi:database-icon',
+  svgstr: databaseIconSvgStr
 });

jupyterlab-amphi/packages/pipeline-components-core/src/index.ts

@@ -8,7 +8,7 @@ import {
   Aggregate, Console, ExcelFileOutput, CsvFileInput, JsonFileInput, JsonFileOutput, ExcelFileInput, CsvFileOutput, CustomTransformations, Filter, RestInput,
   SplitColumn, Deduplicate, ExpandList, Sample, Sort, RenameColumns, TypeConverter, Extract, GoogleSheetsInput, GoogleSheetsOutput, FilterColumns, Join,
   ParquetFileInput, ParquetFileOutput, PostgresInput, PostgresOutput, MySQLInput, MySQLOutput, XmlFileInput, XmlFileOutput, SQLQuery, OpenAILookUp,
-  EnvVariables, EnvFile, Transpose, Unite, Pivot, Annotation, BigQueryInput, 
+  EnvVariables, EnvFile, Transpose, Unite, Pivot, Annotation, BigQueryInput, ODBCInput, PdfTablesInput,
   DataCleansing, GenerateIDColumn, SqlServerInput, OracleInput, Connection, SnowflakeInput, FormulaRow, InlineInput, S3FileOutput, S3FileInput
 } from './components';
 
@@ -33,6 +33,7 @@ const plugin: JupyterFrontEndPlugin<void> = {
     componentService.addComponent(ParquetFileInput.getInstance())
     componentService.addComponent(JsonFileInput.getInstance())
     componentService.addComponent(XmlFileInput.getInstance())
+    componentService.addComponent(PdfTablesInput.getInstance())
     componentService.addComponent(S3FileInput.getInstance())
 
     componentService.addComponent(RestInput.getInstance())
@@ -42,6 +43,8 @@ const plugin: JupyterFrontEndPlugin<void> = {
     componentService.addComponent(OracleInput.getInstance())
     componentService.addComponent(SqlServerInput.getInstance())
     componentService.addComponent(SnowflakeInput.getInstance())
+    componentService.addComponent(ODBCInput.getInstance())
+
     // componentService.addComponent(BigQueryInput.getInstance())
 
     // Processors

jupyterlab-amphi/packages/pipeline-components-manager/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-components-manager",
-  "version": "0.5.95",
+  "version": "0.5.97",
   "description": "Amphi Pipeline Components Manager",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {

jupyterlab-amphi/packages/pipeline-components-manager/src/CodeGenerator.tsx

@@ -412,8 +412,8 @@ ${code}`;
   static formatVariables(code: string): string {
     const lines = code.split('\n');
     const transformedLines = lines.map(line => {
-      if (/f(['"])/.test(line) || /f("""|''')/.test(line) || /r(['"])/.test(line)) {
-        return line; // Return the line as-is if it's already an f-string
+      if (/r(['"]).*\1/.test(line) || /f(['"])/.test(line) || /f("""|''')/.test(line)) {
+        return line; // Return the line as-is if it's an actual raw string or an f-string
       }
       return line
         .replace(/(['"])\{(\w+)\}\1/g, '$2')  // Remove quotes for standalone variables

jupyterlab-amphi/packages/pipeline-components-manager/src/configUtils.tsx

@@ -10,6 +10,7 @@ import TextareaRegular from './forms/TextareaRegular';
 import DataMapping from './forms/dataMapping';
 import KeyValueColumns from './forms/keyValueColumns';
 import KeyValueColumnsSelect from './forms/keyValueColumnsSelect';
+import KeyValueColumnsRadio from './forms/keyValueColumnsRadio';
 import KeyValueForm from './forms/keyValueForm';
 import SelectColumn from './forms/selectColumn';
 import SelectColumns from './forms/selectColumns';
@@ -370,6 +371,8 @@ export const GenerateUIInputs = React.memo(({
         return renderFormItem(field, <KeyValueColumns {...commonProps} initialValues={values} componentService={componentService} commands={commands} nodeId={nodeId} />);
       case "keyvalueColumnsSelect":
         return renderFormItem(field, <KeyValueColumnsSelect {...commonProps} initialValues={values} componentService={componentService} commands={commands} nodeId={nodeId} />);
+      case "keyvalueColumnsRadio":
+        return renderFormItem(field, <KeyValueColumnsRadio {...commonProps} initialValues={values} componentService={componentService} commands={commands} nodeId={nodeId} />);
       case "valuesList":
         return renderFormItem(field, <ValuesListForm {...commonProps} initialValues={values} />);
       case "inputNumber":
@@ -590,7 +593,7 @@ export interface Option {
 export interface FieldDescriptor {
   type: 'file' | 'column' | 'columns' | 'keyvalue' | 'valuesList' | 'input' | 'password' | 'select' | 'textarea' | 'codeTextarea' | 'radio'
   | 'cascader' | 'boolean' | 'inputNumber' | 'selectCustomizable' | 'selectTokenization' | 'transferData' | 'keyvalueColumns' | 'keyvalueColumnsSelect'
-  | 'dataMapping' | 'editableTable' | 'info' | 'cascaderMultiple' | 'selectMultipleCustomizable' | 'formulaColumns';
+  | 'dataMapping' | 'editableTable' | 'info' | 'cascaderMultiple' | 'selectMultipleCustomizable' | 'formulaColumns' | 'keyvalueColumnsRadio';
   label: string;
   id: string;
   placeholder?: any;