Various improvements (#103) and bug fixes (#106, #136)

amphi-etl/amphi/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.5.94'
+__version__ = VERSION = '0.5.95'

amphi-etl/lerna.json

@@ -1,4 +1,4 @@
 {
   "npmClient": "yarn",
-  "version": "0.5.94"
+  "version": "0.5.95"
 }

amphi-etl/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@amphi/amphi-etl",
-    "version": "0.5.94",
+    "version": "0.5.95",
     "keywords": [
         "amphi",
         "etl",

amphi-etl/requirements.txt

@@ -1,4 +1,4 @@
 jupyterlab==4.2.5
-# jupyterlab-amphi==0.5.92
-../jupyterlab-amphi
+jupyterlab-amphi==0.5.95
+# ../jupyterlab-amphi
 .

amphi-etl/setup.py

@@ -23,14 +23,14 @@ def collect_files(src_dir, dest_dir):
 
 setup(
     name='amphi-etl',
-    version='0.5.94',
+    version='0.5.95',
     description='Open-source and Python-based ETL',
     author='Thibaut Gourdel',
     author_email='tgourdel@amphi.ai',
     license='ELv2',
     install_requires=[
         'jupyterlab==4.2.5',
-        'jupyterlab-amphi==0.5.92'
+        'jupyterlab-amphi==0.5.95'
     ],
     keywords=[],  # Added an empty list for keywords to resolve the dynamic 'keywords' issue.
     packages=find_packages(include=['amphi', 'amphi.theme-light', 'amphi.ui-component', 'config', 'packages']),  # Custom package discovery.

jupyterlab-amphi/_version.py

@@ -1,2 +1,2 @@
 """source of truth for ``amphi``` version."""
-__version__ = "0.5.94"
+__version__ = "0.5.95"

jupyterlab-amphi/lerna.json

@@ -1,4 +1,4 @@
 {
   "npmClient": "yarn",
-  "version": "0.5.94"
+  "version": "0.5.95"
 }

jupyterlab-amphi/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@amphi/jupyterlab-amphi",
-    "version": "0.5.94",
+    "version": "0.5.95",
     "keywords": [
         "amphi",
         "etl",

jupyterlab-amphi/packages/pipeline-components-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@amphi/pipeline-components-core",
-  "version": "0.5.94",
+  "version": "0.5.95",
   "description": "Amphi Pipeline Core Components",
   "homepage": "https://github.com/amphi-ai/jupyterlab-amphi",
   "bugs": {

jupyterlab-amphi/packages/pipeline-components-core/src/components/annotations/Annotation.tsx

@@ -12,19 +12,16 @@ import { annotationIcon } from '../../icons';
 import { bracesIcon, settingsIcon } from '../../icons';
 import { generate, green, presetPalettes, red } from '@ant-design/colors';
 
-
-
 type Color = Extract<GetProp<ColorPickerProps, 'value'>, string | { cleared: any }>;
 type Format = GetProp<ColorPickerProps, 'format'>;
 
-
-
 export type AnnotationData = {
   content: string;
   backgroundColor?: string;
+  borderColor?: string;
+  borderWidth?: number;
 };
 
-
 export class Annotation extends PipelineComponent<ComponentItem>() {
 
   public _name = "Annotation";
@@ -50,7 +47,7 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
   }) => {
 
     const handleColorChange = useCallback((colorObj, setColor, field) => {
-      const colorValue = colorObj.toRgbString(); // Use the color object's built-in method
+      const colorValue = colorObj.toRgbString();
       setColor(colorValue);
       handleChange(colorValue, field);
     }, [handleChange]);
@@ -60,13 +57,18 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
       handleChange(newValue, 'borderRadius');
     }, [handleChange]);
 
+    const handleBorderWidthChange = useCallback((newValue: number) => {
+      setBorderWidth(newValue);
+      handleChange(newValue, 'borderWidth');
+    }, [handleChange]);
 
     const [content, setContent] = useState<string>(data.content || '# Annotation');
     const [backgroundColor, setBackgroundColor] = useState<Color>(data.backgroundColor || '#fff');
+    const [borderColor, setBorderColor] = useState<Color>(data.borderColor || '#42766D');
+    const [borderWidth, setBorderWidth] = useState<number>(data.borderWidth || 5);
     const [textColor, setTextColor] = useState<Color>(data.textColor || '#000');
     const [borderRadius, setBorderRadius] = useState<number>(data.borderRadius || 0);
 
-
     useEffect(() => {
       setContent(data.content || '# Annotation');
     }, [data.content]);
@@ -80,7 +82,6 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
         <ConfigProvider
           theme={{
             token: {
-              // Seed Token
               colorPrimary: '#5F9B97',
             },
           }}
@@ -98,28 +99,57 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
               </>
             )}
           >
-            <Form
-              layout="vertical" >
+            <Form layout="vertical">
               <Form.Item label="Background Color">
                 <ColorPicker
                   allowClear
                   placement={"topRight"}
                   defaultFormat={"hex"}
                   format={"hex"}
                   showText
-                  value={backgroundColor} // default or existing color
-                  onChange={(color) => handleColorChange(color, setBackgroundColor, 'backgroundColor')} // Handle color change
-
+                  value={backgroundColor}
+                  onChange={(color) => handleColorChange(color, setBackgroundColor, 'backgroundColor')}
                 />
               </Form.Item>
+              <Form.Item label="Border Color">
+                <ColorPicker
+                  placement={"topRight"}
+                  defaultFormat={"hex"}
+                  format={"hex"}
+                  showText
+                  value={borderColor}
+                  onChange={(color) => handleColorChange(color, setBorderColor, 'borderColor')}
+                />
+              </Form.Item>
+              <Form.Item label="Border Width">
+                <Row>
+                  <Col span={12}>
+                    <Slider
+                      min={0}
+                      max={20}
+                      onChange={handleBorderWidthChange}
+                      value={typeof borderWidth === 'number' ? borderWidth : 5}
+                    />
+                  </Col>
+                  <Col span={4}>
+                    <InputNumber
+                      min={0}
+                      max={20}
+                      style={{ margin: '0 16px' }}
+                      value={borderWidth}
+                      onChange={handleBorderWidthChange}
+                    />
+                  </Col>
+                </Row>
+              </Form.Item>
               <Form.Item label="Text Color">
                 <ColorPicker
                   placement={"topRight"}
                   defaultFormat={"hex"}
                   format={"hex"}
                   showText
-                  value={textColor} // default or existing color
-                  onChange={(color) => handleColorChange(color, setTextColor, 'textColor')} // Handle color change
+                  value={textColor}
+                  onChange={(color) => handleColorChange(color, setTextColor, 'textColor')}
                 />
               </Form.Item>
               <Form.Item label="Border Radius">
@@ -159,43 +189,37 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
   }
 
   public UIComponent({ id, data, context, componentService, manager, commands, settings }) {
-    const { setNodes, deleteElements, setViewport } = useReactFlow();
+    const { setNodes } = useReactFlow();
     const store = useStoreApi();
-    // Selector to determine if the node is selected
     const isSelected = useStore((state) => !!state.nodeInternals.get(id)?.selected);
 
-
-    const selector = (s) => ({
-      nodeInternals: s.nodeInternals,
-      edges: s.edges,
-    });
-
-    const { nodeInternals, edges } = useStore(selector);
-    const nodeId = id;
-
     const handleChange = useCallback((evtTargetValue: any, field: string) => {
-      onChange({ evtTargetValue, field, nodeId, store, setNodes });
-    }, [nodeId, store, setNodes]);
+      onChange({ evtTargetValue, field, nodeId: id, store, setNodes });
+    }, [id, store, setNodes]);
 
     const shiftKeyPressed = useKeyPress('Shift');
 
     const [modalOpen, setModalOpen] = useState(false);
 
     const backgroundColorStyle = data.backgroundColor || 'transparent';
+    const borderColorStyle = data.borderColor || '#42766D';
+    const borderWidthStyle = data.borderWidth || 2;
     const textColorStyle = data.textColor || 'rgba(0, 0, 0, 1)';
-    const borderRadiusStyle = data.borderRadius || 0; // Default to 0 if not provided
-
+    const borderRadiusStyle = data.borderRadius || 0;
 
     return (
       <>
-        <div style={{ 
-          height: '100%', 
-          backgroundColor: backgroundColorStyle, 
+        <div style={{
+          height: '100%',
+          backgroundColor: backgroundColorStyle,
           paddingLeft: '40px',
-          paddingRight: '40px', 
-          position: 'relative', 
+          paddingRight: '40px',
+          paddingTop: '20px',
+          paddingBottom: '20px',
+          position: 'relative',
           zIndex: 0,
-          borderRadius: `${borderRadiusStyle}px`, // Apply the border radius
+          borderRadius: `${borderRadiusStyle}px`,
+          border: `${borderWidthStyle}px solid ${borderColorStyle}`, // Apply the border style
         }}>
           <NodeResizer
             keepAspectRatio={shiftKeyPressed}
@@ -205,27 +229,13 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
             minHeight={50}
           />
           <div style={{ color: textColorStyle }}>
-
-            <Remark
-              rehypePlugins={[
-                function noRefCheck() { },
-                function noRefCheck() { }
-              ]}
-              remarkToRehypeOptions={{
-                allowDangerousHtml: true
-              }}
-            >
-              {data.content}
-            </Remark>
+            <Remark>{data.content}</Remark>
           </div>
-
-        </div>
+
         <NodeToolbar isVisible={isSelected} position={Position.Bottom}>
           <button onClick={() => setModalOpen(true)}><settingsIcon.react /></button>
-        </NodeToolbar>
+        </NodeToolbar></div>
 
-
-        {/* Render ConfigForm */}
         <Annotation.ConfigForm
           nodeId={id}
           data={data}
@@ -239,7 +249,6 @@ export class Annotation extends PipelineComponent<ComponentItem>() {
           modalOpen={modalOpen}
           setModalOpen={setModalOpen}
         />
-
       </>
     );
   }

jupyterlab-amphi/packages/pipeline-components-core/src/components/common/GCPOptionsHandler.ts

@@ -0,0 +1,43 @@
+// GCSOptionsHandler.ts
+
+export class GCSOptionsHandler {
+    // Static method to handle GCS-specific options
+    public static handleGCSOptions(config, storageOptions): object {
+        if (config.fileLocation === 'gcs' && config.connectionMethod === 'storage_options') {
+            return {
+                ...storageOptions, // Preserve any manually added storageOptions
+                service_account_file: config.gcsServiceAccountFilePath
+            };
+        }
+        return storageOptions;
+    }
+
+    public static getGCSFields(): object[] {
+        return [
+            {
+                type: "select",
+                label: "Connection Method",
+                id: "connectionMethod",
+                options: [
+                    { value: "env", label: "Default", tooltip: "This uses the default credentials set in the environment (like Application Default Credentials or gcloud config). If no credentials are found, " },
+                    { value: "storage_options", label: "Service Account (storage_options)", tooltip: "Provide a service account directly via the storage_options parameter." }
+                ],
+                condition: { fileLocation: "gcs" },
+                connection: "GCS",
+                ignoreConnection: true,
+                advanced: true
+            },
+            {
+                type: "file",
+                label: "Service Account Key",
+                id: "gcsServiceAccountFilePath",
+                placeholder: "Type file name",
+                validation: "\\.(json)$",
+                validationMessage: "This field expects a file with a .json extension such as your-service-account-file.json.",
+                connection: "GCS",
+                condition: { fileLocation: "gcs", connectionMethod: "storage_options" },
+                advanced: true
+            },
+        ];
+    }
+}

jupyterlab-amphi/packages/pipeline-components-core/src/components/index.ts

@@ -10,6 +10,7 @@ export { PdfTablesInput } from './inputs/files/PdfTablesInput';
 export { HtmlFileInput } from './inputs/files/HtmlFileInput';
 export { PdfFileInput } from './inputs/files/PdfFileInput';
 export { S3FileInput } from './inputs/files/S3FileInput';
+export { LocalFileInput } from './inputs/files/LocalFileInput';
 
 export { RestInput } from './inputs/cloud/RestInput';
 export { GoogleSheetsInput } from './inputs/cloud/GoogleSheetsInput';
@@ -18,6 +19,8 @@ export { PostgresInput } from './inputs/databases/PostgresInput';
 export { OracleInput } from './inputs/databases/OracleInput';
 export { SqlServerInput } from './inputs/databases/SqlServerInput';
 export { SnowflakeInput } from './inputs/databases/SnowflakeInput';
+export { BigQueryInput } from './inputs/databases/BigQueryInput';
+
 export { RedditInput } from './inputs/cloud/RedditInput';
 
 // Transforms

jupyterlab-amphi/packages/pipeline-components-core/src/components/inputs/cloud/GoogleSheetsInput.tsx

@@ -41,6 +41,12 @@ export class GoogleSheetsInput extends BaseCoreComponent {
     super("G. Sheets Input", "googleSheetsInput", description, "pandas_df_input", [], "inputs", googleSheetsIcon, defaultConfig, form);
   }
 
+  public provideDependencies({ config }): string[] {
+    let deps: string[] = [];
+    deps.push('gspread');
+    return deps;
+  }
+
   public provideImports({ config }): string[] {
     return ["import pandas as pd", "import gspread", "from oauth2client.service_account import ServiceAccountCredentials"];
   }