Rebase promptflow-evals from main

.cspell.json

@@ -16,23 +16,32 @@
   ],
   "ignorePaths": [
     "**/*.js",
+    "**/*.mjs",
+    "**/*.css",
     "**/*.pyc",
     "**/*.log",
     "**/*.jsonl",
     "**/*.xml",
     "**/*.txt",
     ".gitignore",
     "scripts/docs/_build/**",
-    "src/promptflow/promptflow/azure/_restclient/flow/**",
-    "src/promptflow/promptflow/azure/_restclient/swagger.json",
-    "src/promptflow/promptflow/azure/_models/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/flow/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/swagger.json",
+    "src/promptflow-azure/promptflow/azure/_models/**",
+    "src/promptflow-azure/tests/**",
+    "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
+    "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
+    "src/promptflow-devkit/promptflow/_sdk/_service/static/index.html",
     "**/flow.dag.yaml",
+    "**/pyproject.toml",
     "**/setup.py",
     "scripts/installer/curl_install_pypi/**",
     "scripts/installer/windows/**",
-    "src/promptflow/promptflow/_sdk/_service/pfsvc.py"
+    ".github/workflows/**",
+    ".github/actions/**",
+    ".github/pipelines/**"
   ],
   "words": [
     "aoai",
@@ -84,7 +93,6 @@
     "nunit",
     "astext",
     "Likert",
-    "pfsvc",
     "geval",
     "Summ",
     "Bhavik",
@@ -180,7 +188,8 @@
     "addrs",
     "pywin",
     "STARTF",
-    "mltable"
+    "mltable",
+    "setenv"
   ],
   "flagWords": [
     "Prompt Flow"

.github/CODEOWNERS

@@ -1,18 +1,24 @@
 * @microsoft/prompt-flow-approvers
 
-/src/promptflow/promptflow/_core/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/_internal/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/batch/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/executor/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/integrations/ @microsoft/promptflow-execution
-/src/promptflow/promptflow/storage/ @microsoft/promptflow-execution
+/src/promptflow-tracing/promptflow/tracing/_start_trace.py @microsoft/promptflow-sdk
+
+/src/promptflow-tracing/promptflow/tracing/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/_core/ @microsoft/promptflow-execution
+/src/promptflow-devkit/promptflow/_internal/ @microsoft/promptflow-execution
+/src/promptflow-devkit/promptflow/batch/ @microsoft/promptflow-execution
+/src/promptflow-devkit/promptflow/_proxy/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/executor/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/integrations/ @microsoft/promptflow-execution
+/src/promptflow-core/promptflow/storage/ @microsoft/promptflow-execution
 /src/promptflow/tests/executor/ @microsoft/promptflow-execution
 
-/src/promptflow/promptflow/_cli/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/_sdk/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/azure/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/entities/ @microsoft/promptflow-sdk
-/src/promptflow/promptflow/operations/ @microsoft/promptflow-sdk
+/src/promptflow-core/promptflow/core/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/_cli/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/_sdk/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/_proxy/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/entities/ @microsoft/promptflow-sdk
+/src/promptflow-devkit/promptflow/operations/ @microsoft/promptflow-sdk
+/src/promptflow-azure/promptflow/azure/ @microsoft/promptflow-sdk
 /src/promptflow/tests/sdk_cli_test/ @microsoft/promptflow-sdk
 /src/promptflow/tests/sdk_cli_azure_test/ @microsoft/promptflow-sdk
 /src/promptflow/tests/sdk_cli_global_config_test/ @microsoft/promptflow-sdk

.github/actions/step_create_python_environment/action.yml

@@ -12,9 +12,10 @@ runs:
   using: composite
   steps:
   - name: Set up Python
-    uses: actions/setup-python@v4
+    uses: actions/setup-python@v5
     with:
       python-version: ${{ inputs.pythonVersion }}
+  - uses: snok/install-poetry@v1
   - run: |
       python -m pip install --upgrade pip
       pip install -r ${{ inputs.pipFilePath }}

.github/actions/step_sdk_setup/action.yml

@@ -16,20 +16,6 @@ runs:
     shell: pwsh
     run: |
       pip uninstall -y promptflow promptflow-sdk promptflow-tools
-  - name: 'Build and install: promptflow with extra'
-    if: inputs.setupType == 'promptflow_with_extra'
-    shell: pwsh
-    run: |
-      Set-PSDebug -Trace 1
-      pip install -r ./dev_requirements.txt
-      echo "########### pip list (Before) ###########"
-      pip list
-      python ./setup.py bdist_wheel
-      $package = Get-ChildItem ./dist | ? { $_.Name.Contains('.whl')}
-      pip install $($package.FullName + "[azure,executable,azureml-serving,executor-service]")
-      echo "########### pip freeze (After) ###########"
-      pip freeze
-    working-directory: ${{ inputs.scriptPath }}
   - name: 'Build and install: promptflow-sdk'
     if: inputs.setupType == 'promptflow_dev'
     shell: pwsh
@@ -52,3 +38,17 @@ runs:
       echo "########### pip freeze (After) ###########"
       pip freeze
     working-directory: src/promptflow-tools
+  - name: 'Build and install: promptflow with extra'
+    if: inputs.setupType == 'promptflow_with_extra'
+    shell: pwsh
+    run: |
+      Set-PSDebug -Trace 1
+      pip install -r ./dev_requirements.txt
+      echo "########### pip list (Before) ###########"
+      pip list
+      python ./setup.py bdist_wheel
+      $package = Get-ChildItem ./dist | ? { $_.Name.Contains('.whl')}
+      pip install --force-reinstall $($package.FullName + "[azure,executable,azureml-serving,executor-service]")
+      echo "########### pip freeze (After) ###########"
+      pip freeze
+    working-directory: ${{ inputs.scriptPath }}

.github/labeler.yml

@@ -3,21 +3,23 @@ examples:
 documentation:
 - docs/**
 cli:
-- src/promptflow/promptflow/_cli/**
+- src/promptflow-devkit/promptflow/_cli/**
 sdk:
-- src/promptflow/promptflow/_sdk/**
-- src/promptflow/promptflow/azure/**
+- src/promptflow-devkit/promptflow/_sdk/**
+- src/promptflow-azure/promptflow/azure/**
 promptflow:
-- src/promptflow/**
+- src/promptflow-core/**
+promptflow-tracing:
+- src/promptflow-tracing/**
 promptflow-tools:
 - src/promptflow-tools/**
 executor:
-- src/promptflow/promptflow/batch/**
-- src/promptflow/promptflow/executor/**
-- src/promptflow/promptflow/_core/**
-- src/promptflow/promptflow/_internal/**
-- src/promptflow/promptflow/integrations/**
-- src/promptflow/promptflow/storage/**
+- src/promptflow-devkit/promptflow/batch/**
+- src/promptflow-core/promptflow/executor/**
+- src/promptflow-core/promptflow/_core/**
+- src/promptflow-devkit/promptflow/_internal/**
+- src/promptflow-core/promptflow/integrations/**
+- src/promptflow-core/promptflow/storage/**
 - src/promptflow/tests/executor/**
 fundamental:
 - scripts/**

.github/workflows/build_doc_ci.yml

@@ -5,13 +5,15 @@ on:
   pull_request:
     branches:
       - main
-      - preview/docs
     paths:
       - 'README.md'
       - 'docs/**'
       - 'scripts/docs/**'
       - '.github/workflows/build_doc_ci.yml'
-      - 'src/promptflow/promptflow/**'
+      - 'src/promptflow-tracing/promptflow/**'
+      - 'src/promptflow-core/promptflow/**'
+      - 'src/promptflow-devkit/promptflow/**'
+      - 'src/promptflow-azure/promptflow/**'
 
 env:
   packageSetupType: promptflow_with_extra
@@ -30,11 +32,16 @@ jobs:
       - name: Python Setup
         uses: "./.github/actions/step_create_python_environment"
 
-      - name: Dev setup
-        uses: "./.github/actions/step_sdk_setup"
-        with:
-          setupType: ${{ env.packageSetupType }}
-          scriptPath: ${{ env.testWorkingDirectory }}
+      - name: Install packages
+        shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
+        run: |
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip freeze
 
       - name: Build doc with reference doc
         shell: powershell
@@ -53,7 +60,23 @@ jobs:
         with:
           submodules: true
 
+      - name: Python Setup
+        uses: "./.github/actions/step_create_python_environment"
+
+      - name: Install packages
+        shell: pwsh
+        # Note: Use -e to avoid duplicate object warning when build apidoc.
+        run: |
+          pip uninstall -y promptflow-tracing promptflow-core promptflow-devkit promptflow-azure
+          pip install -e ${{ github.workspace }}/src/promptflow-tracing
+          pip install -e ${{ github.workspace }}/src/promptflow-core
+          pip install -e ${{ github.workspace }}/src/promptflow-devkit
+          pip install -e ${{ github.workspace }}/src/promptflow-azure
+          pip freeze
+
       - name: Build LinkCheck
         shell: powershell
         working-directory: scripts/docs/
-        run: ./doc_generation.ps1 -BuildLinkCheck -WarningAsError:$true
+        run: |-
+          pip install langchain
+          ./doc_generation.ps1 -WithReferenceDoc:$true -WarningAsError:$true -BuildLinkCheck

.github/workflows/build_msi_installer.yml

@@ -1,4 +1,4 @@
-name: Build and Package MSI
+name: Build and Package MSI & Portable Installer
 
 on:
   workflow_dispatch:
@@ -74,7 +74,7 @@ jobs:
             if ($version -ne $run_version) {
               throw "Version input does not match the version in promptflow package. Version input: $version, version in promptflow package: $run_version"
             }
-          } 
+          }
           elseif ($env:MSI_PRIVATE_VERSION) {
             $version=$env:MSI_PRIVATE_VERSION
           }
@@ -108,6 +108,24 @@ jobs:
           setupType: promptflow_with_extra
           scriptPath: ${{ env.testWorkingDirectory }}
 
+      - name: Setup and Install dev promptflow
+        shell: pwsh
+        run: |
+          Set-PSDebug -Trace 1
+          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core[executor-service,azureml-serving]
+          pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow,executable]
+          pip install ${{ github.workspace }}/src/promptflow-azure
+          echo "Should fix this after pf-core could install this dependency"
+          pip install azureml-ai-monitoring
+          pip freeze
+
+      - name: Generate promptflow spec file to config pyinstaller
+        working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
+        run: |
+          python generate_dependency.py
+          Get-Content promptflow.spec
 
       - name: Build Pyinstaller project
         working-directory: ${{ github.workspace }}/scripts/installer/windows/scripts
@@ -132,6 +150,11 @@ jobs:
           pyinstaller promptflow.spec
         shell: pwsh
 
+      - name: Generate portable promptflow
+        run: |
+          Compress-Archive -Path ${{ github.workspace }}/scripts/installer/windows/scripts/dist/promptflow -DestinationPath promptflow-${{ steps.get-version.outputs.version }}.zip
+        shell: pwsh
+
       - name: Build WIX project
         working-directory: ${{ github.workspace }}/scripts/installer/windows
         run: |
@@ -167,7 +190,7 @@ jobs:
               "promptflow" = $version
             } | ConvertTo-Json -Depth 100
             $jsonContent | Out-File -FilePath latest_version.json -Encoding UTF8
-            
+
             Write-Output "Created latest_version.json with version: $version"
             az storage blob upload --account-name promptflowartifact --container-name msi-installer --file "latest_version.json" --name "latest_version.json" --overwrite
           } else {
@@ -185,6 +208,13 @@ jobs:
             az storage blob upload --account-name promptflowartifact --container-name msi-installer --file "scripts/installer/windows/out/$($msi_file.Name)" --name "$($msi_file.Name)" --overwrite
           }
           }
+          # Upload zip file
+          if ($env:INPUT_UPLOADASLATEST -ieq 'True') {
+            az storage blob upload --account-name promptflowartifact --container-name portable-installer --file "promptflow-${{ steps.get-version.outputs.version }}.zip" --name "promptflow.zip" --overwrite
+            az storage blob copy start --account-name promptflowartifact --destination-container portable-installer --destination-blob "promptflow-${{ steps.get-version.outputs.version }}.zip" --source-container portable-installer --source-blob "promptflow.zip"
+          } else {
+            az storage blob upload --account-name promptflowartifact --container-name portable-installer --file "promptflow-${{ steps.get-version.outputs.version }}.zip" --name "promptflow-${{ steps.get-version.outputs.version }}.zip" --overwrite
+          }
         env:
           INPUT_UPLOADASLATEST: ${{ github.event.inputs.uploadAsLatest }}
         shell: pwsh

.github/workflows/flowdag_schema_check.yml

@@ -30,6 +30,6 @@ jobs:
           PYTHONPATH: ${{ github.workspace }}/src/promptflow
         run: |
           cd ${{ github.workspace }}/src
-          pip install -e promptflow[azure]
-          pip install -e promptflow-tools
+          pip install ./promptflow[azure]
+          pip install ./promptflow-tools
           python ${{ github.workspace }}/scripts/readme/schema_checker.py