{CI} Enable CodegenCoverage and drop kusto (Azure#30375)

* drop kusto * Update azure-pipelines.yml * Update azure-pipelines.yml * Update azure-pipelines.yml
xaliciayang · Nov 21, 2024 · 7b805c8 · 7b805c8
1 parent 14084ad
commit 7b805c8
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 58 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -1150,15 +1150,14 @@ jobs:
 
 
 - job: CodegenCoverage
-  condition: and(succeededOrFailed(), in(variables['Build.Reason'], 'BatchedCI'))
   displayName: "Codegen Coverage"
   pool:
     name: ${{ variables.ubuntu_pool }}
   steps:
   - task: UsePythonVersion@0
-    displayName: 'Use Python 3.12'
+    displayName: 'Use Python 3.11'
     inputs:
-      versionSpec: 3.12
+      versionSpec: 3.11
   - template: .azure-pipelines/templates/azdev_setup.yml
   - bash: |
       set -ev
@@ -1167,24 +1166,16 @@ jobs:
       cd ..
       git clone --depth 1 -b main https://github.com/Azure/azure-cli-extensions.git ./azure-cli-extensions
       azdev extension repo add ./azure-cli-extensions
+      pip install setuptools==70.0.0 wheel==0.30.0
       azdev extension add "*"
       # Some extension will change the dependence, so run `azdev setup` again after all extensions installed.
       azdev setup -c $CLI_REPO_PATH -r ./azure-cli-extensions
       # CLI repo only
       azdev statistics list-command-table CLI --statistics-only
       # CLI + EXT repos
       azdev statistics list-command-table --statistics-only > /tmp/codegen_report.json
-      # azure-kusto is not compatible with azure-cli
-      # so we need run azdev statistics before installing azure-kusto to avoid the error `No module named azure.cli`
-      pip install azure-kusto-data azure-kusto-ingest
       python s/scripts/ci/codegen_report.py
     env:
       BUILD_ID: $(Build.BuildId)
       BUILD_BRANCH: $(Build.SourceBranchName)
-      KUSTO_CLIENT_ID: $(KUSTO_CLIENT_ID)
-      KUSTO_CLIENT_SECRET: $(KUSTO_CLIENT_SECRET)
-      KUSTO_CLUSTER: $(KUSTO_CLUSTER)
-      KUSTO_DATABASE: $(KUSTO_DATABASE)
-      KUSTO_TABLE: $(KUSTO_TABLE)
-      KUSTO_TENANT_ID: $(KUSTO_TENANT_ID)
-    enabled: false
+    enabled: true
diff --git a/scripts/ci/codegen_report.py b/scripts/ci/codegen_report.py
@@ -5,20 +5,10 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-from azure.kusto.data import KustoConnectionStringBuilder
-from azure.kusto.data.data_format import DataFormat
-from azure.kusto.ingest import (
-    IngestionProperties,
-    QueuedIngestClient,
-    ReportLevel,
-)
-import csv
 import datetime
 import json
 import logging
 import os
-import subprocess
-import sys
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -28,14 +18,6 @@
 
 BUILD_ID = os.environ.get('BUILD_ID', None)
 BUILD_BRANCH = os.environ.get('BUILD_BRANCH', None)
-# authenticate with AAD application.
-KUSTO_CLIENT_ID = os.environ.get('KUSTO_CLIENT_ID')
-KUSTO_CLIENT_SECRET = os.environ.get('KUSTO_CLIENT_SECRET')
-KUSTO_CLUSTER = os.environ.get('KUSTO_CLUSTER')
-KUSTO_DATABASE = os.environ.get('KUSTO_DATABASE')
-KUSTO_TABLE = os.environ.get('KUSTO_TABLE')
-# get tenant id from https://docs.microsoft.com/en-us/onedrive/find-your-office-365-tenant-id
-KUSTO_TENANT_ID = os.environ.get('KUSTO_TENANT_ID')
 
 
 def generate_csv():
@@ -53,31 +35,5 @@ def generate_csv():
     return data
 
 
-def send_to_kusto(data):
-    logger.info('Start send codegen report csv data to kusto db')
-
-    with open(f'/tmp/codegen_report.csv', mode='w', newline='') as file:
-        writer = csv.writer(file)
-        writer.writerows(data)
-
-    kcsb = KustoConnectionStringBuilder.with_aad_application_key_authentication(KUSTO_CLUSTER, KUSTO_CLIENT_ID, KUSTO_CLIENT_SECRET, KUSTO_TENANT_ID)
-    # The authentication method will be taken from the chosen KustoConnectionStringBuilder.
-    client = QueuedIngestClient(kcsb)
-
-    # there are a lot of useful properties, make sure to go over docs and check them out
-    ingestion_props = IngestionProperties(
-        database=KUSTO_DATABASE,
-        table=KUSTO_TABLE,
-        data_format=DataFormat.CSV,
-        report_level=ReportLevel.FailuresAndSuccesses
-    )
-
-    # ingest from file
-    result = client.ingest_from_file(f"/tmp/codegen_report.csv", ingestion_properties=ingestion_props)
-    # Inspect the result for useful information, such as source_id and blob_url
-    print(repr(result))
-    logger.info('Finsh send codegen report csv data to kusto db.')
-
-
 if __name__ == '__main__':
-    send_to_kusto(generate_csv())
+    generate_csv()