Build and archive all JITs in the JIT rolling build (#42559)

* Build and archive all JITs in the JIT rolling build

* Disable debug info uploading

eng/pipelines/coreclr/templates/build-jit-job.yml

@@ -96,14 +96,14 @@ jobs:
 
     # Build CoreCLR JIT
     - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
-      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(compilerArg) -skipruntime -skipalljits -nopgooptimize
+      - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) -ci $(compilerArg) -skipruntime -nopgooptimize
         displayName: Build CoreCLR JIT
     - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
-      - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -skipruntime -skipalljits -nopgooptimize -skiprestoreoptdata
+      - script: set __TestIntermediateDir=int&&$(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci -skipruntime -nopgooptimize -skiprestoreoptdata
         displayName: Build CoreCLR JIT
 
     # Ensure the Python azure-storage-blob package is installed before doing the upload.
-    - script: $(PipScript) install --user azure.storage.blob==12.0.0 --force-reinstall
+    - script: $(PipScript) install --user azure.storage.blob==12.5.0 --force-reinstall
       displayName: Install azure-storage-blob Python package
 
     - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/jitrollingbuild.py upload -build_type $(buildConfig) -arch $(archType) -host_os $(osGroup) -git_hash $(Build.SourceVersion)

src/coreclr/scripts/jitrollingbuild.py

@@ -189,19 +189,23 @@ def list_az_jits(coreclr_args, filter=lambda unused: True):
         return None
 
     # Contents is an XML file with contents like:
-    #
-    # <EnumerationResults ContainerName="https://clrjit.blob.core.windows.net/superpmi">
+    # <EnumerationResults ContainerName="https://clrjit2.blob.core.windows.net/jitrollingbuild">
+    #   <Prefix>builds/</Prefix>
     #   <Blobs>
     #     <Blob>
-    #       <Name>jit-ee-guid/Linux/x64/Linux.x64.Checked.frameworks.mch.zip</Name>
-    #       <Url>https://clrjit.blob.core.windows.net/superpmi/jit-ee-guid/Linux/x64/Linux.x64.Checked.frameworks.mch.zip</Url>
+    #       <Name>builds/755f01659f03196487ec41225de8956911f8049b/Linux/x64/Checked/libclrjit.so</Name>
+    #       <Url>https://clrjit2.blob.core.windows.net/jitrollingbuild/builds/755f01659f03196487ec41225de8956911f8049b/Linux/x64/Checked/libclrjit.so</Url>
     #       <Properties>
     #         ...
     #       </Properties>
     #     </Blob>
     #     <Blob>
-    #       <Name>jit-ee-guid/Linux/x64/Linux.x64.Checked.mch.zip</Name>
-    #       <Url>https://clrjit.blob.core.windows.net/superpmi/jit-ee-guid/Linux/x64/Linux.x64.Checked.mch.zip</Url>
+    #       <Name>builds/755f01659f03196487ec41225de8956911f8049b/OSX/x64/Checked/libclrjit.dylib</Name>
+    #       <Url>https://clrjit2.blob.core.windows.net/jitrollingbuild/builds/755f01659f03196487ec41225de8956911f8049b/OSX/x64/Checked/libclrjit.dylib</Url>
+    #       <Properties>
+    #         ...
+    #       </Properties>
+    #     </Blob>
     #     ... etc. ...
     #   </Blobs>
     # </EnumerationResults>
@@ -244,16 +248,22 @@ def upload_blob(file, blob_name):
         with open(file, "rb") as data:
             blob_client.upload_blob(data)
 
-    # 1. Find the JIT in the product directory
-    # 2. Upload it
-    # 3. optional? upload JIT PDB/symbols
+    # 1. Find all the JIT builds in the product directory
+    # 2. Upload them
+    #
+    # We could also upload debug info, but it's not clear it's needed for most purposes, and it is very big:
+    # it increases the upload size from about 190MB to over 900MB for each roll.
+    #
+    # For reference, the JIT debug info is found:
+    #    a. For Windows, in the PDB subdirectory, e.g. PDB\clrjit.pdb
+    #    b. For Linux .dbg files, and Mac .dwarf files, in the same directory as the jit, e.g., libcoreclr.so.dbg
 
     # Target directory: <root>/git-hash/OS/architecture/build-flavor/
     # Note that build-flavor will probably always be Checked.
-    # Lower-case all the directory names.
 
     files = []
 
+    # First, find the primary JIT that we expect to find.
     jit_name = determine_jit_name(coreclr_args)
     jit_path = os.path.join(coreclr_args.product_location, jit_name)
     if not os.path.isfile(jit_path):
@@ -262,6 +272,42 @@ def upload_blob(file, blob_name):
 
     files.append(jit_path)
 
+    # Next, look for any and all cross-compilation JITs. These are named, e.g.:
+    #   clrjit_unix_x64_x64.dll
+    #   clrjit_win_arm_x64.dll
+    #   clrjit_win_arm64_x64.dll
+    # and so on, and live in the same product directory as the primary JIT.
+    #
+    # Note that the expression below explicitly filters out the primary JIT since we added that above.
+    # We handle the primary JIT specially so we can error if it is missing. For the cross-compilation
+    # JITs, we don't bother trying to ensure that all the ones we might expect are actually there.
+    #
+    # We don't do a recursive walk because the JIT is also copied to the "sharedFramework" subdirectory,
+    # so we don't want to pick that up.
+
+    if coreclr_args.host_os == "OSX":
+        allowed_extensions = [ ".dylib" ]
+        # Add .dwarf for debug info
+    elif coreclr_args.host_os == "Linux":
+        allowed_extensions = [ ".so" ]
+        # Add .dbg for debug info
+    elif coreclr_args.host_os == "Windows_NT":
+        allowed_extensions = [ ".dll" ]
+    else:
+        raise RuntimeError("Unknown OS.")
+
+    cross_jit_paths = [os.path.join(coreclr_args.product_location, item)
+            for item in os.listdir(coreclr_args.product_location)
+            if re.match(r'.*clrjit.*', item) and item != jit_name and any(item.endswith(extension) for extension in allowed_extensions)]
+    files += cross_jit_paths
+
+    # On Windows, grab the PDB files from a sub-directory.
+    #if coreclr_args.host_os == "Windows_NT":
+    #    pdb_dir = os.path.join(coreclr_args.product_location, "PDB")
+    #    if os.path.isdir(pdb_dir):
+    #        pdb_paths = [os.path.join(pdb_dir, item) for item in os.listdir(pdb_dir) if re.match(r'.*clrjit.*', item)]
+    #        files += pdb_paths
+
     print("Uploading:")
     for item in files:
         print("  {}".format(item))
@@ -376,19 +422,39 @@ def download_urls(urls, target_dir):
     return local_files
 
 
+def get_jit_urls(coreclr_args, find_all=False):
+    """ Helper method: collect a list of URLs for all the JIT files to download or list.
+
+    Args:
+        coreclr_args (CoreclrArguments): parsed args
+        find_all (bool): True to show all, or False to filter based on coreclr_args
+    """
+
+    blob_filter_string = "{}/{}/{}/{}".format(coreclr_args.git_hash, coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)
+    blob_prefix_filter = "{}/{}/{}".format(az_blob_storage_container_uri, az_builds_root_folder, blob_filter_string).lower()
+
+    # Determine if a URL in Azure Storage should be allowed. The URL looks like:
+    #   https://clrjit.blob.core.windows.net/jitrollingbuild/builds/git_hash/Linux/x64/Checked/clrjit.dll
+    # Filter to just the current git_hash, OS, architecture, and build_flavor.
+    # If "find_all" is True, then no filtering happens: everything is returned.
+    def filter_jits(url):
+        url = url.lower()
+        return find_all or url.startswith(blob_prefix_filter)
+
+    return list_az_jits(coreclr_args, filter_jits)
+
+
 def download_command(coreclr_args):
     """ Download the JIT
 
     Args:
         coreclr_args (CoreclrArguments): parsed args
     """
 
-    print("JIT download")
+    urls = get_jit_urls(coreclr_args, find_all=False)
+    if urls is None:
+        return
 
-    jit_name = determine_jit_name(coreclr_args)
-    blob_folder_name = "{}/{}/{}/{}/{}/{}".format(az_builds_root_folder, coreclr_args.git_hash, coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type, jit_name)
-    blob_uri = "{}/{}".format(az_blob_storage_container_uri, blob_folder_name)
-    urls = [ blob_uri ]
     download_urls(urls, coreclr_args.target_dir)
 
 
@@ -399,26 +465,17 @@ def list_command(coreclr_args):
         coreclr_args (CoreclrArguments) : parsed args
     """
 
-    blob_filter_string = "{}/{}/{}/{}".format(coreclr_args.git_hash, coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)
-    blob_prefix_filter = "{}/{}/{}".format(az_blob_storage_container_uri, az_builds_root_folder, blob_filter_string).lower()
-
-    # Determine if a URL in Azure Storage should be allowed. The URL looks like:
-    #   https://clrjit.blob.core.windows.net/jitsuperpmi/git_hash/Linux/x64/Checked/clrjit.dll.zip
-    # By default, filter to just the current git_hash, OS, architecture, and build_flavor.
-    def filter_jits(url):
-        url = url.lower()
-        return coreclr_args.all or url.startswith(blob_prefix_filter)
-
-    urls = list_az_jits(coreclr_args, filter_jits)
+    urls = get_jit_urls(coreclr_args, find_all=coreclr_args.all)
     if urls is None:
         return
 
     count = len(urls)
 
     if coreclr_args.all:
-        print("{} JIT".format(count))
+        print("{} JIT files".format(count))
     else:
-        print("{} JIT for {}".format(count, blob_filter_string))
+        blob_filter_string = "{}/{}/{}/{}".format(coreclr_args.git_hash, coreclr_args.host_os, coreclr_args.arch, coreclr_args.build_type)
+        print("{} JIT files for {}".format(count, blob_filter_string))
     print("")
 
     for url in urls: