Merge branch 'master' into azure-blob-storage

.github/issue_template.md

@@ -0,0 +1,17 @@
+---
+name: Issue
+about: Use this to open new issues.
+title: ''
+labels: ''
+assignees: ''
+
+---
+<!-- Describe the bug report / feature request here -->
+
+
+
+
+<!-- If relevant, fill in versioning info to help us troubleshoot -->
+_Version & Commit info:_
+* `sky -v`: PLEASE_FILL_IN
+* `sky -c`: PLEASE_FILL_IN

.github/workflows/docker-nightly-build.yaml

@@ -0,0 +1,58 @@
+name: docker-nightly-build
+
+on:
+  schedule:
+    # Set the time to be 20 mins after the pypi nightly build
+    - cron: '55 10 * * *' # 10:55am UTC, 2:55am PST, 5:55am EST
+  workflow_dispatch:
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Find the release version
+        id: version
+        run: |
+          PACKAGE_NAME="skypilot-nightly"
+          # Fetch package info from PyPI
+          PACKAGE_INFO=$(curl -s https://pypi.org/pypi/$PACKAGE_NAME/json)
+          # Parse JSON and get the latest version
+          LATEST_VERSION=$(echo $PACKAGE_INFO | jq -r '.info.version')
+          echo "latest_version=$LATEST_VERSION" >> $GITHUB_OUTPUT
+
+      - name: Setup Dockerfile
+        run: |
+          touch Dockerfile
+          cat <<EOF > Dockerfile
+          # Use the latest version with Python 3.10
+          FROM continuumio/miniconda3:23.3.1-0
+
+          # Install dependencies
+          RUN conda install -c conda-forge google-cloud-sdk && \
+              apt update -y && \
+              apt install rsync vim -y && \
+              rm -rf /var/lib/apt/lists/*
+          
+          RUN pip install "skypilot-nightly[all]==${{ steps.version.outputs.latest_version }}" --no-cache-dir
+          EOF
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          # TODO: add support for linux/arm64 for M series Macs
+          # It currently fails due to the failure to install google-cloud-sdk
+          platforms: linux/amd64
+          push: true
+          tags: "${{ secrets.DOCKER_USERNAME }}/skypilot-nightly:latest,${{ secrets.DOCKER_USERNAME }}/skypilot-nightly:${{ steps.version.outputs.latest_version }}"
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/format.yml

@@ -33,13 +33,11 @@ jobs:
     - name: Running yapf
       run: |
         yapf --diff --recursive ./ --exclude 'sky/skylet/ray_patches/**' \
-            --exclude 'sky/skylet/providers/gcp/**' \
             --exclude 'sky/skylet/providers/azure/**' \
             --exclude 'sky/skylet/providers/ibm/**'
     - name: Running black
       run: |
-        black --diff --check sky/skylet/providers/gcp/ \
-            sky/skylet/providers/azure/ \
+        black --diff --check sky/skylet/providers/azure/ \
             sky/skylet/providers/ibm/
     - name: Running isort for black formatted files
       run: |
@@ -48,6 +46,5 @@ jobs:
     - name: Running isort for yapf formatted files
       run: |
         isort --diff --check ./ --sg 'sky/skylet/ray_patches/**' \
-            --sg 'sky/skylet/providers/gcp/**' \
             --sg 'sky/skylet/providers/azure/**' \
             --sg 'sky/skylet/providers/ibm/**'

.github/workflows/pylint.yml

@@ -28,7 +28,7 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install ".[all]"
-        pip install pylint==2.8.2
+        pip install pylint==2.14.5
         pip install pylint-quotes==0.2.3
     - name: Analysing the code with pylint
       run: |

.github/workflows/pypi-nightly-build.yml

@@ -57,8 +57,10 @@ jobs:
         with:
           password: ${{ secrets.TEST_PYPI_API_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
+          skip-existing: true
       - name: Publish distribution to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
+          skip-existing: true
 

.github/workflows/pytest.yml

@@ -27,7 +27,7 @@ jobs:
           - tests/test_optimizer_random_dag.py
           - tests/test_storage.py
           - tests/test_wheels.py
-          - tests/test_spot.py
+          - tests/test_spot_serve.py
           - tests/test_yaml_parser.py
     runs-on: ubuntu-latest
     steps:

.github/workflows/test-doc-build.yml

@@ -0,0 +1,35 @@
+name: Documentation Build
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the master branch
+  push:
+    branches:
+      - master
+      - 'releases/**'
+  pull_request:
+    branches:
+      - master
+      - 'releases/**'
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+        cd docs
+        pip install -r ./requirements-docs.txt
+    - name: Build documentation
+      run: |
+        cd ./docs
+        ./build.sh

.pylintrc

@@ -5,7 +5,7 @@
 # Its canonical open-source location is:
 #   https://google.github.io/styleguide/pylintrc
 
-[MASTER]
+[MAIN]
 
 # Files or directories to be skipped. They should be base names, not paths.
 ignore=third_party,ray_patches,providers
@@ -50,7 +50,8 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=abstract-method,
+disable=R,
+        abstract-method,
         apply-builtin,
         arguments-differ,
         attribute-defined-outside-init,
@@ -60,6 +61,7 @@ disable=abstract-method,
         buffer-builtin,
         c-extension-no-member,
         consider-using-enumerate,
+        consider-using-f-string, # FIXME(sky): make pass
         cmp-builtin,
         cmp-method,
         coerce-builtin,
@@ -76,7 +78,7 @@ disable=abstract-method,
         global-statement,
         hex-method,
         idiv-method,
-        implicit-str-concat-in-sequence,
+        implicit-str-concat,
         import-error,
         import-self,
         import-star-module-level,
@@ -139,6 +141,7 @@ disable=abstract-method,
         trailing-newlines,
         unichr-builtin,
         unicode-builtin,
+        unnecessary-lambda-assignment, # FIXME(sky): make pass.
         unnecessary-pass,
         unpacking-in-except,
         useless-else-on-loop,
@@ -157,12 +160,6 @@ disable=abstract-method,
 # mypackage.mymodule.MyReporterClass.
 output-format=text
 
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]". This option is deprecated
-# and it will be removed in Pylint 2.0.
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=no
 
@@ -269,7 +266,7 @@ generated-members=
 # Maximum number of characters on a single line.
 max-line-length=80
 
-# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt
+# TODO(https://github.com/pylint-dev/pylint/issues/3352): Direct pylint to exempt
 # lines made too long by directives to pytype.
 
 # Regexp for a line that is allowed to be longer than the limit.
@@ -281,12 +278,6 @@ ignore-long-lines=(?x)(
 # else.
 single-line-if-stmt=yes
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=
-
 # Maximum number of lines in a module
 max-module-lines=99999
 

.readthedocs.yml

@@ -6,9 +6,9 @@
 version: 2
 
 build:
-  os: "ubuntu-20.04"
+  os: "ubuntu-22.04"
   tools:
-    python: "3.8"
+    python: "3.10"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

Dockerfile

@@ -1,15 +1,11 @@
-FROM continuumio/miniconda3:4.11.0
+# Use the latest version with Python 3.10
+FROM continuumio/miniconda3:23.3.1-0
 
 # Install dependencies
 RUN conda install -c conda-forge google-cloud-sdk && \
     apt update -y && \
-    apt install rsync -y && \
+    apt install rsync vim -y && \
     rm -rf /var/lib/apt/lists/*
 
-COPY . /sky/
-
-WORKDIR /sky/
-
 # Install sky
-RUN cd /sky/ && \
-    pip install ".[all]"
+RUN pip install --no-cache-dir "skypilot[all]==0.5.0"

Dockerfile_k8s

@@ -1,11 +1,11 @@
-FROM continuumio/miniconda3:22.11.1
+FROM continuumio/miniconda3:23.3.1-0
 
 # TODO(romilb): Investigate if this image can be consolidated with the skypilot
 #  client image (`Dockerfile`)
 
 # Initialize conda for root user, install ssh and other local dependencies
 RUN apt update -y && \
-    apt install gcc rsync sudo patch openssh-server pciutils nano fuse -y && \
+    apt install gcc rsync sudo patch openssh-server pciutils nano fuse socat netcat -y && \
     rm -rf /var/lib/apt/lists/* && \
     apt remove -y python3 && \
     conda init
@@ -25,26 +25,21 @@ RUN useradd -m -s /bin/bash sky && \
 # Switch to sky user
 USER sky
 
-# Install SkyPilot pip dependencies
+# Install SkyPilot pip dependencies preemptively to speed up provisioning time
 RUN pip install wheel Click colorama cryptography jinja2 jsonschema && \
     pip install networkx oauth2client pandas pendulum PrettyTable && \
-    pip install ray[default]==2.4.0 rich tabulate filelock && \
+    pip install ray[default]==2.9.3 rich tabulate filelock && \
     pip install packaging 'protobuf<4.0.0' pulp && \
-    pip install awscli boto3 pycryptodome==3.12.0 && \
-    pip install docker kubernetes
+    pip install pycryptodome==3.12.0 && \
+    pip install docker kubernetes==28.1.0
 
 # Add /home/sky/.local/bin/ to PATH
 RUN echo 'export PATH="$PATH:$HOME/.local/bin"' >> ~/.bashrc
 
-# Install SkyPilot. This is purposely separate from installing SkyPilot
-# dependencies to optimize rebuild time
+# Copy SkyPilot code base. This is required for the ssh jump pod to find the
+# lifecycle management scripts
 COPY --chown=sky . /skypilot/sky/
 
-# TODO(romilb): Installing SkyPilot may not be necessary since ray up will do it
-RUN cd /skypilot/ && \
-    sudo mv -v sky/setup_files/* . && \
-    pip install ".[aws]"
-
 # Set PYTHONUNBUFFERED=1 to have Python print to stdout/stderr immediately
 ENV PYTHONUNBUFFERED=1
 

Dockerfile_k8s_gpu

@@ -1,13 +1,13 @@
 # TODO(romilb) - The base image used here (ray) is very large (11.4GB).
 #  as a result, this built image is about 13.5GB. We need to pick a lighter base
 #  image.
-FROM rayproject/ray:2.4.0-py310-gpu
+FROM rayproject/ray:2.9.3-py310-gpu
 
 # Initialize conda for root user, install ssh and other local dependencies
 # We remove cuda lists to avoid conflicts with the cuda version installed by ray
 RUN sudo rm -rf /etc/apt/sources.list.d/cuda* && \
     sudo apt update -y && \
-    sudo apt install gcc rsync sudo patch openssh-server pciutils nano fuse unzip -y && \
+    sudo apt install gcc rsync sudo patch openssh-server pciutils nano fuse unzip socat netcat -y && \
     sudo rm -rf /var/lib/apt/lists/* && \
     sudo apt remove -y python3 && \
     conda init
@@ -38,21 +38,16 @@ RUN pip install wheel Click colorama cryptography jinja2 jsonschema && \
     pip install networkx oauth2client pandas pendulum PrettyTable && \
     pip install rich tabulate filelock && \
     pip install packaging 'protobuf<4.0.0' pulp && \
-    pip install awscli boto3 pycryptodome==3.12.0 && \
-    pip install docker kubernetes
+    pip install pycryptodome==3.12.0 && \
+    pip install docker kubernetes==28.1.0
 
 # Add /home/sky/.local/bin/ to PATH
 RUN echo 'export PATH="$PATH:$HOME/.local/bin"' >> ~/.bashrc
 
-# Install SkyPilot. This is purposely separate from installing SkyPilot
-# dependencies to optimize rebuild time
+# Copy SkyPilot code base. This is required for the ssh jump pod to find the
+# lifecycle management scripts
 COPY --chown=sky . /skypilot/sky/
 
-# TODO(romilb): Installing SkyPilot may not be necessary since ray up will do it
-RUN cd /skypilot/ && \
-    sudo mv -v sky/setup_files/* . && \
-    pip install ".[aws]"
-
 # Set PYTHONUNBUFFERED=1 to have Python print to stdout/stderr immediately
 ENV PYTHONUNBUFFERED=1