Merge pull request #34 from anthropics/zak/add-computer-use

Add computer use demo
anthropics · Oct 22, 2024 · 0313b3e · 0313b3e
2 parents 9d5bb66 + 6ad79cc
commit 0313b3e
Show file tree

Hide file tree

Showing 44 changed files with 3,052 additions and 5 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -0,0 +1,129 @@
+env:
+  REGISTRY: ghcr.io/anthropics/anthropic-quickstarts
+name: build
+on:
+  pull_request:
+    paths:
+     - computer-use-demo/**
+  push:
+    branches:
+      - main
+    paths:
+      - computer-use-demo/**
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: true
+      matrix:
+        platform:
+          - amd64
+          - arm64
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set image tag
+        run: |
+          short_sha=$(git rev-parse --short ${{ github.sha }})
+          echo "TAG=${REGISTRY}:computer-use-demo-${short_sha}" >> "$GITHUB_ENV"
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/${{ matrix.platform }}
+          context: computer-use-demo
+          push: false
+          tags: ${{ env.TAG }}
+          cache-from: type=gha,scope=computer-use-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=computer-use-${{ matrix.platform }}
+          load: true
+      - name: Run container
+        run: docker run -d -p 8051:8051 ${{ env.TAG }}
+      - name: Check streamlit
+        run: |
+          timeout=60
+          start_time=$(date +%s)
+          docker_id=$(docker ps --filter "ancestor=${{ env.TAG }}" --format "{{.ID}}")
+          echo "docker_id=$docker_id" >> "$GITHUB_ENV"
+          while true; do
+            current_time=$(date +%s)
+            elapsed=$((current_time - start_time))
+            if [ $elapsed -ge $timeout ]; then
+              echo "Timeout reached. Container did not respond within $timeout seconds."
+              exit 1
+            fi
+            response=$(docker exec $docker_id curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:8501 || echo "000")
+            if [ "$response" = "200" ]; then
+              echo "Container responded with 200 OK"
+              exit 0
+            fi
+          done
+      - name: Check VNC
+        run: docker exec $docker_id nc localhost 5900 -z
+      - name: Check noVNC
+        run: docker exec $docker_id curl -s -o /dev/null -w "%{http_code}" http://localhost:6080 | grep -q 200 || exit 1
+      - name: Check landing page
+        run: docker exec $docker_id curl -s -o /dev/null -w "%{http_code}" http://localhost:8080 | grep -q 200 || exit 1
+      - name: Determine push tags
+        run: |
+          if [ "${{ github.event_name }}" == "pull_request" ]; then
+            echo "PUSH_TAGS=${TAG}-${{ matrix.platform }}" >> "$GITHUB_ENV"
+          else
+            echo "PUSH_TAGS=${TAG}-${{ matrix.platform }},${REGISTRY}:computer-use-demo-latest-${{ matrix.platform }}" >> "$GITHUB_ENV"
+          fi
+      - name: Push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/${{ matrix.platform }}
+          context: .
+          push: true
+          tags: ${{ env.PUSH_TAGS }}
+          cache-from: type=gha,scope=computer-use-${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=computer-use-${{ matrix.platform }}
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.actor}}
+          password: ${{secrets.GITHUB_TOKEN}}
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Set image tag
+        run: |
+          echo "SHORT_SHA=$(git rev-parse --short ${{ github.sha }})" >> "$GITHUB_ENV"
+      - name: Create SHA manifest and push
+        run: |
+          docker buildx imagetools create -t \
+            ${REGISTRY}:computer-use-demo-${SHORT_SHA} \
+            ${REGISTRY}:computer-use-demo-${SHORT_SHA}-amd64 \
+            ${REGISTRY}:computer-use-demo-${SHORT_SHA}-arm64
+
+      - name: Create latest manifest and push
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          docker buildx imagetools create -t \
+            ${REGISTRY}:computer-use-demo-latest \
+            ${REGISTRY}:computer-use-demo-latest-amd64 \
+            ${REGISTRY}:computer-use-demo-latest-arm64
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -0,0 +1,49 @@
+name: tests
+on:
+  pull_request: {}
+  push:
+    branches:
+      - main
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: computer-use-demo
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v1
+  pyright:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: computer-use-demo
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          cache: "pip"
+          python-version: "3.11.6"
+      - run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install -r dev-requirements.txt
+      - run: echo "$PWD/.venv/bin" >> $GITHUB_PATH
+      - uses: jakebailey/pyright-action@v1
+  pytest:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: computer-use-demo
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          cache: "pip"
+          python-version: "3.11.6"
+      - run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install -r dev-requirements.txt
+      - run: echo "$PWD/.venv/bin" >> $GITHUB_PATH
+      - run: pytest tests --junitxml=junit/test-results.xml
diff --git a/README.md b/README.md
@@ -1,25 +1,31 @@
 # Anthropic Quickstarts
 
-Anthropic Quickstarts is a collection of projects designed to help developers quickly get started with building deployable applications using the Anthropic API. Each quickstart provides a foundation that you can easily build upon and customize for your specific needs.
+Anthropic Quickstarts is a collection of projects designed to help developers quickly get started with building  applications using the Anthropic API. Each quickstart provides a foundation that you can easily build upon and customize for your specific needs.
 
 ## Getting Started
 
 To use these quickstarts, you'll need an Anthropic API key. If you don't have one yet, you can sign up for free at [console.anthropic.com](https://console.anthropic.com).
 
 ## Available Quickstarts
 
-### 1. Customer Support Agent
+### Customer Support Agent
 
-Our first quickstart project is a customer support agent powered by Claude. This project demonstrates how to leverage Claude's natural language understanding and generation capabilities to create an AI-assisted customer support system with access to a knowledge base.
+A customer support agent powered by Claude. This project demonstrates how to leverage Claude's natural language understanding and generation capabilities to create an AI-assisted customer support system with access to a knowledge base.
 
 [Go to Customer Support Agent Quickstart](./customer-support-agent)
 
-### 2. Financial Data Analyst
+### Financial Data Analyst
 
-Our second quickstart is a financial data analyst powered by Claude. This project demonstrates how to leverage Claude's capabilities with interactive data visualization to analyze financial data via chat.
+A financial data analyst powered by Claude. This project demonstrates how to leverage Claude's capabilities with interactive data visualization to analyze financial data via chat.
 
 [Go to Financial Data Analyst Quickstart](./financial-data-analyst)
 
+### Computer Use Demo
+
+An environment and tools that Claude can use to control a desktop computer. This project demonstrates how to leverage the computer use capabilities of the the new Claude 3.5 Sonnet model.
+
+[Go to Computer Use Demo Quickstart](./computer-use-demo)
+
 ## General Usage
 
 Each quickstart project comes with its own README and setup instructions. Generally, you'll follow these steps:

diff --git a/computer-use-demo/.gitignore b/computer-use-demo/.gitignore
@@ -0,0 +1,4 @@
+.venv
+.ruff_cache
+__pycache__
+.pytest_cache
diff --git a/computer-use-demo/.zed/settings.json b/computer-use-demo/.zed/settings.json
@@ -0,0 +1,12 @@
+{
+  "preferred_line_length": 88,
+  "languages": {
+    "Python": {
+      "language_servers": ["pyright", "ruff"]
+    }
+  },
+  "telemetry": {
+    "diagnostics": false,
+    "metrics": false
+  }
+}
diff --git a/computer-use-demo/Dockerfile b/computer-use-demo/Dockerfile
@@ -0,0 +1,105 @@
+FROM docker.io/ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_PRIORITY=high
+
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install \
+    build-essential \
+    # UI Requirements
+    xvfb \
+    xterm \
+    xdotool \
+    scrot \
+    imagemagick \
+    sudo \
+    mutter \
+    x11vnc \
+    # Python/pyenv reqs
+    build-essential \
+    libssl-dev  \
+    zlib1g-dev \
+    libbz2-dev \
+    libreadline-dev \
+    libsqlite3-dev \
+    curl \
+    git \
+    libncursesw5-dev \
+    xz-utils \
+    tk-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libffi-dev \
+    liblzma-dev \
+    # Network tools
+    net-tools \
+    netcat \
+    # PPA req
+    software-properties-common && \
+    # Userland apps
+    sudo add-apt-repository ppa:mozillateam/ppa && \
+    sudo apt-get install -y --no-install-recommends \
+    libreoffice \
+    firefox-esr \
+    x11-apps \
+    xpdf \
+    gedit \
+    xpaint \
+    tint2 \
+    galculator \
+    pcmanfm \
+    unzip && \
+    apt-get clean
+
+# Install noVNC
+RUN git clone --branch v1.5.0 https://github.com/novnc/noVNC.git /opt/noVNC && \
+    git clone --branch v0.12.0 https://github.com/novnc/websockify /opt/noVNC/utils/websockify && \
+    ln -s /opt/noVNC/vnc.html /opt/noVNC/index.html
+
+# setup user
+ENV USERNAME=computeruse
+ENV HOME=/home/$USERNAME
+RUN useradd -m -s /bin/bash -d $HOME $USERNAME
+RUN echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER computeruse
+WORKDIR $HOME
+
+# setup python
+RUN git clone https://github.com/pyenv/pyenv.git ~/.pyenv && \
+    cd ~/.pyenv && src/configure && make -C src && cd .. && \
+    echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc && \
+    echo 'command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc && \
+    echo 'eval "$(pyenv init -)"' >> ~/.bashrc
+ENV PYENV_ROOT="$HOME/.pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PATH"
+ENV PYENV_VERSION_MAJOR=3
+ENV PYENV_VERSION_MINOR=11
+ENV PYENV_VERSION_PATCH=6
+ENV PYENV_VERSION=$PYENV_VERSION_MAJOR.$PYENV_VERSION_MINOR.$PYENV_VERSION_PATCH
+RUN eval "$(pyenv init -)" && \
+    pyenv install $PYENV_VERSION && \
+    pyenv global $PYENV_VERSION && \
+    pyenv rehash
+
+ENV PATH="$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH"
+
+RUN python -m pip install --upgrade pip==23.1.2 setuptools==58.0.4 wheel==0.40.0 && \
+    python -m pip config set global.disable-pip-version-check true
+
+# only reinstall if requirements.txt changes
+COPY --chown=$USERNAME:$USERNAME computer_use_demo/requirements.txt $HOME/computer_use_demo/requirements.txt
+RUN python -m pip install -r $HOME/computer_use_demo/requirements.txt
+
+# setup desktop env & app
+COPY --chown=$USERNAME:$USERNAME image/ $HOME
+COPY --chown=$USERNAME:$USERNAME computer_use_demo/ $HOME/computer_use_demo/
+
+ARG DISPLAY_NUM=1
+ARG HEIGHT=768
+ARG WIDTH=1024
+ENV DISPLAY_NUM=$DISPLAY_NUM
+ENV HEIGHT=$HEIGHT
+ENV WIDTH=$WIDTH
+
+ENTRYPOINT [ "./entrypoint.sh" ]
diff --git a/computer-use-demo/LICENSE b/computer-use-demo/LICENSE
@@ -0,0 +1,7 @@
+Copyright 2024 Anthropic, PBC.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.