Use server id and clean up server records during startup (#681)

* support server id; fix duplicated server client error

* print messages about servers

* Fix functions

* Fix list client

* add mode to fix issue

* Fix s3

* support deleting files

* use user_info.id

* Fix http

* Fix visibility

* Clean up for http tests

* Fix test

* Add uninstall

* change server id

* allow passing env

* Fix helm chart docs

* Print server info

* replace py38 with py312

* disable check and clean up

* random server id if not provided

* Fix server id

* Fix visibility

* restore visibility changes

* Fix test http

* run in executor

* Fix manager_id

* Fix updating service

* set the default the workspace to current

* Fix manager id

* Fix helm chart

* Update helm install

* Fix installation

* add sleep

* small fix

* import asyncio

* Fix test

* Output detailed error message

* create workspace if not exists

.github/workflows/test-helm-chart.yml

@@ -37,17 +37,29 @@ jobs:
  run: |
  cd helm-charts
  helm package hypha-server
- helm repo index . --url https://amun-ai.github.io/hypha/helm-charts
+ mkdir -p ../dist
+ mv hypha-server-*.tgz ../dist
+ helm package redis
+ mv redis-*.tgz ../dist
 
- - name: Publish to gh-pages
- uses: peaceiris/actions-gh-pages@v3
- with:
- github_token: ${{ secrets.GITHUB_TOKEN }}
- publish_dir: ./helm-charts
- publish_branch: gh-pages
+ - name: Install Hypha Helm chart from packaged .tgz
+ run: |
+ helm install redis ./dist/redis-*.tgz --namespace=hypha
+ helm install hypha-server ./dist/hypha-server-*.tgz --namespace=hypha
 
  - name: Uninstall Hypha Helm chart
- run: helm uninstall hypha-server --namespace=hypha
+ run: |
+ helm uninstall hypha-server --namespace=hypha
+ helm uninstall redis --namespace=hypha
 
  - name: Delete namespace
  run: kubectl delete namespace hypha
+
+ - name: Upload to GitHub Pages
+ uses: peaceiris/actions-gh-pages@v3
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+ with:
+ github_token: ${{ secrets.GITHUB_TOKEN }}
+ publish_dir: ./dist
+ publish_branch: gh-pages
+ keep_files: true

.gitignore

@@ -1,3 +1,4 @@
+helm-charts/*.tgz
 **/.minio.sys
 *.DS_Store
 *.app

helm-charts/README.md

@@ -52,6 +52,21 @@ To uninstall the chart:
 helm uninstall hypha-server --namespace=hypha
 ```
 
+### Install Released Helm Charts
+
+You can also install the released helm charts from the [hypha helm repository](https://amun-ai.github.io/hypha/helm-charts):
+
+```bash
+helm repo add hypha https://amun-ai.github.io/hypha/helm-charts
+helm repo update
+helm install hypha-server hypha/hypha-server --namespace=hypha
+```
+
+To override the values, you can prepare a `values.yaml` file with the values you want to override and install the helm chart with the following command:
+```bash
+helm install hypha-server hypha/hypha-server --namespace=hypha -f values.yaml
+```
+
 ## Install Redis for scaling
 
 Hypha can use an external Redis to store global state and pub/sub messages shared between the server instances.
@@ -74,7 +89,23 @@ startupCommand:
  - "--redis-uri=redis://redis.hypha.svc.cluster.local:6379/0"
 ```
 
-Now, upgrade the helm chart:
+To actually support multiple server instances, you need to set the `replicaCount` to more than 1 in the `values.yaml` file:
+
+```yaml
+replicaCount: 3
+```
+
+You also need to set the `HYPHA_SERVER_ID` environment variable to the pod's UID in the `values.yaml` file:
+```yaml
+env:
+ - name: HYPHA_SERVER_ID
+ valueFrom:
+ # Use the pod's UID as the server ID
+ fieldRef:
+ fieldPath: metadata.uid
+```
+
+Make sure to update the `values.yaml` file with the correct `redis-uri` and `replicaCount`, and add the `HYPHA_SERVER_ID` environment variable, then upgrade the helm chart:
 ```bash
 helm upgrade hypha-server ./hypha-server --namespace=hypha
 ```

helm-charts/aks-hypha.md

@@ -218,9 +218,18 @@ First, install the Redis Helm chart:
 helm install redis ./redis --namespace hypha
 ```
 
-This will install Redis in the `hypha` namespace, make sure you update the `values.yaml` file to add the `redis-uri` to the `startupCommand`:
+This will install Redis in the `hypha` namespace, make sure you update the `values.yaml` file to add the `redis-uri` to the `startupCommand`, set the `replicaCount` to more than 1, and add the `HYPHA_SERVER_ID` environment variable (using the pod id).
 
 ```yaml
+replicaCount: 2
+
+env:
+ - name: HYPHA_SERVER_ID
+ valueFrom:
+ fieldRef:
+ # Use the pod's UID as the server ID
+ fieldPath: metadata.uid
+
 startupCommand:
  command: ["python", "-m", "hypha.server"]
  args:

helm-charts/hypha-server/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.20.35
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm-charts/hypha-server/values.yaml

@@ -102,6 +102,12 @@ env:
  key: JWT_SECRET
  - name: PUBLIC_BASE_URL
  value: "https://hypha.amun.ai"
+ # Use the pod's UID as the server ID
+ # This is important to ensure Hypha Server can handle multiple replicas
+ - name: HYPHA_SERVER_ID
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.uid
 
 # Define command-line arguments here
 startupCommand:
@@ -111,4 +117,3 @@ startupCommand:
  - "--port=9520"
  - "--public-base-url=$(PUBLIC_BASE_URL)"
  # - "--redis-uri=redis://redis.hypha.svc.cluster.local:6379/0"
-

hypha/VERSION

@@ -1,3 +1,3 @@
 {
- "version": "0.20.35.post2"
+ "version": "0.20.35.post3"
 }

hypha/card.py

@@ -35,15 +35,6 @@ def __init__(
  self.workspace_bucket = workspace_bucket
 
  self.s3client = self.s3_controller.create_client_sync()
-
- try:
- self.s3client.create_bucket(Bucket=self.workspace_bucket)
- logger.info("Bucket created: %s", self.workspace_bucket)
- except self.s3client.exceptions.BucketAlreadyExists:
- pass
- except self.s3client.exceptions.BucketAlreadyOwnedByYou:
- pass
-
  router = APIRouter()
 
  @router.get("/{workspace}/cards/{path:path}")

hypha/core/__init__.py

@@ -51,6 +51,7 @@ class ServiceConfig(BaseModel):
  workspace: Optional[str] = None
  flags: List[str] = []
  singleton: Optional[bool] = False
+ created_by: Optional[Dict] = None
 
 
 class ServiceInfo(BaseModel):

hypha/core/auth.py

@@ -137,9 +137,9 @@ def valid_token(authorization: str):
  status_code=401, detail="The token has expired. Please fetch a new one"
  ) from err
  except jwt.JWTError as err:
- raise HTTPException(status_code=401, detail=traceback.format_exc()) from err
+ raise HTTPException(status_code=401, detail=str(err)) from err
  except Exception as err:
- raise HTTPException(status_code=401, detail=traceback.format_exc()) from err
+ raise HTTPException(status_code=401, detail=str(err)) from err
 
 
 def generate_anonymous_user(scope=None) -> UserInfo: