Merge pull request #854 from eclipsesource/documentation-and-setup

docker compose file and extended documentation

Author: amvanbaren

README.md

@@ -62,6 +62,11 @@ After you created the GitHub OAuth app, the next step is to copy the _Client ID_
 
 With these settings in place, you should be able to log in by authorizing your OAuth app.
 
+### Docker Compose Setup
+
+If you prefer to quickly get started with a local, docker-based development environment, you can use the approach described in our [docker compose setup](doc/development.md#using-docker-compose).
+You can use our docker compose profiles, allowing you the option to either run a service directly in a docker container or to manually build and run it on your local machine.
+
 ### Google Cloud Setup
 
 If you would like to test file storage via Google Cloud, follow these steps:

cli/package.json

@@ -61,7 +61,8 @@
         "lint": "eslint -c ./configs/eslintrc.json --ext .ts src",
         "prepare": "yarn run clean && yarn run build",
         "publish:next": "yarn npm publish --tag next",
-        "publish:latest": "yarn npm publish --tag latest"
+        "publish:latest": "yarn npm publish --tag latest",
+        "load-extensions": "node scripts/load-test-extensions.js"
     },
     "packageManager": "yarn@4.1.0"
 }

cli/scripts/load-test-extensions.js

@@ -26,7 +26,7 @@ if (process.argv.length >= 4) {
 
 async function loadTestExtensions() {
     const publicReg = new Registry();
-    const localReg = new Registry({url: 'http://localhost:8080'});
+    const localReg = new Registry({registryUrl: process.env.OVSX_REGISTRY_URL ?? 'http://localhost:8080'});
     /** @type {{ extensions: import('../lib/registry').Extension[] } & import('../lib/registry').Response} */
     const search = await publicReg.getJson(new URL(`${DEFAULT_URL}/api/-/search?size=${searchSize}`));
     if (search.error) {
@@ -41,17 +41,25 @@ async function loadTestExtensions() {
             continue;
         }
         const fileName = await download(publicReg, meta);
-        const nsResult = await localReg.createNamespace(meta.namespace, accessToken);
-        if (nsResult.error && !nsResult.error.startsWith('Namespace already exists')) {
-            console.error(nsResult.error);
-        } else if (nsResult.success) {
-            console.log(nsResult.success);
+        try {
+          const nsResult = await localReg.createNamespace(meta.namespace, accessToken);
+          console.log(nsResult.success);
+        } catch (error) {
+          if (!error.message.startsWith('Namespace already exists')) {
+            console.error(error);
+            process.exit(1);
+          }
         }
-        const published = await localReg.publish(fileName, accessToken);
-        if (published.error && !published.error.endsWith('is already published.')) {
-            console.error(`\u274c  ${published.error}`);
-        } else if (published.namespace && published.name) {
+
+        try {
+          const published = await localReg.publish(fileName, accessToken);
+          if (published.namespace && published.name) {
             console.log(`\u2713  Published ${published.namespace}.${published.name}@${published.version}`);
+          }
+        } catch (error) {
+          if (!error.message.endsWith('is already published.')) {
+              console.error(`\u274c  ${error}`);
+          }
         }
     }
 }

doc/development.md

@@ -6,6 +6,29 @@ To get started quickly, it is recommended to use Gitpod as default with the deve
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/eclipse/openvsx)
 
+### Using Docker Compose
+
+To run the Open VSX registry in a development environment, you can use `docker compose` by following these steps:
+
+ * Verify Docker Compose is installed by running `docker compose version`. If an error occurs, you may need to [install docker compose](https://docs.docker.com/compose/install/) on your machine.
+ * Decide which profile(s) to run based on your needs. By default, only the PostgreSQL and Elasticsearch containers start, which suits running the OpenVSX server and web UI locally for easier debugging. The [docker-compose.yml] file defines additional profiles for specific components:
+   * `backend`: Starts the OpenVSX server container (java).
+   * `frontend`: Starts the web UI container.
+   * `commandline`: Starts a container with the OpenVSX CLI tools.
+   * `openvsx`: Combines `backend`, `frontend`, and `commandline` profiles to start all related services.
+   * `kibana`: Starts a kibana instance for easier access to the Elasticsearch service.
+ * In the project root, initiate Docker Compose:
+   * Without profiles: `docker compose up`.
+   * With profiles: `docker compose --profile <profile_name> up`. Use multiple `--profile` flags for multiple profiles, e.g., `docker compose --profile openvsx --profile kibana up`.
+
+ * Depending on which profile(s) you selected, after some seconds, the respective services become available:
+   * registry backend is available at [http://localhost:8080/](http://localhost:8080/) if the `backend` or `openvsx` profile was selected.
+   * web ui is available at [http://localhost:3000/](http://localhost:3000/) if the `frontend` or `openvsx` profile was selected.
+   * kibana is exposed at [http://localhost:5601/](http://localhost:5601/) if the `kibana` profile was selected.
+ * Open VSX CLI commands can be run via `docker compose exec cli lib/ovsx` if the `commandline` or `openvsx` profile was selected.
+ * To load some extensions from the main registry (openvsx.org), run `docker compose exec cli yarn load-extensions <N>`, where N is the number of extensions you would like to publish in your local registry.
+ * For troubleshooting or manual intervention, access a service's interactive shell with `docker compose run --rm <service> /bin/bash`. Service names are listed in the [docker-compose.yml](docker-compose.yml) file.
+
 ### Setup locally on WSL
 
 - Install WSL
@@ -96,7 +119,7 @@ To get started quickly, it is recommended to use Gitpod as default with the deve
 
 - Download the Elasticsearch image from Docker Hub and enable it in the Docker Desktop
 
-### Run the application
+### Run the application locally
 
 - cd cli
 
@@ -115,3 +138,19 @@ To get started quickly, it is recommended to use Gitpod as default with the deve
   - yarn build:default
   - yarn start:default
 - Go to localhost:3000 on browser and it should be up and running
+
+### Optional: Deploy example extensions to your local registry
+
+Run:
+
+- in `server/`:  
+  `gradlew downloadTestExtensions` to download vsix files from the official store and from Github.
+- in project root (the server application must be running):
+  ```bash
+  export OVSX_REGISTRY_URL=http://localhost:8080
+  export OVSX_PAT=super_token
+  export PUBLISHERS="DotJoshJohnson eamodio felixfbecker formulahendry HookyQR ms-azuretools ms-mssql ms-python ms-vscode octref redhat ritwickdey sburg vscode vscodevim Wscats"
+  for pub in $PUBLISHERS; do cli/lib/ovsx create-namespace $pub; done
+  find server/build/test-extensions-builtin -name '*.vsix' -exec cli/lib/ovsx publish '{}' \;
+  find server/build/test-extensions -name '*.vsix' -exec cli/lib/ovsx publish '{}' \;
+  ```

docker-compose.yml

@@ -0,0 +1,91 @@
+services:
+
+  postgres:
+    image: postgres:latest
+    environment:
+      - POSTGRES_USER=openvsx
+      - POSTGRES_PASSWORD=openvsx
+    logging:
+      options:
+        max-size: 10m
+        max-file: "3"
+    ports:
+      - '5432:5432'
+
+  elasticsearch:
+    image: elasticsearch:8.7.1
+    environment:
+      - xpack.security.enabled=false
+      - xpack.ml.enabled=false
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ports:
+      - 9200:9200
+      - 9300:9300
+    healthcheck:
+      test: curl -s http://elasticsearch01:9200 >/dev/null || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 50
+      start_period: 5s
+
+  kibana:
+    image: kibana:8.7.1
+    ports:
+      - "5601:5601"
+    environment:
+      - ELASTICSEARCH_URL=http://elasticsearch:9200
+    depends_on:
+      - elasticsearch
+    profiles:
+      - kibana
+
+  server:
+    image: openjdk:17
+    working_dir: /app
+    command: sh -c 'scripts/generate-properties.sh --docker && ./gradlew assemble && ./gradlew runServer'
+    volumes:
+      - ./server:/app
+    ports:
+      - 8080:8080
+    depends_on:
+      - postgres
+      - elasticsearch
+    healthcheck:
+      test: "curl --fail --silent localhost:8081/actuator/health | grep UP || exit 1"
+      interval: 10s
+      timeout: 5s
+      retries: 50
+      start_period: 5s
+    profiles:
+      - openvsx
+      - backend
+    
+  webui:
+    image: node:18
+    working_dir: /app
+    command: sh -c 'yarn && yarn build && yarn build:default && yarn start:default'
+    volumes:
+      - ./webui:/app
+    ports:
+      - 3000:3000
+    depends_on:
+      - server
+    profiles:
+      - openvsx
+      - frontend
+
+  cli:
+    image: node:18
+    working_dir: /app
+    command: sh -c 'yarn && yarn watch'
+    volumes:
+      - ./cli:/app
+    depends_on:
+      - server
+    environment:
+      - OVSX_REGISTRY_URL=http://server:8080
+    profiles:
+      - openvsx
+      - commandline

server/scripts/generate-properties.sh

@@ -1,13 +1,30 @@
-#!/bin/sh
+#!/bin/bash
+
+INSIDE_DOCKER=false
+if [[ $* == *--docker* ]]
+then
+    INSIDE_DOCKER=true;
+fi
 
 # This script must be run from the 'server' directory
 export OVSX_APP_PROFILE=$PWD/src/dev/resources/application-ovsx.properties
 
 # Clear the content of the ovsx application profile
 echo "# Generated by scripts/generate-properties.sh" > $OVSX_APP_PROFILE
 
-# Set the Elasticsearch host
-echo "ovsx.elasticsearch.host=localhost:9200" >> $OVSX_APP_PROFILE
+if [ "$INSIDE_DOCKER" = true ]
+then
+    # Set the Elasticsearch host
+    echo "ovsx.elasticsearch.host=elasticsearch:9200" >> $OVSX_APP_PROFILE
+
+    # Set the Postgres host
+    echo "spring.datasource.url=jdbc:postgresql://postgres:5432/postgres" >> $OVSX_APP_PROFILE
+    echo "spring.datasource.username=openvsx" >> $OVSX_APP_PROFILE
+    echo "spring.datasource.password=openvsx" >> $OVSX_APP_PROFILE
+else
+    # Set the Elasticsearch host
+    echo "ovsx.elasticsearch.host=localhost:9200" >> $OVSX_APP_PROFILE
+fi
 
 # Set the web UI URL
 if command -v gp > /dev/null