syn2mas tool to import users from Synapse

.dockerignore

@@ -3,6 +3,7 @@ crates/*/target
 crates/*/node_modules
 frontend/node_modules
 frontend/dist
+tools/syn2mas/**
 docs/
 .devcontainer/
 .git/

.github/dependabot.yml

@@ -64,3 +64,8 @@ updates:
       graphql-codegen:
         patterns:
           - "@graphql-codegen/*"
+
+  - package-ecosystem: "npm"
+    directory: "/tools/syn2mas/"
+    schedule:
+      interval: "weekly"

.github/workflows/build.yaml

@@ -22,6 +22,8 @@ env:
   CARGO_TERM_COLOR: always
   CARGO_NET_GIT_FETCH_WITH_CLI: "true"
   IMAGE: ghcr.io/matrix-org/matrix-authentication-service
+  IMAGE_SYN2MAS: ghcr.io/matrix-org/matrix-authentication-service/syn2mas
+  BUILDCACHE: ghcr.io/matrix-org/matrix-authentication-service/buildcache
 
 jobs:
   build-binaries:
@@ -44,11 +46,6 @@ jobs:
         with:
           version: 0.55.0
 
-      - name: Install Node
-        uses: actions/setup-node@v3.8.1
-        with:
-          node-version: 18
-
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
@@ -74,7 +71,12 @@ jobs:
       - name: Download the macOS SDK
         run: curl -L "https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz" | tar -J -x -C /opt
 
-      - name: Install Node dependencies
+      - name: Install frontend Node
+        uses: actions/setup-node@v3.8.1
+        with:
+          node-version: 18
+
+      - name: Install frontend Node dependencies
         working-directory: ./frontend
         run: npm ci
 
@@ -171,6 +173,22 @@ jobs:
             type=semver,pattern={{major}}
             type=sha
 
+      - name: Docker meta (syn2mas)
+        id: meta-syn2mas
+        uses: docker/metadata-action@v5.0.0
+        with:
+          images: "${{ env.IMAGE_SYN2MAS }}"
+          bake-target: docker-metadata-action-syn2mas
+          flavor: |
+            latest=auto
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+
       - name: Setup Cosign
         uses: sigstore/cosign-installer@v3.1.2
 
@@ -199,9 +217,11 @@ jobs:
             docker-bake.hcl
             ${{ steps.meta.outputs.bake-file }}
             ${{ steps.meta-debug.outputs.bake-file }}
+            ${{ steps.meta-syn2mas.outputs.bake-file }}
           set: |
             base.context=https://github.com/${{ github.repository }}.git#${{ github.ref }}
-            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
+            syn2mas.context=https://github.com/${{ github.repository }}.git#${{ github.ref }}:tools/syn2mas/
+            base.cache-from=type=registry,ref=${{ env.BUILDCACHE }}:buildcache
 
       - name: Build and push
         id: bake
@@ -214,9 +234,10 @@ jobs:
             ${{ steps.meta-debug.outputs.bake-file }}
           set: |
             base.context=https://github.com/${{ github.repository }}.git#${{ github.ref }}
+            syn2mas.context=https://github.com/${{ github.repository }}.git#${{ github.ref }}:tools/syn2mas/
             base.output=type=image,push=true
-            base.cache-from=type=registry,ref=${{ env.IMAGE }}:buildcache
-            base.cache-to=type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max
+            base.cache-from=type=registry,ref=${{ env.BUILDCACHE }}:buildcache
+            base.cache-to=type=registry,ref=${{ env.BUILDCACHE }}:buildcache,mode=max
 
       - name: Sign the images with GitHub Actions provided token
         # Only sign on tags and on commits on main branch
@@ -227,7 +248,8 @@ jobs:
         run: |-
           cosign sign --yes \
             "${{ env.IMAGE }}@${{ fromJSON(steps.bake.outputs.metadata).regular['containerimage.digest'] }}" \
-            "${{ env.IMAGE }}@${{ fromJSON(steps.bake.outputs.metadata).debug['containerimage.digest'] }}"
+            "${{ env.IMAGE }}@${{ fromJSON(steps.bake.outputs.metadata).debug['containerimage.digest'] }}" \
+            "${{ env.IMAGE_SYN2MAS }}@${{ fromJSON(steps.bake.outputs.metadata).syn2mas['containerimage.digest'] }}"
 
 
   release:
@@ -293,6 +315,17 @@ jobs:
                 ```
                 ${{ fromJSON(needs.build-image.outputs.metadata).debug['image.name'] }}
                 ```
+
+            `syn2mas` migration tool: 
+
+              - Digest: 
+                ```
+                ${{ env.IMAGE_SYN2MAS }}@${{ fromJSON(needs.build-image.outputs.metadata).syn2mas['containerimage.digest'] }}
+                ```
+              - Tags:
+                ```
+                ${{ fromJSON(needs.build-image.outputs.metadata).syn2mas['image.name'] }}
+                ```
 
           files: |
             mas-cli-aarch64-linux.tar.gz

.github/workflows/ci.yaml

@@ -281,6 +281,33 @@ jobs:
         run: |
           cargo test --workspace
 
+  syn2mas:
+    name: Check syn2mas
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@v4.1.0
+
+      - name: Install Node
+        uses: actions/setup-node@v3.8.1
+        with:
+          node-version-file: ./tools/syn2mas/.nvmrc
+
+      - name: Install Node dependencies
+        working-directory: ./tools/syn2mas
+        run: npm ci
+
+      - name: Lint
+        working-directory: ./tools/syn2mas
+        run: npm run lint
+
+      - name: Build
+        working-directory: ./tools/syn2mas
+        run: npm run build
 
   tests-done:
     name: Tests done
@@ -294,6 +321,7 @@ jobs:
       - clippy
       - check-schema
       - test
+      - syn2mas
     runs-on: ubuntu-latest
 
     steps:

docker-bake.hcl

@@ -1,9 +1,10 @@
 // This is what is baked by GitHub Actions
-group "default" { targets = ["regular", "debug"] }
+group "default" { targets = ["regular", "debug", "syn2mas"] }
 
 // Targets filled by GitHub Actions: one for the regular tag, one for the debug tag
 target "docker-metadata-action" {}
 target "docker-metadata-action-debug" {}
+target "docker-metadata-action-syn2mas" {}
 
 // This sets the platforms and is further extended by GitHub Actions to set the
 // output and the cache locations
@@ -22,3 +23,8 @@ target "debug" {
   inherits = ["base", "docker-metadata-action-debug"]
   target = "debug"
 }
+
+target "syn2mas" {
+  inherits = ["base", "docker-metadata-action-syn2mas"]
+  context = "./tools/syn2mas"
+}

docs/SUMMARY.md

@@ -12,6 +12,7 @@
 - [Database setup](./setup/database.md)
 - [Homeserver configuration](./setup/homeserver.md)
 - [Configuring a reverse proxy](./setup/reverse-proxy.md)
+- [Configuring .well-known](./setup/well-known.md)
 - [Running the service](./setup/running.md)
 - [Migrating an existing homeserver](./setup/migration.md)
 # Usage

docs/setup/migration.md

@@ -12,4 +12,69 @@ Features that are provided to support this include:
 - Ability to import existing upstream IdP subject ID mappings
 - Provides a compatibility layer for legacy Matrix authentication
 
+If
 There will be tools to help with the migration process itself. But these aren't quite ready yet.
+
+## Preparing for the migration
+
+The deployment is non-trivial so it is important to read through and understand the steps involved and make a plan before starting.
+
+### Run the migration advisor
+
+You can use the advisor mode of the `syn2mas` tool to identify extra configuration steps or issues with the configuration of the homeserver.
+
+```sh
+syn2mas --command=advisor --synapseConfigFile=homeserver.yaml
+```
+
+This will output `WARN` entries for any identified actions and `ERROR` entries in the case of any issues that will prevent the migration from working.
+
+### Install and configure MAS alongside your existing homeserver
+
+Follow the instructions in the [installation guide](installation.md) to install MAS alongside your existing homeserver.
+
+### Map any upstream SSO providers
+
+If you are using an upstream SSO provider then you will need to provision the upstream provide in MAS manually.
+
+Each upstream provider will need to be given as an `--upstreamProviderMapping` command line option to the import tool.
+
+### Do a dry-run of the import to test
+
+```sh
+syn2mas --command migrate --synapseConfigFile homeserver.yaml --masConfigFile config.yaml --dryRun
+```
+
+If no errors are reported then you can proceed to the next step.
+
+## Doing the migration
+
+Having done the preparation, you can now proceed with the actual migration. Note that this will require downtime for the homeserver and is not easily reversible.
+
+### Backup your data
+
+As with any migration, it is important to backup your data before proceeding.
+
+### Shutdown the homeserver
+
+This is to ensure that no new sessions are created whilst the migration is in progress.
+
+### Configure the homeserver
+
+Follow the instructions in the [homeserver configuration guide](homeserver.md) to configure the homeserver to use MAS.
+
+### Do the import
+
+Run `syn2mas` in non-dry-run mode.
+
+```sh
+syn2mas --command migrate --synapseConfigFile homeserver.yaml --masConfigFile config.yaml --dryRun false
+```
+
+### Start up the homeserver
+
+Start up the homeserver again with the new configuration.
+
+### Update or serve the .well-known
+
+The `.well-known/matrix/client` needs to be served as described [here](./well-known.md).

docs/setup/well-known.md

@@ -0,0 +1,23 @@
+# .well-known configuration
+
+A `.well-known/matrix/client` file is required to be served to allow clients to discover the authentication service.
+
+If no `.well-known/matrix/client` file is served currently then this will need to be enabled.
+
+If the homeserver is Synapse and serving this file already then the correct values will already be included when the homeserver is [configured to use MAS](./homeserver.md).
+
+If the .well-known is hosted elsewhere then `org.matrix.msc2965.authentication` entries need to be included similar to the following:
+
+```json
+{
+  "m.homeserver": {
+    "base_url": "https://matrix.example.com"
+  },
+  "org.matrix.msc2965.authentication": {
+    "issuer": "https://example.com/",
+    "account": "https://auth.example.com/account"
+  }
+}
+```
+
+For more context on what the correct values are, see [here](./).

tools/syn2mas/.eslintrc.cjs

@@ -0,0 +1,22 @@
+module.exports = {
+  plugins: [
+    'matrix-org',
+  ],
+  extends: [
+    'plugin:matrix-org/typescript',
+  ],
+  env: {
+    browser: false,
+    node: true,
+  },
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: "./tsconfig.json",
+  },
+  rules: {
+    '@typescript-eslint/no-floating-promises': 'error',
+    '@typescript-eslint/no-misused-promises': 'error',
+    '@typescript-eslint/promise-function-async': 'error',
+    '@typescript-eslint/await-thenable': 'error',
+  },
+};

tools/syn2mas/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

tools/syn2mas/.nvmrc

@@ -0,0 +1 @@
+18

tools/syn2mas/Dockerfile

@@ -0,0 +1,28 @@
+
+# Build Node.js app
+FROM --platform=${BUILDPLATFORM} node:18-alpine AS builder
+
+WORKDIR /syn2mas
+
+COPY ./package.json ./package-lock.json ./
+RUN --network=default \
+  npm ci
+
+COPY ./tsconfig.json ./
+COPY ./src ./src/
+
+RUN --network=none \
+  npm run build
+
+# Put together a runtime image
+FROM --platform=${BUILDPLATFORM} node:18-alpine
+
+WORKDIR /syn2mas
+
+COPY ./package.json ./package-lock.json ./
+RUN --network=default \
+  npm ci --omit=dev
+
+COPY --from=builder /syn2mas/dist ./dist
+
+ENTRYPOINT ["node", "dist/index.js"]

tools/syn2mas/README.md

@@ -0,0 +1,16 @@
+# syn2mas -  Synapse to Matrix Authentication Service
+
+Tool to help with the migration of a Matrix Synapse installation to the Matrix Authentication Service.
+
+The tool has two modes of operation:
+
+- Advisor mode: Analyses the Synapse configuration and reports on any issues that would prevent a successful migration.
+- Migration mode: Performs the migration of the Synapse database into the Matrix Authentication Service database.
+
+## Usage
+
+Pre-migration advisor:
+
+```sh
+npm run dev -- advisor --synapseConfigFile homeserver.yaml
+```