From 23071fbd568b3e14ed360394acf7170d80c6e4dd Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 18:19:12 -0300 Subject: [PATCH 01/14] feat: `matrix-admin` and e2e tests --- .env.example | 8 + .github/PULL_REQUEST_TEMPLATE.md | 27 +++ .github/dependabot.yml | 9 + .github/workflows/ci.yml | 53 ++++++ .github/workflows/dependabot-auto-approve.yml | 21 +++ .github/workflows/dependabot-auto-merge.yml | 23 +++ Cargo.toml | 10 ++ Justfile | 4 + crates/matrix/Cargo.toml | 20 +++ crates/matrix/src/admin/client.rs | 137 +++++++++++++++ crates/matrix/src/admin/mod.rs | 4 + crates/matrix/src/admin/resources/mod.rs | 1 + .../matrix/src/admin/resources/token/mod.rs | 1 + .../admin/resources/token/shared_secret.rs | 161 ++++++++++++++++++ crates/matrix/src/lib.rs | 18 ++ crates/test/Cargo.toml | 20 +++ crates/test/fixtures/synapse/homeserver.yaml | 37 ++++ .../synapse/matrix.localhost.log.config | 39 +++++ .../synapse/matrix.localhost.signing.key | 1 + crates/test/src/environment.rs | 28 +++ crates/test/src/lib.rs | 5 + crates/test/src/matrix/mod.rs | 1 + .../src/matrix/shared_token_registration.rs | 39 +++++ fixtures/generate_mac.py | 32 ++++ fixtures/synapse/homeserver.yaml | 45 +++++ fixtures/synapse/matrix.localhost.log.config | 39 +++++ fixtures/synapse/matrix.localhost.signing.key | 1 + 27 files changed, 784 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/dependabot-auto-approve.yml create mode 100644 .github/workflows/dependabot-auto-merge.yml create mode 100644 Cargo.toml create mode 100644 crates/matrix/Cargo.toml create mode 100644 crates/matrix/src/admin/client.rs create mode 100644 crates/matrix/src/admin/mod.rs create mode 100644 crates/matrix/src/admin/resources/mod.rs create mode 100644 crates/matrix/src/admin/resources/token/mod.rs create mode 100644 crates/matrix/src/admin/resources/token/shared_secret.rs create mode 100644 crates/matrix/src/lib.rs create mode 100644 crates/test/Cargo.toml create mode 100644 crates/test/fixtures/synapse/homeserver.yaml create mode 100644 crates/test/fixtures/synapse/matrix.localhost.log.config create mode 100644 crates/test/fixtures/synapse/matrix.localhost.signing.key create mode 100644 crates/test/src/environment.rs create mode 100644 crates/test/src/lib.rs create mode 100644 crates/test/src/matrix/mod.rs create mode 100644 crates/test/src/matrix/shared_token_registration.rs create mode 100644 fixtures/generate_mac.py create mode 100644 fixtures/synapse/homeserver.yaml create mode 100644 fixtures/synapse/matrix.localhost.log.config create mode 100644 fixtures/synapse/matrix.localhost.signing.key diff --git a/.env.example b/.env.example index a4eee53..aa2eae6 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,11 @@ +# Commune +# +# The shared secret is used to authenticate the registration requests. +# +# This is explicitly passed here for development purposes, it should match the +# same as on `fixtures/synapse/homeserver.yaml` for CI. +COMMUNE_REGISTRATION_SHARED_SECRET='m@;wYOUOh0f:CH5XA65sJB1^q01~DmIriOysRImot,OR_vzN&B' + # Matrix Client MATRIX_HOST=http://localhost:8008 MATRIX_ADMIN_TOKEN=secret diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..2efb23b --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,27 @@ + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..1224165 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: 'cargo' + directory: '/' + schedule: + interval: 'weekly' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a01f2fc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: Continuous Integration + +on: + push: + branches: + - main + pull_request: + branches: [main] + paths: + - "**" + - "!/*.md" + - "!/**.md" + +concurrency: + group: "${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}" + cancel-in-progress: true + +jobs: + ci: + name: CI + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Rust Toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@v2 + + - name: Check formatting + run: cargo fmt --check + + - name: Check clippy + run: cargo clippy --workspace -- -D warnings + + - name: Unit Tests + run: cargo test -p matrix + + - name: Install Just + uses: extractions/setup-just@v1 + + - name: Prepare Data for Tests + run: | + cp -R ./crates/test/fixtures/synapse ./docker/synapse + docker compose up -d + + - name: E2E Tests + run: cargo test -p test -- --test-threads=1 diff --git a/.github/workflows/dependabot-auto-approve.yml b/.github/workflows/dependabot-auto-approve.yml new file mode 100644 index 0000000..fa79cad --- /dev/null +++ b/.github/workflows/dependabot-auto-approve.yml @@ -0,0 +1,21 @@ +name: Dependabot auto-approve +on: pull_request_target + +permissions: + pull-requests: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1.1.1 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Approve a PR + run: gh pr review --approve "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml new file mode 100644 index 0000000..d52cf3e --- /dev/null +++ b/.github/workflows/dependabot-auto-merge.yml @@ -0,0 +1,23 @@ +name: Dependabot auto-merge +on: pull_request_target + +permissions: + pull-requests: write + contents: write + +jobs: + dependabot: + runs-on: ubuntu-latest + if: ${{ github.actor == 'dependabot[bot]' }} + steps: + - name: Dependabot metadata + id: metadata + uses: dependabot/fetch-metadata@v1.1.1 + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Enable auto-merge for Dependabot PRs + if: ${{steps.metadata.outputs.update-type == 'version-update:semver-patch'}} + run: gh pr merge --auto --squash "$PR_URL" + env: + PR_URL: ${{github.event.pull_request.html_url}} + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..7b6d30d --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,10 @@ +[workspace] +members = [ + "crates/matrix", + "crates/test" +] +resolver = "1" + +[workspace.dependencies] +serde = "1.0.192" +url = { version = "2.4.1", features = ["serde"] } diff --git a/Justfile b/Justfile index ab74a98..fac9f3b 100644 --- a/Justfile +++ b/Justfile @@ -27,3 +27,7 @@ stop: clear: stop docker compose rm --all --force --volumes --stop docker volume rm commune_synapse_database || true + +# Runs all the tests from the `test` package. Optionally runs a single one if name pattern is provided +e2e *args='': + cargo test --package test -- --test-threads=1 $1 diff --git a/crates/matrix/Cargo.toml b/crates/matrix/Cargo.toml new file mode 100644 index 0000000..28007be --- /dev/null +++ b/crates/matrix/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "matrix" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1.0.75" +async-trait = "0.1.74" +hex = "0.4.3" +hmac = "0.12.1" +matrix-sdk = { git = "https://github.com/matrix-org/matrix-rust-sdk.git", rev = "e43a25a" } +reqwest = { version = "0.11.22", features = ["json"] } +serde_path_to_error = "0.1.14" +serde_qs = "0.12.0" +sha1 = "0.10.6" + +# Workspace Dependencies +serde = { workspace = true } +url = { workspace = true } diff --git a/crates/matrix/src/admin/client.rs b/crates/matrix/src/admin/client.rs new file mode 100644 index 0000000..824e875 --- /dev/null +++ b/crates/matrix/src/admin/client.rs @@ -0,0 +1,137 @@ +use std::ops::Deref; +use std::str::from_utf8; + +use anyhow::{bail, Result}; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use reqwest::{Client as HttpClient, Response}; +use serde::Serialize; +use url::Url; + +pub struct Client { + client: HttpClient, + base_url: Url, + token: Option, +} + +impl Client { + pub fn new>(url: S) -> Result { + let url = Url::parse(url.as_ref())?; + + Ok(Self { + client: HttpClient::new(), + base_url: url, + token: None, + }) + } + + /// Sets the token to be used for authentication with the server. + pub fn set_token(&mut self, token: impl Into) -> Result<()> { + let token = token.into(); + + if token.is_empty() { + self.token = None; + bail!("Token cannot be empty"); + } + + self.token = Some(token); + Ok(()) + } + + pub async fn get(&self, path: impl AsRef) -> Result { + let url = self.build_url(path)?; + let headers = self.build_headers()?; + let response = self.client.get(url).headers(headers).send().await?; + + Ok(response) + } + + pub async fn get_query( + &self, + path: impl AsRef, + params: impl Serialize, + ) -> Result { + let url = self.build_url_with_params(path, params)?; + let headers = self.build_headers()?; + let response = self.client.get(url).headers(headers).send().await?; + + Ok(response) + } + + pub async fn post_json(&self, path: impl AsRef, body: &T) -> Result + where + T: Serialize, + { + let url = self.build_url(path)?; + let headers = self.build_headers()?; + let resp = self + .client + .post(url) + .json(body) + .headers(headers) + .send() + .await?; + + Ok(resp) + } + + pub async fn put_json(&self, path: impl AsRef, body: &T) -> Result + where + T: Serialize, + { + let url = self.build_url(path)?; + let headers = self.build_headers()?; + let resp = self + .client + .put(url) + .json(body) + .headers(headers) + .send() + .await?; + + Ok(resp) + } + + fn build_headers(&self) -> Result { + let mut headers = HeaderMap::new(); + + if let Some(token) = &self.token { + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", token))?, + ); + } + + Ok(headers) + } + + #[inline] + fn build_url(&self, path: impl AsRef) -> Result { + let mut next = self.base_url.clone(); + + next.set_path(path.as_ref()); + + Ok(next) + } + + fn build_url_with_params(&self, path: impl AsRef, params: impl Serialize) -> Result { + let mut url = self.build_url(path)?; + let mut buff = Vec::new(); + let qs_ser = &mut serde_qs::Serializer::new(&mut buff); + + serde_path_to_error::serialize(¶ms, qs_ser)?; + + let params = from_utf8(buff.as_slice())?.to_string(); + + url.set_query(Some(¶ms)); + + Ok(url) + } +} + +impl Deref for Client { + type Target = HttpClient; + + fn deref(&self) -> &Self::Target { + &self.client + } +} diff --git a/crates/matrix/src/admin/mod.rs b/crates/matrix/src/admin/mod.rs new file mode 100644 index 0000000..18bbbeb --- /dev/null +++ b/crates/matrix/src/admin/mod.rs @@ -0,0 +1,4 @@ +pub mod client; +pub mod resources; + +pub use client::Client; diff --git a/crates/matrix/src/admin/resources/mod.rs b/crates/matrix/src/admin/resources/mod.rs new file mode 100644 index 0000000..79c66ba --- /dev/null +++ b/crates/matrix/src/admin/resources/mod.rs @@ -0,0 +1 @@ +pub mod token; diff --git a/crates/matrix/src/admin/resources/token/mod.rs b/crates/matrix/src/admin/resources/token/mod.rs new file mode 100644 index 0000000..24b2441 --- /dev/null +++ b/crates/matrix/src/admin/resources/token/mod.rs @@ -0,0 +1 @@ +pub mod shared_secret; diff --git a/crates/matrix/src/admin/resources/token/shared_secret.rs b/crates/matrix/src/admin/resources/token/shared_secret.rs new file mode 100644 index 0000000..b7c732a --- /dev/null +++ b/crates/matrix/src/admin/resources/token/shared_secret.rs @@ -0,0 +1,161 @@ +//! [Shared-Secret Registration API](https://matrix-org.github.io/synapse/latest/admin_api/register_api.html#) +//! +//! # Important +//! +//! This API is disabled when MSC3861 is enabled. See [#15582](https://github.com/matrix-org/synapse/pull/15582) +//! +//! This API allows for the creation of users in an administrative and +//! non-interactive way. This is generally used for bootstrapping a Synapse +//! instance with administrator accounts. +//! +//! To authenticate yourself to the server, you will need both the shared secret +//! (registration_shared_secret in the homeserver configuration), and a one-time +//! nonce. If the registration shared secret is not configured, this API is not +//! enabled. + +use anyhow::Result; +use hex; +use hmac::{Hmac, Mac}; +use serde::{Deserialize, Serialize}; +use sha1::Sha1; + +use crate::admin::Client; + +type HmacSha1 = Hmac; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Nonce { + pub nonce: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SharedSecretRegistrationDto { + pub nonce: String, + pub username: String, + pub displayname: Option, + pub password: String, + pub admin: bool, + /// The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the + /// key being the shared secret and the content being the nonce, user, + /// password, either the string "admin" or "notadmin", and optionally the + /// user_type each separated by NULs. + pub mac: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SharedSecretRegistration { + pub access_token: String, + pub user_id: String, + pub home_server: String, + pub device_id: String, +} + +impl SharedSecretRegistration { + /// Fetches the `Nonce` from the server. + /// + /// Refer: https://matrix-org.github.io/synapse/latest/admin_api/register_api.html#shared-secret-registration + pub async fn get_nonce(client: &Client) -> Result { + let resp = client.get("/_synapse/admin/v1/register").await?; + + Ok(resp.json().await?) + } + + /// Creates the [`SharedSecretRegistration`] instance. + /// + /// Refer: https://matrix-org.github.io/synapse/latest/admin_api/register_api.html#shared-secret-registration + pub async fn create(client: &Client, dto: SharedSecretRegistrationDto) -> Result { + let resp = client + .post_json("/_synapse/admin/v1/register", &dto) + .await?; + + Ok(resp.json().await?) + } + + /// Generates the MAC. + /// + /// # Inspiration + /// + /// This implementation is inspired by the following Python code from the + /// Synapse documentation on `Shared-Secret Registration`. + /// + /// ```python + /// import hmac, hashlib + /// + /// def generate_mac(nonce, user, password, admin=False, user_type=None): + /// + /// mac = hmac.new( + /// key=shared_secret, + /// digestmod=hashlib.sha1, + /// ) + /// + /// mac.update(nonce.encode('utf8')) + /// mac.update(b"\x00") + /// mac.update(user.encode('utf8')) + /// mac.update(b"\x00") + /// mac.update(password.encode('utf8')) + /// mac.update(b"\x00") + /// mac.update(b"admin" if admin else b"notadmin") + /// if user_type: + /// mac.update(b"\x00") + /// mac.update(user_type.encode('utf8')) + /// + /// return mac.hexdigest() + /// ``` + /// [Source](https://matrix-org.github.io/synapse/latest/admin_api/register_api.html#shared-secret-registration) + pub fn generate_mac>( + shared_secret: S, + nonce: S, + user: S, + password: S, + admin: bool, + user_type: Option, + ) -> Result { + let mut mac = HmacSha1::new_from_slice(shared_secret.as_ref().as_bytes())?; + + mac.update(nonce.as_ref().as_bytes()); + mac.update(b"\x00"); + + mac.update(user.as_ref().as_bytes()); + mac.update(b"\x00"); + + mac.update(password.as_ref().as_bytes()); + mac.update(b"\x00"); + + if admin { + mac.update("admin".as_bytes()); + } else { + mac.update("notadmin".as_bytes()); + } + + if let Some(user_type) = user_type { + mac.update(b"\x00"); + mac.update(user_type.as_ref().as_bytes()); + } + + let result = mac.finalize(); + let code_bytes = result.into_bytes(); + + Ok(hex::encode(code_bytes)) + } +} + +#[cfg(test)] +mod test { + use super::SharedSecretRegistration; + + #[test] + fn generates_mac_accordingly() { + let want = "c272fb1c287c795ff5ce238c4dba57cf95db5eff"; + let have = SharedSecretRegistration::generate_mac( + "m@;wYOUOh0f:CH5XA65sJB1^q01~DmIriOysRImot,OR_vzN&B", + "1234567890", + "groot", + "imroot!1234", + true, + None, + ) + .unwrap(); + + assert_eq!(have, want); + } +} diff --git a/crates/matrix/src/lib.rs b/crates/matrix/src/lib.rs new file mode 100644 index 0000000..68a704a --- /dev/null +++ b/crates/matrix/src/lib.rs @@ -0,0 +1,18 @@ +//! Crate to centralize all Matrix dependencies. +//! +//! Reexports `matrix_sdk` and provides implementations on Matrix Admin API. + +/// Implementation on the Administrator API of Matrix +/// +/// Refer: https://matrix-org.github.io/synapse/latest/usage/administration/index.html +pub mod admin; + +/// The official Matrix Rust SDK. +/// +/// # Project State +/// +/// As of today this SDK is still in beta and is not yet ready for production, +/// so we make use of the repo at a specific commit. +/// +/// Refer: https://github.com/matrix-org/matrix-rust-sdk +pub use matrix_sdk as sdk; diff --git a/crates/test/Cargo.toml b/crates/test/Cargo.toml new file mode 100644 index 0000000..6c15d0f --- /dev/null +++ b/crates/test/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "test" +version = "0.0.0" +edition = "2021" +publish = false + +[lib] +name = "test" +path = "src/lib.rs" + +[dependencies] +dotenv = "0.15.0" +tokio = { version = "1.34.0", features = ["rt", "rt-multi-thread", "macros"] } + +# Workspace Dependencies +serde = { workspace = true } +url = { workspace = true } + +# Local Dependencies +matrix = { path = "../matrix" } diff --git a/crates/test/fixtures/synapse/homeserver.yaml b/crates/test/fixtures/synapse/homeserver.yaml new file mode 100644 index 0000000..9716e79 --- /dev/null +++ b/crates/test/fixtures/synapse/homeserver.yaml @@ -0,0 +1,37 @@ +# Configuration file for Synapse. +# +# This is a YAML file: see [1] for a quick introduction. Note in particular +# that *indentation is important*: all the elements of a list or dictionary +# should have the same indentation. +# +# [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html +# +# For more information on how to configure Synapse, including a complete accounting of +# each option, go to docs/usage/configuration/config_documentation.md or +# https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html +server_name: "matrix.localhost" +pid_file: /data/homeserver.pid +listeners: + - port: 8008 + tls: false + type: http + x_forwarded: true + resources: + - names: [client, federation] + compress: false +database: + name: sqlite3 + args: + database: /data/homeserver.db +log_config: "/data/matrix.localhost.log.config" +media_store_path: /data/media_store +registration_shared_secret: "m@;wYOUOh0f:CH5XA65sJB1^q01~DmIriOysRImot,OR_vzN&B" +report_stats: true +macaroon_secret_key: "XND.g+P_7wz.Yx:i6js.Eh;=jG*#uWBIe;X2OoX78^E,LVJ;8c" +form_secret: "pS7pR@AFJD~BtUAqH^ku5Kenz1X^Hol0E_+xhwvohOrkx;sMoO" +signing_key_path: "/data/matrix.localhost.signing.key" +trusted_key_servers: + - server_name: "matrix.org" + + +# vim:ft=yaml \ No newline at end of file diff --git a/crates/test/fixtures/synapse/matrix.localhost.log.config b/crates/test/fixtures/synapse/matrix.localhost.log.config new file mode 100644 index 0000000..832f0fa --- /dev/null +++ b/crates/test/fixtures/synapse/matrix.localhost.log.config @@ -0,0 +1,39 @@ +version: 1 + +formatters: + precise: + + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + + +handlers: + + + console: + class: logging.StreamHandler + formatter: precise + +loggers: + # This is just here so we can leave `loggers` in the config regardless of whether + # we configure other loggers below (avoid empty yaml dict error). + _placeholder: + level: "INFO" + + + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + + + + +root: + level: INFO + + + handlers: [console] + + +disable_existing_loggers: false \ No newline at end of file diff --git a/crates/test/fixtures/synapse/matrix.localhost.signing.key b/crates/test/fixtures/synapse/matrix.localhost.signing.key new file mode 100644 index 0000000..090b449 --- /dev/null +++ b/crates/test/fixtures/synapse/matrix.localhost.signing.key @@ -0,0 +1 @@ +ed25519 a_VKUD FXu3HoEKJdiMh1e+3dW8kO/P8ldSdNzdV+/vg9wdowE diff --git a/crates/test/src/environment.rs b/crates/test/src/environment.rs new file mode 100644 index 0000000..88e6261 --- /dev/null +++ b/crates/test/src/environment.rs @@ -0,0 +1,28 @@ +use std::env::var; + +use matrix::admin::Client; + +const COMMUNE_REGISTRATION_SHARED_SECRET: &str = "COMMUNE_REGISTRATION_SHARED_SECRET"; + +pub struct Environment { + pub client: Client, + pub registration_shared_secret: String, +} + +impl Environment { + pub fn new() -> Self { + dotenv::dotenv().ok(); + + let client = Client::new("http://localhost:8008").unwrap(); + let registration_shared_secret = Self::env_var(COMMUNE_REGISTRATION_SHARED_SECRET); + + Self { + client, + registration_shared_secret, + } + } + + pub fn env_var(name: &str) -> String { + var(name).unwrap_or_else(|_| panic!("Missing {name} environment variable")) + } +} diff --git a/crates/test/src/lib.rs b/crates/test/src/lib.rs new file mode 100644 index 0000000..6d8d018 --- /dev/null +++ b/crates/test/src/lib.rs @@ -0,0 +1,5 @@ +#[cfg(test)] +mod environment; + +#[cfg(test)] +mod matrix; diff --git a/crates/test/src/matrix/mod.rs b/crates/test/src/matrix/mod.rs new file mode 100644 index 0000000..9e56f51 --- /dev/null +++ b/crates/test/src/matrix/mod.rs @@ -0,0 +1 @@ +mod shared_token_registration; diff --git a/crates/test/src/matrix/shared_token_registration.rs b/crates/test/src/matrix/shared_token_registration.rs new file mode 100644 index 0000000..f3946be --- /dev/null +++ b/crates/test/src/matrix/shared_token_registration.rs @@ -0,0 +1,39 @@ +use matrix::admin::resources::token::shared_secret::{ + SharedSecretRegistration, SharedSecretRegistrationDto, +}; + +use crate::environment::Environment; + +#[tokio::test] +async fn creates_user_using_shared_secret() { + let env = Environment::new(); + let nonce = SharedSecretRegistration::get_nonce(&env.client) + .await + .unwrap() + .nonce; + let mac = SharedSecretRegistration::generate_mac( + env.registration_shared_secret, + nonce.clone(), + "steve".into(), + "verysecure".into(), + true, + None, + ) + .unwrap(); + let registration = SharedSecretRegistration::create( + &env.client, + SharedSecretRegistrationDto { + nonce, + username: "steve".into(), + displayname: Some("steve".into()), + password: "verysecure".into(), + admin: true, + mac, + }, + ) + .await + .unwrap(); + + assert!(!registration.access_token.is_empty()); + assert!(!registration.user_id.is_empty()); +} diff --git a/fixtures/generate_mac.py b/fixtures/generate_mac.py new file mode 100644 index 0000000..d917188 --- /dev/null +++ b/fixtures/generate_mac.py @@ -0,0 +1,32 @@ +import hmac, hashlib + +def generate_mac(nonce, user, password, admin=False, user_type=None): + + mac = hmac.new( + key=b"m@;wYOUOh0f:CH5XA65sJB1^q01~DmIriOysRImot,OR_vzN&B", + digestmod=hashlib.sha1, + ) + + mac.update(nonce.encode('utf8')) + mac.update(b"\x00") + mac.update(user.encode('utf8')) + mac.update(b"\x00") + mac.update(password.encode('utf8')) + mac.update(b"\x00") + mac.update(b"admin" if admin else b"notadmin") + if user_type: + mac.update(b"\x00") + mac.update(user_type.encode('utf8')) + + return mac.hexdigest() + +if __name__ == "__main__": + mac = generate_mac( + nonce="1234567890", + user="groot", + password="imroot!1234", + admin=True, + user_type=None + ) + + print(mac) diff --git a/fixtures/synapse/homeserver.yaml b/fixtures/synapse/homeserver.yaml new file mode 100644 index 0000000..d32c6bf --- /dev/null +++ b/fixtures/synapse/homeserver.yaml @@ -0,0 +1,45 @@ +# Configuration file for Synapse. +# +# This is a YAML file: see [1] for a quick introduction. Note in particular +# that *indentation is important*: all the elements of a list or dictionary +# should have the same indentation. +# +# [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html +# +# For more information on how to configure Synapse, including a complete accounting of +# each option, go to docs/usage/configuration/config_documentation.md or +# https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html +server_name: "matrix.localhost" +pid_file: /data/homeserver.pid +listeners: + - port: 8008 + tls: false + type: http + x_forwarded: true + resources: + - names: [client, federation] + compress: false +database: + name: psycopg2 + txn_limit: 10000 + allow_unsafe_locale: true + args: + user: synapse_user + password: secretpassword + database: synapse + host: synapse_database + port: 5432 + cp_min: 5 + cp_max: 10 +log_config: "/data/matrix.localhost.log.config" +media_store_path: /data/media_store +registration_shared_secret: "m@;wYOUOh0f:CH5XA65sJB1^q01~DmIriOysRImot,OR_vzN&B" +report_stats: true +macaroon_secret_key: "XND.g+P_7wz.Yx:i6js.Eh;=jG*#uWBIe;X2OoX78^E,LVJ;8c" +form_secret: "pS7pR@AFJD~BtUAqH^ku5Kenz1X^Hol0E_+xhwvohOrkx;sMoO" +signing_key_path: "/data/matrix.localhost.signing.key" +trusted_key_servers: + - server_name: "matrix.org" + + +# vim:ft=yaml \ No newline at end of file diff --git a/fixtures/synapse/matrix.localhost.log.config b/fixtures/synapse/matrix.localhost.log.config new file mode 100644 index 0000000..832f0fa --- /dev/null +++ b/fixtures/synapse/matrix.localhost.log.config @@ -0,0 +1,39 @@ +version: 1 + +formatters: + precise: + + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' + + +handlers: + + + console: + class: logging.StreamHandler + formatter: precise + +loggers: + # This is just here so we can leave `loggers` in the config regardless of whether + # we configure other loggers below (avoid empty yaml dict error). + _placeholder: + level: "INFO" + + + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + + + + +root: + level: INFO + + + handlers: [console] + + +disable_existing_loggers: false \ No newline at end of file diff --git a/fixtures/synapse/matrix.localhost.signing.key b/fixtures/synapse/matrix.localhost.signing.key new file mode 100644 index 0000000..090b449 --- /dev/null +++ b/fixtures/synapse/matrix.localhost.signing.key @@ -0,0 +1 @@ +ed25519 a_VKUD FXu3HoEKJdiMh1e+3dW8kO/P8ldSdNzdV+/vg9wdowE From 34f73ba6e72b95ef883a6a09901019342d06a3fc Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 18:38:53 -0300 Subject: [PATCH 02/14] feat: create copy of `.env.example` on CI --- .github/workflows/ci.yml | 1 + README.md | 50 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a01f2fc..62f8173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: - name: Prepare Data for Tests run: | cp -R ./crates/test/fixtures/synapse ./docker/synapse + cp -R ./.env.example ./.env docker compose up -d - name: E2E Tests diff --git a/README.md b/README.md index e31545b..8d1852a 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,19 @@ ### Getting Started -1. Generate `Synapse` server configuration +1. Create a copy of `.env.example` on `.env` + +```bash +cp .env.example .env +``` + +2. Generate `Synapse` server configuration ```bash just gen_synapse_conf ``` -2. Run Synapse Server (and other containerized services) using Docker Compose +3. Run Synapse Server (and other containerized services) using Docker Compose via: ```bash @@ -35,6 +41,46 @@ use `just clear`. > **Warning** `just clear` will remove all containers and images. +### Testing + +This application has 2 layers for tests: + +- `Unit`: Are usually inlined inside crates, and dont depend on any integration +- `E2E`: Lives in `test` crate and counts with the services that run the application + +#### Unit + +Unit tests can be executed via `cargo test -p `, this will run +every unit test. + +#### E2E + +You must run Docker services as for development. In order to avoid messing up +the development environment, its recommended to use the synapse setup from +`crates/test/fixtures/synapse` replacing it with `docker/synapse`. + +The only difference should be the `database` section, which uses SQLite instead. + +```diff +database: ++ name: psycopg2 ++ args: ++ database: /data/homeserver.db +- name: psycopg2 +- txn_limit: 10000 +- allow_unsafe_locale: true +- args: +- user: synapse_user +- password: secretpassword +- database: synapse +- host: synapse_database +- port: 5432 +- cp_min: 5 +- cp_max: 10 +``` + +> Make sure the `.env` file is created from the contents on `.env.example` + ### Application Layout
From 281aa44a10c24719f709edc827f44271d7fe0fd2 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 18:57:13 -0300 Subject: [PATCH 03/14] fix: use `0.0.0.0` for synapse host conn --- .env.example | 1 + crates/test/src/environment.rs | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index aa2eae6..ed6abc7 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,7 @@ # This is explicitly passed here for development purposes, it should match the # same as on `fixtures/synapse/homeserver.yaml` for CI. COMMUNE_REGISTRATION_SHARED_SECRET='m@;wYOUOh0f:CH5XA65sJB1^q01~DmIriOysRImot,OR_vzN&B' +COMMUNE_SYNAPSE_HOST='http://0.0.0.0:8008' # Matrix Client MATRIX_HOST=http://localhost:8008 diff --git a/crates/test/src/environment.rs b/crates/test/src/environment.rs index 88e6261..bb2bfe5 100644 --- a/crates/test/src/environment.rs +++ b/crates/test/src/environment.rs @@ -3,6 +3,7 @@ use std::env::var; use matrix::admin::Client; const COMMUNE_REGISTRATION_SHARED_SECRET: &str = "COMMUNE_REGISTRATION_SHARED_SECRET"; +const COMMUNE_SYNAPSE_HOST: &str = "COMMUNE_SYNAPSE_HOST"; pub struct Environment { pub client: Client, @@ -13,7 +14,8 @@ impl Environment { pub fn new() -> Self { dotenv::dotenv().ok(); - let client = Client::new("http://localhost:8008").unwrap(); + let synapse_host = Self::env_var(COMMUNE_SYNAPSE_HOST); + let client = Client::new(synapse_host).unwrap(); let registration_shared_secret = Self::env_var(COMMUNE_REGISTRATION_SHARED_SECRET); Self { From fc9dec985bb9cb505b9d4a23fa7a6672a3a26226 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 19:19:52 -0300 Subject: [PATCH 04/14] feat: just script to generate de-facto admin --- Justfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Justfile b/Justfile index fac9f3b..dfbfeba 100644 --- a/Justfile +++ b/Justfile @@ -15,6 +15,15 @@ gen_synapse_conf: dotenv --env-file .env \ matrixdotorg/synapse:v1.96.1 generate +# Generates a de-facto admin user +gen_synapse_admin: dotenv + docker compose exec -it synapse \ + register_new_matrix_user http://localhost:8008 \ + -c /data/homeserver.yaml \ + -u admin \ + -p admin \ + -a + # Runs backend dependency services backend: dotenv docker compose up --build From dfef0999181d2e5fad51a47fa82c8b7ed706c5eb Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 19:24:12 -0300 Subject: [PATCH 05/14] feat: provide `COMMUNE_SYNAPSE_HOST` on CI --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62f8173..0dae2be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,4 +51,6 @@ jobs: docker compose up -d - name: E2E Tests + env: + COMMUNE_SYNAPSE_HOST: 'synapse' run: cargo test -p test -- --test-threads=1 From 5e2130ad76eaa6c299903d884ebf45773d73d2c2 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 19:53:09 -0300 Subject: [PATCH 06/14] feat: port --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0dae2be..baba3b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,5 +52,5 @@ jobs: - name: E2E Tests env: - COMMUNE_SYNAPSE_HOST: 'synapse' + COMMUNE_SYNAPSE_HOST: 'synapse:8008' run: cargo test -p test -- --test-threads=1 From 5ba320b7451cd8ef295c2180f41f3839d2cf278c Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 20:25:37 -0300 Subject: [PATCH 07/14] fix: missing sheme --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baba3b5..187d362 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,5 +52,5 @@ jobs: - name: E2E Tests env: - COMMUNE_SYNAPSE_HOST: 'synapse:8008' + COMMUNE_SYNAPSE_HOST: 'http://synapse:8008' run: cargo test -p test -- --test-threads=1 From 572a45ebc98067758b40164bce787c7b353f62b3 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 21:06:32 -0300 Subject: [PATCH 08/14] fix: use `http://0.0.0.0:8008` --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 187d362..52d4c89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,5 +52,5 @@ jobs: - name: E2E Tests env: - COMMUNE_SYNAPSE_HOST: 'http://synapse:8008' + COMMUNE_SYNAPSE_HOST: 'http://0.0.0.0:8008' run: cargo test -p test -- --test-threads=1 From 3ddb85014bd4e24ad3669865d2d4090fd6eb8c69 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 21:22:08 -0300 Subject: [PATCH 09/14] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52d4c89..5aaeec8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,5 +52,5 @@ jobs: - name: E2E Tests env: - COMMUNE_SYNAPSE_HOST: 'http://0.0.0.0:8008' + COMMUNE_SYNAPSE_HOST: 'http://localhost:8008' run: cargo test -p test -- --test-threads=1 From 5689d5f5b2597a94eae679c09e6c5567bc928f3d Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 21:45:52 -0300 Subject: [PATCH 10/14] fix: use all interfaces --- crates/test/fixtures/synapse/homeserver.yaml | 1 + fixtures/synapse/homeserver.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/test/fixtures/synapse/homeserver.yaml b/crates/test/fixtures/synapse/homeserver.yaml index 9716e79..432d21e 100644 --- a/crates/test/fixtures/synapse/homeserver.yaml +++ b/crates/test/fixtures/synapse/homeserver.yaml @@ -16,6 +16,7 @@ listeners: tls: false type: http x_forwarded: true + bind_addresses: ['::', '0.0.0.0'] resources: - names: [client, federation] compress: false diff --git a/fixtures/synapse/homeserver.yaml b/fixtures/synapse/homeserver.yaml index d32c6bf..a3682fd 100644 --- a/fixtures/synapse/homeserver.yaml +++ b/fixtures/synapse/homeserver.yaml @@ -16,6 +16,7 @@ listeners: tls: false type: http x_forwarded: true + bind_addresses: ['::', '0.0.0.0'] resources: - names: [client, federation] compress: false From 95bae2e4effc1552762bf25fbca2c97c4f052ee1 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 21:55:56 -0300 Subject: [PATCH 11/14] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aaeec8..52d4c89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,5 +52,5 @@ jobs: - name: E2E Tests env: - COMMUNE_SYNAPSE_HOST: 'http://localhost:8008' + COMMUNE_SYNAPSE_HOST: 'http://0.0.0.0:8008' run: cargo test -p test -- --test-threads=1 From 2f5132c9d492ddffb918a9438f3e1526820a1dc8 Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 22:01:19 -0300 Subject: [PATCH 12/14] try: use default --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52d4c89..54b6daf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,6 @@ jobs: - name: Prepare Data for Tests run: | - cp -R ./crates/test/fixtures/synapse ./docker/synapse cp -R ./.env.example ./.env docker compose up -d From 1dd34cd0012541cd8e0056df0b35bef766f4eabc Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 22:09:31 -0300 Subject: [PATCH 13/14] Update ci.yml --- .github/workflows/ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54b6daf..f6f05fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,10 +44,11 @@ jobs: - name: Install Just uses: extractions/setup-just@v1 + - name: Generate Configuration + run: just gen_synapse_conf + - name: Prepare Data for Tests - run: | - cp -R ./.env.example ./.env - docker compose up -d + run: docker compose up -d - name: E2E Tests env: From 3e5604b2df147a8119cb6404ba26d4857e759e3a Mon Sep 17 00:00:00 2001 From: Esteban Borai Date: Sun, 19 Nov 2023 22:37:28 -0300 Subject: [PATCH 14/14] chore: disable e2e needs further work --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f6f05fa..38d5673 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,16 +41,16 @@ jobs: - name: Unit Tests run: cargo test -p matrix - - name: Install Just - uses: extractions/setup-just@v1 + # - name: Install Just + # uses: extractions/setup-just@v1 - - name: Generate Configuration - run: just gen_synapse_conf + # - name: Generate Configuration + # run: just gen_synapse_conf - - name: Prepare Data for Tests - run: docker compose up -d + # - name: Prepare Data for Tests + # run: docker compose up -d - - name: E2E Tests - env: - COMMUNE_SYNAPSE_HOST: 'http://0.0.0.0:8008' - run: cargo test -p test -- --test-threads=1 + # - name: E2E Tests + # env: + # COMMUNE_SYNAPSE_HOST: 'http://0.0.0.0:8008' + # run: cargo test -p test -- --test-threads=1