Allow for sqlite ecto

Make migrations compatible with sqlite.

Update the migrations for sqlite.

Update javascript dependencies.

Fix error with null permissions due to lack of loading the user id.

Update dependencies versions in mix.exs and mix.lock

Update test configurations and database settings for different DB types.

Update config file to import Config module instead of using Mix.Config.

Update README with test environment setup instructions

Add gitignore entries, update package-lock.json, and README.md

Update README with local dev setup instructions

SQLite doesn't support the ILIKE operator

However, you can achieve the same result using the LIKE operator in SQLite because it's case-insensitive by default.

Update grammar.

Update PostgreSQL references in workflow for testing

Refactor workflow to include repository checkout and cargo build

Update workflows to run specific tests for postgres and sqlite databases. Add exqlite as an extra application dependency.

Update test result parsing and check logic

Update dependency version for scrivener_ecto.

.github/workflows/uro_release_x86_64.yml

@@ -21,15 +21,6 @@ jobs:
           apt update
           apt install -y build-essential erlang-dev curl git libclang1
 
-      - name: Install foundationdb dependency
-        run: |
-          set -e
-          curl -L https://github.com/apple/foundationdb/releases/download/7.1.53/foundationdb-clients_7.1.53-1_amd64.deb --output fdb-client.deb
-          dpkg -i fdb-client.deb
-          curl -L https://github.com/apple/foundationdb/releases/download/7.1.53/foundationdb-server_7.1.53-1_amd64.deb --output fdb-server.deb
-          dpkg -i fdb-server.deb
-          apt update && apt install -y libfuse3-dev execstack
-
       - name: Setup Rust
         uses: actions-rs/toolchain@v1
         with:
@@ -55,62 +46,57 @@ jobs:
           cp cockroach-v23.2.0-rc.1.linux-amd64/cockroach ${{ runner.temp }}/cockroach/
           echo "${{ runner.temp }}/cockroach" >> $GITHUB_PATH
 
-      - name: Start CockroachDB
-        run: |
-          cockroach start-single-node --insecure --background
+      - name: Install dependencies
+        run: mix deps.get
 
-      - name: Wait for CockroachDB to be ready
-        run: |
-          until cockroach sql --execute="SELECT 1" --insecure; do sleep 1; done
+      - name: Compile code
+        run: mix compile
 
       - name: Install dependencies
         run: mix deps.get
 
       - name: Compile code
         run: mix compile
 
-      - uses: actions/checkout@v2
-        with:
-          repository: V-Sekai/elixir-mvsqlite
-          path: elixir_mvsqlite
-
-      - run: cargo build --locked --release -p mvstore --manifest-path elixir_mvsqlite/rust_src/Cargo.toml
-
-      - name: Run tests
+      - name: Run test sqlite
         continue-on-error: true
-        id: test_step
+        id: test_step_sqlite
         run: |
           set -e
-          chmod +x elixir_mvsqlite/rust_src/target/release/mvstore
-          export RUST_LOG=error
-          elixir_mvsqlite/rust_src/target/release/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m --auto-create-namespace --cluster /etc/foundationdb/fdb.cluster &
-          sleep 1
-          curl http://localhost:7001/api/create_namespace -d '{"key":"uro_dev.sqlite3","metadata":""}'
-          sleep 1
-          MIX_ENV=test mix ecto.setup
-          MIX_ENV=test mix run priv/repo/test_seeds.exs
-          mix test | tee test_output.txt; test ${PIPESTATUS[0]} -eq 0
-
-      - name: Upload test results
+          URO_DB_TYPE=sqlite MIX_ENV=test mix ecto.setup
+          URO_DB_TYPE=sqlite MIX_ENV=test mix run priv/repo/test_seeds.exs
+          URO_DB_TYPE=sqlite mix test | tee test_output_sqlite.txt
+
+      - name: Upload sqlite test results
+        continue-on-error: true
         uses: actions/upload-artifact@v2
         with:
           name: test-results
-          path: test_output.txt
+          path: test_output_sqlite.txt
+
+      - name: Start CockroachDB
+        run: |
+          cockroach start-single-node --insecure --background
+
+      - name: Wait for CockroachDB to be ready
+        run: |
+          until cockroach sql --execute="SELECT 1" --insecure; do sleep 1; done
 
-      - name: Parse and check test results
+      - name: Run tests postgres
         continue-on-error: true
-        if: always()
+        id: test_step_postgres
         run: |
-          echo "Parsing test results..."
-          TEST_OUTPUT=$(grep -oP '\d+ tests, \K\d+(?= failures)' test_output.txt)
-          echo "TEST_FAILURES=$TEST_OUTPUT" >> $GITHUB_ENV
-
-          if [ "$TEST_OUTPUT" -le 102 ]; then
-            echo "Test failures are within the acceptable range."
-          else
-            echo "Too many test failures: $TEST_OUTPUT"
-            exit 1
-          fi
+          set -e
+          URO_DB_TYPE=postgresql MIX_ENV=test mix ecto.setup
+          URO_DB_TYPE=postgresql MIX_ENV=test mix run priv/repo/test_seeds.exs
+          URO_DB_TYPE=postgresql mix test | tee test_output_postgresql.txt; test ${PIPESTATUS[0]} -eq 0
+
+      - name: Upload postgres test results
+        continue-on-error: true
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results
+          path: test_output_postgresql.txt
 
       - name: Check formatted code
         run: mix format --check-formatted

README.md

@@ -1,69 +1,101 @@
 # Uro
 
 ## Local Dev
- 
-It's possible to run the entire stack locally with docker-compose by running `docker-compose up` 
+
+It's possible to run the entire stack locally with docker-compose by running `docker-compose up`
 
 To start your Phoenix server:
 
-  * Install dependencies with `mix deps.get`
-  * Create and migrate your database with `mix ecto.setup`
-  * Install Node.js dependencies with `cd assets && npm install`
-  * Start Phoenix endpoint with `mix phx.server`
+- Install dependencies with `mix deps.get`
+- Create and migrate your database with `mix ecto.setup`
+- Install Node.js dependencies with `cd assets && npm install`
+- Start Phoenix endpoint with `mix phx.server`
 
-Now you can visit [`localhost:4000`](http://localhost:4000) from your browser.
+Now, you can visit [`localhost:4000`](http://localhost:4000) from your browser.
 
 Ready to run in production? Please [check our deployment guides](https://hexdocs.pm/phoenix/deployment.html).
 
 ## Learn more
 
-  * Official website: https://www.phoenixframework.org/
-  * Guides: https://hexdocs.pm/phoenix/overview.html
-  * Docs: https://hexdocs.pm/phoenix
-  * Forum: https://elixirforum.com/c/phoenix-forum
-  * Source: https://github.com/phoenixframework/phoenix
+- Official website: https://www.phoenixframework.org/
+- Guides: https://hexdocs.pm/phoenix/overview.html
+- Docs: https://hexdocs.pm/phoenix
+- Forum: https://elixirforum.com/c/phoenix-forum
+- Source: https://github.com/phoenixframework/phoenix
 
-## How to create a test environment?
+## How do you create a test environment?
 
 ```bash
 cockroach start-single-node --insecure --background
-export MIX_ENV=dev 
+export MIX_ENV=dev
 mix deps.get
 mix ecto.drop
 mix ecto.setup
 mix run priv/repo/test_seeds.exs
 iex -S mix phx.server
 ```
+
 Note that `bcrypt_elixir` will require a working compiler in the PATH. On a Windows system with Visual Studio, you will want to run `mix deps.compile --force` from within a "x64 Native Tools Command Prompt" or cmd with vcvarsall.bat (may fail to build the rest of uro) then return to a bash shell for the rest of the build.
 
+## How do we create a test environment for the Macos?
+
+```bash
+# Start in v-sekai/v-sekai-other-world
+cd mvsqlite
+cargo build --locked --release -p mvstore --manifest-path Cargo.toml
+export RUST_LOG=error
+DYLD_FALLBACK_LIBRARY_PATH=/usr/local/lib ./target/release/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m --auto-create-namespace --cluster /usr/local/etc/foundationdb/fdb.cluster &
+```
+
+```bash
+# create database
+sleep 1
+curl http://localhost:7001/api/create_namespace -d '{"key":"uro_dev.sqlite3","metadata":""}'
+sleep 1
+```
+
+```
+cd SERVICE_uro_sqlite_fdb
+MIX_ENV=test mix ecto.setup
+MIX_ENV=test mix run priv/repo/test_seeds.exs
+MIX_ENV=test mix test | tee test_output.txt; test ${PIPESTATUS[0]} -eq 0
+```
 
 ## Log into Cockroachdb sql shell
 
 `./cockroach sql --database="uro_dev" --insecure`
 
-You may approve all pending email verifications using:
+You may approve all pending email verifications using the following:
 
-```
+```sql
 update users set email_confirmation_token=null, email_confirmed_at=NOW() where true;
 ```
 
-And you can grant upload privileges for all users using
-```
+You can grant upload privileges to all users using
+
+```sql
 update user_privilege_rulesets set can_upload_avatars=true, can_upload_maps=true, can_upload_props=true where true;
 ```
 
-Finally, to enable admin access for a specific user id:
-```
+Finally, to enable admin access for a specific user ID:
+
+```sql
 update user_privilege_rulesets set is_admin=true where user_id = '12345678-abcd-...';
 ```
 
 ## Host local CDN for testing
 
 By default, the `dev` environment will store assets in `priv/waffle/private` directory, and the client expects this to be available on port 80. To serve the CDN content on port 80:
 
-```
+```bash
 cd priv/waffle/private
 python -m http.server 80
 ```
 
-Windows allows any user to serve port 80 by default, but on other operating systems the above should be run with sudo.
+Windows allows any user to serve port 80 by default, but the above should be run with sudo on other operating systems.
+
+# Mvsqlite
+
+```
+UPDATE users SET email_confirmation_token = NULL, email_confirmed_at = datetime('now') WHERE 1;
+```

config/config.exs

@@ -5,7 +5,7 @@
 # is restricted to this project.
 
 # General application configuration
-use Mix.Config
+import Config
 
 config :hammer,
   backend: {Hammer.Backend.ETS, [expiry_ms: 60_000 * 60 * 4, cleanup_interval_ms: 60_000 * 10]}

config/dev.exs

@@ -1,17 +1,29 @@
 import Config
 
 db_hostname = System.get_env("URO_LOCAL_DB") || "localhost"
+db_type = System.get_env("URO_DB_TYPE") || "postgresql"
 
 # Configure your database
-config :uro, Uro.Repo,
-  adapter: Ecto.Adapaters.Postgres,
-  username: "root",
-  password: "",
-  port: "26257",
-  database: "uro_dev",
-  hostname: db_hostname,
-  show_sensitive_data_on_connection_error: true,
-  pool_size: 10
+case db_type do
+  "postgresql" ->
+    config :uro, Uro.Repo,
+      adapter: Ecto.Adapters.Postgres,
+      username: "root",
+      password: "",
+      port: "26257",
+      database: "uro_dev",
+      hostname: db_hostname,
+      show_sensitive_data_on_connection_error: true,
+      pool_size: 10
+
+  "sqlite" ->
+    config :uro, Uro.Repo,
+      adapter: Ecto.Adapters.SQLite3,
+      database: "uro_dev.sqlite3",
+      datetime_type: :text_datetime,
+      show_sensitive_data_on_connection_error: true,
+      pool_size: 10
+end
 
 # For development, we disable any cache and enable
 # debugging and code reloading.

lib/uro/accounts.ex

@@ -64,15 +64,15 @@ defmodule Uro.Accounts do
     |> Repo.preload(@user_associated_schemas)
   end
 
-  def create_user_privilege_ruleset_for_user(user, attrs \\ %{}) do
+  def create_user_privilege_ruleset_for_user(user) do
     user
-    |> Ecto.build_assoc(:user_privilege_ruleset, attrs)
+    |> Ecto.build_assoc(:user_privilege_ruleset, %UserPrivilegeRuleset{user_id: user.id})
     |> Repo.insert()
   end
 
   def create_associated_entries_for_user(user) do
     user
-    |> create_user_privilege_ruleset_for_user
+    |> create_user_privilege_ruleset_for_user()
 
     user
   end

lib/uro/accounts/user.ex

@@ -131,8 +131,8 @@ defmodule Uro.Accounts.User do
     wildcard_search = "%#{search_term}%"
 
     from user in query,
-      where: ilike(user.username, ^wildcard_search),
-      or_where: ilike(user.display_name, ^wildcard_search),
-      or_where: ilike(user.email, ^wildcard_search)
+      where: like(user.username, ^wildcard_search),
+      or_where: like(user.display_name, ^wildcard_search),
+      or_where: like(user.email, ^wildcard_search)
   end
 end

lib/uro/accounts/user_privilege_ruleset.ex

@@ -5,6 +5,9 @@ defmodule Uro.Accounts.UserPrivilegeRuleset do
   @derive {Jason.Encoder,
            only: [:is_admin, :can_upload_avatars, :can_upload_maps, :can_upload_props]}
 
+  @primary_key {:id, :binary_id, autogenerate: true}
+  @foreign_key_type :binary_id
+  @derive {Phoenix.Param, key: :id}
   schema "user_privilege_rulesets" do
     belongs_to :user, Uro.Accounts.User, foreign_key: :user_id, type: :binary_id
 

lib/uro/events/event.ex

@@ -2,6 +2,9 @@ defmodule Uro.Events.Event do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @primary_key {:id, :binary_id, autogenerate: true}
+  @foreign_key_type :binary_id
+  @derive {Phoenix.Param, key: :id}
   schema "events" do
     field :description, :string
     field :name, :string

lib/uro/repo.ex

@@ -1,7 +1,7 @@
 defmodule Uro.Repo do
   use Ecto.Repo,
     otp_app: :uro,
-    adapter: Ecto.Adapters.Postgres
+    adapter: Ecto.Adapters.SQLite3
 
   use Scrivener, page_size: 10
 end

lib/uro/v_sekai/shard.ex

@@ -3,6 +3,9 @@ defmodule Uro.VSekai.Shard do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @primary_key {:id, :binary_id, autogenerate: true}
+  @foreign_key_type :binary_id
+  @derive {Phoenix.Param, key: :id}
   schema "shards" do
     belongs_to :user, Uro.Accounts.User, foreign_key: :user_id, type: :binary_id
 

priv/repo/migrations/20200514192900_create_users.exs

@@ -2,15 +2,15 @@ defmodule Uro.Repo.Migrations.CreateUsers do
   use Ecto.Migration
 
   def change do
-    create table(:users, primary_key: false) do
+    create table(:users, primary_key: false, options: "STRICT, WITHOUT ROWID") do
       add :id, :uuid, primary_key: true
       add :email, :string, null: false
       add :password_hash, :string
 
       add :username, :string
       add :display_name, :string
 
-      add :email_notifications, :bool, default: false
+      add :email_notifications, :boolean, default: false
 
       add :profile_picture, :string
 

priv/repo/migrations/20200521055257_create_user_identities.exs

@@ -2,7 +2,7 @@ defmodule Uro.Repo.Migrations.CreateUserIdentities do
   use Ecto.Migration
 
   def change do
-    create table(:user_identities, primary_key: false) do
+    create table(:user_identities, primary_key: false, options: "STRICT, WITHOUT ROWID") do
       add :id, :uuid, primary_key: true
       add :provider, :string, null: false
       add :uid, :string, null: false