add docs and remove unused code

.dockerignore

@@ -3,4 +3,5 @@
 
 !project.clj
 !src/
+!test/
 !resources/

.github/workflows/deploy.yml

@@ -44,7 +44,8 @@ jobs:
     - name: Login to Artifact Registry
       uses: docker/login-action@v1
       with:
-        registry: "asia-northeast2-docker.pkg.dev" ## TODO env var
+        # registry: "asia-northeast2-docker.pkg.dev" ## TODO env var
+        registry: gcr.io
         username: oauth2accesstoken
         password: ${{ steps.auth.outputs.access_token }}
 
@@ -53,7 +54,7 @@ jobs:
       run: |-
         docker build ./ \
           --tag "$GAR_REGION-docker.pkg.dev/$PROJECT_ID/$REPOSITORY/$IMAGE:$GITHUB_SHA"
-
+    # TODO run tests with built image. Add needs:test to push command
     - id: docker-push-tagged
       name: Tag Docker image and push to Google Artifact Registry
       uses: docker/build-push-action@v2
@@ -62,7 +63,6 @@ jobs:
         tags: |
           ${{ steps.get-repo.outputs.repo }}:${{ steps.get-tag.outputs.tag }}
           ${{ steps.get-repo.outputs.repo }}:latest
-      
-      # TODO run tests. If master/staging branch then tag as prod/test and deploy to replace images
-      # access Compute Registry to reboot server or whatever based of container
-      # gcloud compute instances update-container
+      # TODO run tests with built image. Add needs:test to push command
+      # get-instance: echo ::set-output name=name:: if master master-djinn elif staging master-djinn-apprentice fi
+      # gcloud compute instances update-container  ${{ steps.get-instance.outputs.name }}

deployments.md

@@ -1,7 +1,9 @@
 ## Step 0 - Installs
 - gcloud CLI on local machine - `curl https://sdk.cloud.google.com | zsh`
+- `gcloud auth login` then `gcloud auth configure-docker {artifact-registry-zone.pkg.dev}`
 - [docker](https://docs.docker.com/engine/install/ubuntu/) on remote server 
 
+
 ## Locally
 ```
 source .env && 
@@ -12,9 +14,11 @@ docker-compose up
 ## Step 2 - Update
 `git push` OR `gcloud `
 
-## Step 3 - Deploying To new remote Server
+
+## Step 3 - Deploying To new GCE server
 Ensure you have installed docker on your server first.
 - Docker container must be mapped to port 80/443 on server
+- enable `Allow full access to all Cloud APIs` in IAM otherwise cant read from Artifact Rgistry even if you give it a custom role with access.
 - 
 ```
 touch .env docker-compose.yml

docker-compose.yml

@@ -5,6 +5,6 @@ services:
     # image: djinn:test # local image
     image: asia-northeast2-docker.pkg.dev/jinni-api-microservices/master-djinn-test/master:latest # GCP prod instance
     ports:
-      - '80:8888'
+      - '80:80'
       - '8888:8888'
     env_file: ./.env

project.clj

@@ -30,7 +30,7 @@
                 ;; tbh no idea where these came from. pretty sure part of pedestal template. Look at other projects to see if we can delete
                 [ch.qos.logback/logback-classic "1.2.10" :exclusions [org.slf4j/slf4j-api]]
                 [org.slf4j/jul-to-slf4j "1.7.35"]
-                ;; [org.slf4j/jcl-over-slf4j "1.7.35"]
+                [org.slf4j/jcl-over-slf4j "1.7.35"]
                 [org.slf4j/log4j-over-slf4j "1.7.35"]
                 ]
   :plugins [[lein-environ "0.4.0"]]
@@ -39,20 +39,7 @@
             "dev" ["run" "-m" "master-djinn.server/run-dev"]}
   ;; If you use HTTP/2 or ALPN, use the java-agent to pull in the correct alpn-boot dependency
   ;:java-agents [[org.mortbay.jetty.alpn/jetty-alpn-agent "2.0.5"]]
-  :profiles {:dev [:project/dev :profiles/dev] ;; merge profiles.clj into project.clj config
-           ;; only edit :profiles/* in profiles.clj
-           :profiles/dev  {}
-           :project/dev {:aliases {"run-dev" ["trampoline" "run" "-m" "master-djinn.server/run-dev"]}
+  :profiles {:dev {:aliases {"run-dev" ["trampoline" "run" "-m" "master-djinn.server/run-dev"]}
                    :dependencies [[io.pedestal/pedestal.service-tools "0.6.1"]]}
-
-          ;;  :test [:project/test :profiles/test] ;; merge profiles.clj into project.clj config
-          ;;  :profiles/test {}
-          ;;  :project/test {}
-          ;;  :staging [:project/staging :profiles/staging] ;; merge profiles.clj into project.clj config
-          ;;  :profiles/staging {}
-          ;;  :project/staging {}
-          ;;  :prod [:project/prod :profiles/prod] ;; merge profiles.clj into project.clj config
-          ;;  :profiles/prod {}
-          ;;  :project/prod {}
             :uberjar {:aot [master-djinn.server]}}
   :main ^{:skip-aot true} master-djinn.server)

src/master_djinn/incantations/conjure/core.clj

@@ -10,7 +10,7 @@
 ;; Also figure out best way to use clj-http. ideally async bc then everything is in tail but had issues with that 
 ;; creating response in (let) then accessing is ok but not concurrent and i could see how it might not handle errors great
 (defn get-provider-id
-    "DOCS: "
+    "Uses portal/oauth-providers config to auto fetch players' profile and parse ID on integrated service"
     [player-id provider token]
     (try (let [config ((keyword provider) portal/oauth-providers)
                 url (:user-info-uri config)
@@ -36,6 +36,7 @@
 )
 
 (defn sync-provider-id
+    "Gets players id on an integration and save to their in game :Identity"
     [player-id provider]
     (let [id (iddb/getid player-id provider)]
         (cond

src/master_djinn/incantations/evoke/spotify.clj

@@ -53,7 +53,7 @@
 
 (defn create-silent-disco
     "No API call required to Spotify, happens locally on their device. Just here to track actions
-        TODO would be dope to somehow get the jam share url and target player sto join jam from inside Jinni"
+        TODO would be dope to somehow get the jam share url and target players to join jam from inside Jinni"
     [player-id playlist-id]
     (let [version "0.0.1" start-time (now)]
         (println "spotify create silent disco" player-id playlist-id)

src/master_djinn/incantations/transmute/android_health_connect.clj

@@ -39,6 +39,7 @@
 
 
 (defn transmute
+  "Transform raw data collected from phone into game :Action types"
   [data]
   ;; {:pre  [spec/valid? types/::action-source-data args] ;; TODO predicate for valid submit_data arg
   ;;     :post [(map string? %)]}

src/master_djinn/incantations/transmute/core.clj

@@ -15,8 +15,8 @@
     :AndroidHealthConnect ahc/transmute
     (fn [args]
       (if (types/is-data-provider? provider) ;; TODO redundant with spec
-        (println "Trans:multiplexer: VALID DATA PROVIDER WITH NO TRANSMUTER" args)
-        (println "Trans:multiplexer: INVALID DATA PROVIDER" args) )
+        (println "Trans:multiplexer: VALID DATA PROVIDER WITH NO TRANSMUTER - " provider)
+        (println "Trans:multiplexer: INVALID DATA PROVIDER - " provider) )
       []))) ;; default return empty actions instead of nil/error to prevent code complexity. can check if empty to prevent unneccessary queries
 
 ;; @DEV: is defmulti/defmethod more semantic/terse? I prefer current format personally

src/master_djinn/portal/core.clj

@@ -77,6 +77,7 @@
       (neo4j/with-transaction db/connection tx
         (->> {:pid pid
             :provider (kebab->capital provider)
+            :label (kebab->capital provider)
             :nonce nonce}
           (id/init-player-identity tx)
           doall ;; this returns null list if player does not exist. Does not throw/revert
@@ -113,7 +114,7 @@
   "Part #2 of OAuth2 flow
 
   1. use code to verify user authorization in our app
-  2. receive access_token and refresh_token back from 
+  2. receive access_token and refresh_token back from
 
   OAuth provider returns - { :body { :scope :access_token :refresh_token :expires_in }}
 
@@ -153,7 +154,9 @@
           (println "oauth token response" id body)
           ;; (println "oauth token" (str/split (:scope body) #" ") (:access_token body))
           {:status 301
-            ;; TODO redirect not working. AI generated mf
+            ;; TODO universal links instead of direct deep links
+            ;; https://stackoverflow.com/questions/77214219/expo-linking-with-custom-scheme-does-not-redirect-back-to-app
+            ;; https://docs.expo.dev/guides/deep-linking/
             :headers {"Location" (str "jinni-health://inventory/" provider)} ;; redirect with deeplink 
             :body (map->json {
               :id creds
@@ -235,24 +238,8 @@
 
 (defn oauthed-request-config
   "For sending requests on behalf of a user to an OAuth2 server
-  User must have completed oauth flow and have:Identity in db already"
+  User must have completed oauth flow and have an :Identity in db already for service being called"
   [access-token]
   {:accept :json :async? false ;; TODO bottleneck but not important with minimal users
   :headers  {"Authorization" (str "Bearer " access-token)
-              "Content-Type" "application/json"}})
-
-;; TODO OAuth providers return a new access token on each request
-;; Create a helper function that updates :Identity in DB with new access token
-
-
-
-;;; OAuth2 best practices
-;;; https://docs.cloud.coinbase.com/sign-in-with-coinbase/docs/sign-in-with-coinbase-security
-;;; included a state GET parameter during the OAuth2 authorization process. Verifying that this variable matches upon receipt of an authorization code 
-;;; validates our SSL certificate when it connects over https
-
-;; stealing code from:
-;; https://github.com/cemerick/friend
-;; https://github.com/propan/geheimtur
-;; https://github.com/yetanalytics/pedestal-oidc
-
+              "Content-Type" "application/json"}})

src/master_djinn/portal/service.clj

@@ -5,21 +5,12 @@
             [master-djinn.util.gql.schema :as schema]
             [master-djinn.portal.core :as id]
             [master-djinn.util.crypto :refer [handle-signed-POST-query]]
-
-            ;; for default lacinia config
             [com.walmartlabs.lacinia.pedestal2 :as p2]
             [com.walmartlabs.lacinia.pedestal :refer [inject]]
             [com.walmartlabs.lacinia.pedestal.internal :as lp-internal]
-
-            ;; for manual serverr config
             [io.pedestal.interceptor :refer [interceptor]]
             [io.pedestal.http.body-params :as body-params]
-            [master-djinn.util.types.core :refer [load-config]]
-
-            ;; [com.walmartlabs.lacinia :refer [execute]]
-            ;; [clojure.data.json :as json]
-            ;; [clojure.string :as str]
-            ))
+            [master-djinn.util.types.core :refer [load-config]]))
 
 (def ^:private signed-request-interceptor
   "checks if query includes a signed query and 
@@ -35,17 +26,17 @@
     {:name ::parse-signed-request
      :error lp-internal/on-error-error-response
     :enter (fn [context]
-      (let [vars (get-in context [:request :graphql-vars :verification])]
+      (let [verification (get-in context [:request :graphql-vars :verification])]
         ;; (println "service.signed-request : "  (:signature vars) (:_raw_query vars))
 
-        (if (and (:signature vars) (:_raw_query vars))
+        (if (and (:signature verification) (:_raw_query verification))
           ;; if signed query sent handle
           (handle-signed-POST-query context)
           ;; else pass along as normal GQL query
           context)))
       }))
 
-(defonce gql-dev-server-options {
+(defonce gql-server-config {
     :api-path "/graphql"
     :ide-path "/graphiql"
     :oauth-init-path "/oauth/init"
@@ -56,12 +47,6 @@
     :host (or (:api-host (load-config)) "0.0.0.0") ;; jetty defaults to serving on 0.0.0.0
 })
 
-;; This is an adapted service map, that can be started and stopped
-;; From the REPL you can call server/start and server/stop on this service
-(def default-gql-service (-> (schema/jinni-schema)
-                 (p2/default-service nil)
-                 http/create-server)) ;; might be p2/ not p2/ bc they use that internally
-
 (defn gql-interceptors [compiled-schema] 
   (-> compiled-schema
       (p2/default-interceptors nil)
@@ -73,23 +58,23 @@
         {:keys [port host oauth-init-path oauth-cb-path oauth-refresh-path]} options
         ;; aaaa (println "custom gql" interceptors)
         routes (into #{["/graphql" :post interceptors :route-name ::graphql-api]
-                      ["/graphiql" :get (p2/graphiql-ide-handler gql-dev-server-options) :route-name ::graphql-ide]
+                      ["/graphiql" :get (p2/graphiql-ide-handler gql-server-config) :route-name ::graphql-ide]
                       [oauth-init-path :get (conj [(body-params/body-params)] id/oauth-init-handler) :route-name ::oauth-init]
                       [oauth-cb-path :post (conj [(body-params/body-params)] id/oauth-callback-handler) :route-name ::oauth-callback-post]
                       [oauth-cb-path :get (conj [(body-params/body-params)] id/oauth-callback-handler) :route-name ::oauth-callback-get]
                       ;; [oauth-refresh-path :post (conj [(body-params/body-params)] id/oauth-refresh-token-handler) :route-name ::oauth-refresh]
                       }
-                  (p2/graphiql-asset-routes (:gql-asset-path gql-dev-server-options)))]
-    ;; (println "custom gql" gql-dev-server-options)
-    ;; (println "custom gql" (:host gql-dev-server-options))
+                  (p2/graphiql-asset-routes (:gql-asset-path gql-server-config)))]
+    ;; (println "custom gql" gql-server-config)
+    ;; (println "custom gql" (:host gql-server-config))
     (-> {:env :dev
          ::http/routes routes
          ::http/port port
          ::http/host host
          ::http/type :jetty
          ::http/join? false}
          p2/enable-graphiql
-        (p2/enable-subscriptions compiled-schema gql-dev-server-options)
+        (p2/enable-subscriptions compiled-schema gql-server-config)
         http/create-server)))
 
-(def custom-gql-service (create-gql-service (schema/jinni-schema) gql-dev-server-options))
+(def custom-gql-service (create-gql-service (schema/jinni-schema) gql-server-config))

src/master_djinn/util/core.clj

@@ -5,11 +5,14 @@
 (defn json->map [j] (json/read-str j :key-fn keyword))
 (defn map->json [m] (json/write-str m))
 
-(defn now []
+(defn now "get current time in ISO 8601 locale time" []
+
   (let [formatter (java.time.format.DateTimeFormatter/ofPattern "yyyy-MM-dd'T'HH:mm:ss'Z'")
         utc (java.time.ZoneId/of "UTC")
         now (java.time.ZonedDateTime/now utc)]
     (.format formatter now)))
 
-(defn get-signer [gql-ctx]
+(defn get-signer
+  "extract API request signer injected by /util/crypto pedestal interceptor"
+  [gql-ctx]
   (or (get-in gql-ctx [:request :signer]) TEST_SIGNER))

src/master_djinn/util/crypto.clj

@@ -42,7 +42,7 @@
     "@DEV: only works for <256 bytes because of Java Long. If larger values being converted switch to (BigInteger. )"
     [i]
     (Integer/toString i 16))
-;; TODO defmulti
+
 (defn bigint->hex
     "@DEV: for >256 bytes if too big for Java Long"
     [i]
@@ -84,8 +84,7 @@
         v (hex->bytes (if (< (hex->int _v) 27) (int->hex (+ (hex->int _v) 27)) _v)) ;; so coerce to ETH native 27/28
         signature-data (new org.web3j.crypto.Sign$SignatureData (first v) r s)
         hashed-msg (.getBytes original-msg)
-        ;; this really shouldnt be necessary. Error in type conversions? 
-        ;; byte mismatch consistenet with "/n" count and locations roughly.
+        ;; byte conversion mismatch on "/n".
         fixed-hashed-msg (hex->bytes (clojure.string/replace (bytes->hex hashed-msg) #"0a" "5c6e"))
         ;; eee (println "ECRECOVER hashed msg: " (bytes->hex hashed-msg) (bytes->hex fixed-hashed-msg) )
         ;; Using Sign.signedPrefixedMessageToKey for EIP-712 compliant signatures
@@ -120,17 +119,17 @@
 
         ;; MAJOR SECURITY BUG: if `sig` or `q` are mismatched we get WRONG address from ecrecover, NOT `nil` as expected
         ;; @DEV: TODO FIXES
-        ;; 1. fix security bug!!!  How? check that signer is :Identity in DB (bad), pass in pid with :verification data (bad), 
-        ;; 1. if signature/_raw_query on in POST variables even if they aren't required for the query sent
+        ;; - fix security bug!!!  How? check that signer is :Identity in DB (bad), pass in pid with :verification data (bad), 
+        ;; - if signature/_raw_query on in POST variables even if they aren't required for the query sent
         ;; then we will still go through this code path even if we dont need to
-        ;; 2. if you do `mutation submit_data(...) but define mutation/query some_other_name{...}
+        ;; - if you do `mutation submit_data(...) but define mutation/query some_other_name{...}
         ;; then lacinia will throw an invalid operation bc we are telling it to execute some_other_name
         ;; but they are still seeing submit_data somewhere in the data where we havent cleaned up properly
         ;; WORKAROUND: use unamed query/mutation in signed query and raw query
-        ;; 3. create generalized query & mutation on schema so we only have one entry point for each when using signed requests
+        ;; - create generalized query & mutation on schema so we only have one entry point for each when using signed requests
         ;; since we will extract full query from signed message and then execute that
-        ;; 4. ideally operation-name could map to a predefined query structure stored servers so we dont to pass the entire query data structure in
-        ;; theoretically this could also all for "service discovery" oce we decentralize where servers can perform certain computations based on exposed operation-names
+        ;; - ideally operation-name could map to a predefined query structure stored servers so we dont to pass the entire query data structure in
+        ;; theoretically this could also all for "service discovery" once we decentralize where servers can perform certain computations based on exposed operation-names
         ;; this requires having shared types/lib between frontend and backend without duplicating code which is a longer-term lift
 
         (if (and sig (not signer))

src/master_djinn/util/gql/incantations.clj

@@ -21,10 +21,10 @@
         jid (uuid nil pid (str (java.util.UUID/randomUUID)))]
     (println djinn (MASTER_DJINNS djinn))
     (println pid jid)
-        ;; TODO calc kin, archetype, tone for human + jinn bdays and add to Avatar model
+    ;; TODO check that pid doesnt already have a jid already
     (cond
       ;; TODO throw API errors. create resolver wrapper
-      ;; TODO define in specs not code here
+      ;; TODO define in clojure.specs not code here
       (nil? pid) (do 
         (println "Gql:Resolv:ActivateJinni:ERROR - Player must give their majik to activation")
         {:status 400 :body (map->json { :error "Player must give their majik to activation"})})