mlfeed cache canister update

.github/workflows/deploy-on-merge-to-main.yml

@@ -38,6 +38,8 @@ jobs:
       - uses: superfly/flyctl-actions/setup-flyctl@master
       - name: Set cloudflare token
         run: |
+          echo "temp printing public key. this is safe since its public key - $MLFEED_JWT_PUBLIC_KEY and ${{ env.PUBLIC_KEY_FOR_CALLING_ML_FEED_SERVER_FROM_OFFCHAIN_AGENT_SERVER }}"
+          flyctl secrets set "MLFEED_JWT_PUBLIC_KEY=$MLFEED_JWT_PUBLIC_KEY" --app "icp-off-chain-agent" --stage
           flyctl secrets set "CF_R2_ACCESS_KEY_TEMP=$CF_R2_ACCESS_KEY_TEMP" --app "icp-off-chain-agent" --stage
           flyctl secrets set "CF_R2_SECRET_ACCESS_KEY_TEMP=$CF_R2_SECRET_ACCESS_KEY_TEMP" --app "icp-off-chain-agent" --stage
           flyctl secrets set "CF_WORKER_ACCESS_OFF_CHAIN_AGENT_KEY=$CF_WORKER_ACCESS_OFF_CHAIN_AGENT_KEY" --app "icp-off-chain-agent" --stage
@@ -63,6 +65,7 @@ jobs:
           UPSTASH_VECTOR_READ_WRITE_TOKEN: ${{ secrets.UPSTASH_VECTOR_READ_WRITE_TOKEN }}
           ML_SERVER_JWT_TOKEN: ${{ secrets.ML_SERVER_JWT_TOKEN }}
           CLOUDFLARE_STREAM_READ_AND_LIST_ACCESS_TOKEN: ${{ secrets.CLOUDFLARE_STREAM_READ_AND_LIST_ACCESS_TOKEN }}
+          MLFEED_JWT_PUBLIC_KEY: ${{ env.PUBLIC_KEY_FOR_CALLING_ML_FEED_SERVER_FROM_OFFCHAIN_AGENT_SERVER }}
       - name: Deploy a docker container to fly.io
         run: flyctl deploy --remote-only
         env:

build.rs

@@ -101,6 +101,17 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .out_dir(out_dir)
         .compile(&[proto_file], &["proto"])?;
 
+    // offchain_canister
+    let proto_file = "contracts/projects/off_chain/offchain_canister.proto";
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+
+    tonic_build::configure()
+        .build_client(false)
+        .build_server(true)
+        .file_descriptor_set_path(out_dir.join("offchain_canister_descriptor.bin"))
+        .out_dir(out_dir)
+        .compile(&[proto_file], &["proto"])?;
+
     let proto_file = "contracts/projects/ml_feed/ml_feed.proto";
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
 

did/individual_user_template.did

@@ -36,6 +36,7 @@ type BettingStatus = variant {
   };
   BettingClosed;
 };
+type DeviceIdentity = record { device_id : text; timestamp : nat64 };
 type FeedScore = record {
   current_score : nat64;
   last_synchronized_at : SystemTime;
@@ -119,6 +120,12 @@ type KnownPrincipalType = variant {
   CanisterIdSnsGovernance;
   UserIdGlobalSuperAdmin;
 };
+type MLFeedCacheItem = record {
+  post_id : nat64;
+  canister_id : principal;
+  video_id : text;
+  creator_principal_id : opt principal;
+};
 type MigrationErrors = variant {
   InvalidToCanister;
   InvalidFromCanister;
@@ -234,37 +241,38 @@ type PostViewStatistics = record {
   average_watch_percentage : nat8;
   threshold_view_count : nat64;
 };
-type Result = variant { Ok : nat64; Err : text };
-type Result_1 = variant {
-  Ok : BettingStatus;
-  Err : BetOnCurrentlyViewingPostError;
-};
-type Result_10 = variant { Ok : vec SuccessHistoryItemV1; Err : text };
-type Result_11 = variant {
+type Result = variant { Ok : bool; Err : text };
+type Result_1 = variant { Ok : nat64; Err : text };
+type Result_10 = variant { Ok : SessionType; Err : text };
+type Result_11 = variant { Ok : vec SuccessHistoryItemV1; Err : text };
+type Result_12 = variant {
   Ok : vec record { nat64; TokenEvent };
   Err : GetPostsOfUserProfileError;
 };
-type Result_12 = variant { Ok : vec WatchHistoryItem; Err : text };
-type Result_13 = variant { Ok : vec text; Err : NamespaceErrors };
-type Result_14 = variant { Ok; Err : MigrationErrors };
-type Result_15 = variant { Ok : text; Err : text };
-type Result_16 = variant {
+type Result_13 = variant { Ok : vec WatchHistoryItem; Err : text };
+type Result_14 = variant { Ok : vec text; Err : NamespaceErrors };
+type Result_15 = variant { Ok; Err : MigrationErrors };
+type Result_16 = variant { Ok : text; Err : text };
+type Result_17 = variant {
   Ok : UserProfileDetailsForFrontend;
   Err : UpdateProfileDetailsError;
 };
-type Result_17 = variant { Ok; Err : text };
-type Result_18 = variant { Ok; Err : UpdateProfileSetUniqueUsernameError };
-type Result_2 = variant { Ok : NamespaceForFrontend; Err : NamespaceErrors };
-type Result_3 = variant { Ok : opt text; Err : NamespaceErrors };
-type Result_4 = variant { Ok; Err : NamespaceErrors };
-type Result_5 = variant { Ok : bool; Err : FollowAnotherUserProfileError };
-type Result_6 = variant { Ok : Post; Err };
-type Result_7 = variant { Ok : SystemTime; Err : text };
-type Result_8 = variant {
+type Result_18 = variant { Ok; Err : text };
+type Result_19 = variant { Ok; Err : UpdateProfileSetUniqueUsernameError };
+type Result_2 = variant {
+  Ok : BettingStatus;
+  Err : BetOnCurrentlyViewingPostError;
+};
+type Result_3 = variant { Ok : NamespaceForFrontend; Err : NamespaceErrors };
+type Result_4 = variant { Ok : opt text; Err : NamespaceErrors };
+type Result_5 = variant { Ok; Err : NamespaceErrors };
+type Result_6 = variant { Ok : bool; Err : FollowAnotherUserProfileError };
+type Result_7 = variant { Ok : Post; Err };
+type Result_8 = variant { Ok : SystemTime; Err : text };
+type Result_9 = variant {
   Ok : vec PostDetailsForFrontend;
   Err : GetPostsOfUserProfileError;
 };
-type Result_9 = variant { Ok : SessionType; Err : text };
 type RoomBetPossibleOutcomes = variant { HotWon; BetOngoing; Draw; NotWon };
 type RoomDetails = record {
   total_hot_bets : nat64;
@@ -324,14 +332,6 @@ type UserCanisterDetails = record {
   user_canister_id : principal;
   profile_owner : principal;
 };
-type UserProfile = record {
-  unique_user_name : opt text;
-  profile_picture_url : opt text;
-  display_name : opt text;
-  principal_id : opt principal;
-  profile_stats : UserProfileGlobalStats;
-  referrer_details : opt UserCanisterDetails;
-};
 type UserProfileDetailsForFrontend = record {
   unique_user_name : opt text;
   lifetime_earnings : nat64;
@@ -371,19 +371,20 @@ type WatchHistoryItem = record {
   cf_video_id : text;
 };
 service : (IndividualUserTemplateInitArgs) -> {
-  add_post_v2 : (PostDetailsFromFrontend) -> (Result);
-  backup_data_to_backup_canister : (principal, principal) -> ();
-  bet_on_currently_viewing_post : (PlaceBetArg) -> (Result_1);
+  add_device_id : (text) -> (Result);
+  add_post_v2 : (PostDetailsFromFrontend) -> (Result_1);
+  bet_on_currently_viewing_post : (PlaceBetArg) -> (Result_2);
   check_and_update_scores_and_share_with_post_cache_if_difference_beyond_threshold : (
       vec nat64,
     ) -> ();
   clear_snapshot : () -> ();
-  create_a_namespace : (text) -> (Result_2);
-  delete_key_value_pair : (nat64, text) -> (Result_3);
-  delete_multiple_key_value_pairs : (nat64, vec text) -> (Result_4);
-  do_i_follow_this_user : (FolloweeArg) -> (Result_5) query;
+  create_a_namespace : (text) -> (Result_3);
+  delete_key_value_pair : (nat64, text) -> (Result_4);
+  delete_multiple_key_value_pairs : (nat64, vec text) -> (Result_5);
+  do_i_follow_this_user : (FolloweeArg) -> (Result_6) query;
   download_snapshot : (nat64, nat64) -> (blob) query;
-  get_entire_individual_post_detail_by_id : (nat64) -> (Result_6) query;
+  get_device_identities : () -> (vec DeviceIdentity) query;
+  get_entire_individual_post_detail_by_id : (nat64) -> (Result_7) query;
   get_hot_or_not_bet_details_for_this_post : (nat64) -> (BettingStatus) query;
   get_hot_or_not_bets_placed_by_this_profile_with_pagination : (nat64) -> (
       vec PlacedBetDetail,
@@ -392,13 +393,14 @@ service : (IndividualUserTemplateInitArgs) -> {
       opt PlacedBetDetail,
     ) query;
   get_individual_post_details_by_id : (nat64) -> (PostDetailsForFrontend) query;
-  get_last_access_time : () -> (Result_7) query;
-  get_last_canister_functionality_access_time : () -> (Result_7) query;
+  get_last_access_time : () -> (Result_8) query;
+  get_last_canister_functionality_access_time : () -> (Result_8) query;
+  get_ml_feed_cache_paginated : (nat64, nat64) -> (vec MLFeedCacheItem) query;
   get_posts_of_this_user_profile_with_pagination : (nat64, nat64) -> (
-      Result_8,
+      Result_9,
     ) query;
   get_posts_of_this_user_profile_with_pagination_cursor : (nat64, nat64) -> (
-      Result_8,
+      Result_9,
     ) query;
   get_principals_that_follow_this_profile_paginated : (opt nat64) -> (
       vec record { nat64; FollowEntryDetail },
@@ -410,68 +412,59 @@ service : (IndividualUserTemplateInitArgs) -> {
   get_profile_details_v2 : () -> (UserProfileDetailsForFrontendV2) query;
   get_rewarded_for_referral : (principal, principal) -> ();
   get_rewarded_for_signing_up : () -> ();
-  get_session_type : () -> (Result_9) query;
+  get_session_type : () -> (Result_10) query;
   get_stable_memory_size : () -> (nat32) query;
-  get_success_history : () -> (Result_10) query;
+  get_success_history : () -> (Result_11) query;
   get_user_caniser_cycle_balance : () -> (nat) query;
   get_user_utility_token_transaction_history_with_pagination : (
       nat64,
       nat64,
-    ) -> (Result_11) query;
+    ) -> (Result_12) query;
   get_utility_token_balance : () -> (nat64) query;
   get_version : () -> (text) query;
   get_version_number : () -> (nat64) query;
-  get_watch_history : () -> (Result_12) query;
+  get_watch_history : () -> (Result_13) query;
   get_well_known_principal_value : (KnownPrincipalType) -> (
       opt principal,
     ) query;
   http_request : (HttpRequest) -> (HttpResponse) query;
-  list_namespace_keys : (nat64) -> (Result_13) query;
+  list_namespace_keys : (nat64) -> (Result_14) query;
   list_namespaces : (nat64, nat64) -> (vec NamespaceForFrontend) query;
   load_snapshot : (nat64) -> ();
-  read_key_value_pair : (nat64, text) -> (Result_3) query;
+  read_key_value_pair : (nat64, text) -> (Result_4) query;
   receive_and_save_snaphot : (nat64, blob) -> ();
-  receive_bet_from_bet_makers_canister : (PlaceBetArg, principal) -> (Result_1);
+  receive_bet_from_bet_makers_canister : (PlaceBetArg, principal) -> (Result_2);
   receive_bet_winnings_when_distributed : (nat64, BetOutcomeForBetMaker) -> ();
-  receive_data_from_hotornot : (principal, nat64, vec Post) -> (Result_14);
-  receive_my_created_posts_from_data_backup_canister : (vec Post) -> ();
-  receive_my_profile_from_data_backup_canister : (UserProfile) -> ();
-  receive_my_utility_token_balance_from_data_backup_canister : (nat64) -> ();
-  receive_my_utility_token_transaction_history_from_data_backup_canister : (
-      vec record { nat64; TokenEvent },
-    ) -> ();
-  receive_principals_i_follow_from_data_backup_canister : (vec principal) -> ();
-  receive_principals_that_follow_me_from_data_backup_canister : (
-      vec principal,
-    ) -> ();
+  receive_data_from_hotornot : (principal, nat64, vec Post) -> (Result_15);
   return_cycles_to_user_index_canister : (opt nat) -> ();
   save_snapshot_json : () -> (nat32);
-  transfer_tokens_and_posts : (principal, principal) -> (Result_14);
-  update_last_access_time : () -> (Result_15);
+  transfer_tokens_and_posts : (principal, principal) -> (Result_15);
+  update_last_access_time : () -> (Result_16);
   update_last_canister_functionality_access_time : () -> ();
+  update_ml_feed_cache : (vec MLFeedCacheItem) -> (Result_16);
   update_post_add_view_details : (nat64, PostViewDetailsFromFrontend) -> ();
   update_post_as_ready_to_view : (nat64) -> ();
   update_post_increment_share_count : (nat64) -> (nat64);
   update_post_status : (nat64, PostStatus) -> ();
   update_post_toggle_like_status_by_caller : (nat64) -> (bool);
   update_profile_display_details : (UserProfileUpdateDetailsFromFrontend) -> (
-      Result_16,
+      Result_17,
     );
-  update_profile_owner : (opt principal) -> (Result_17);
-  update_profile_set_unique_username_once : (text) -> (Result_18);
+  update_profile_owner : (opt principal) -> (Result_18);
+  update_profile_set_unique_username_once : (text) -> (Result_19);
   update_profiles_i_follow_toggle_list_with_specified_profile : (
       FolloweeArg,
-    ) -> (Result_5);
+    ) -> (Result_6);
   update_profiles_that_follow_me_toggle_list_with_specified_profile : (
       FollowerArg,
-    ) -> (Result_5);
-  update_referrer_details : (UserCanisterDetails) -> (Result_15);
-  update_session_type : (SessionType) -> (Result_15);
-  update_success_history : (SuccessHistoryItemV1) -> (Result_15);
-  update_watch_history : (WatchHistoryItem) -> (Result_15);
+    ) -> (Result_6);
+  update_referrer_details : (UserCanisterDetails) -> (Result_16);
+  update_session_type : (SessionType) -> (Result_16);
+  update_success_history : (SuccessHistoryItemV1) -> (Result_16);
+  update_watch_history : (WatchHistoryItem) -> (Result_16);
   update_well_known_principal : (KnownPrincipalType, principal) -> ();
-  write_key_value_pair : (nat64, text, text) -> (Result_3);
+  write_key_value_pair : (nat64, text, text) -> (Result_4);
   write_multiple_key_value_pairs : (nat64, vec record { text; text }) -> (
-      Result_4,
+      Result_5,
     );
 }

src/app_state.rs

@@ -96,12 +96,11 @@ pub async fn init_agent() -> Agent {
     #[cfg(feature = "local-bin")]
     {
         let agent = Agent::builder()
-            .with_url("https://ic0.app")
+            .with_url("http://127.0.0.1:4943")
             .build()
             .unwrap();
 
-        // ‼️‼️comment below line in mainnet‼️‼️
-        // agent.fetch_root_key().await.unwrap();
+        agent.fetch_root_key().await.unwrap();
 
         agent
     }

src/auth.rs

@@ -1,10 +1,13 @@
+use std::collections::HashSet;
 use std::env;
 
 use axum::extract::FromRequestParts;
 use axum::http::StatusCode;
 use axum::response::{IntoResponse, Response};
 use axum::{async_trait, Json};
 use http::request::Parts;
+use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
+use serde::{Deserialize, Serialize};
 use serde_json::json;
 use tonic::metadata::MetadataValue;
 use tonic::{Request, Status};
@@ -70,3 +73,39 @@ pub fn check_auth_grpc(req: Request<()>) -> Result<Request<()>, Status> {
 pub fn check_auth_grpc_test(req: Request<()>) -> Result<Request<()>, Status> {
     Ok(req)
 }
+
+#[derive(Debug, Serialize, Deserialize, PartialEq)]
+pub struct MLFeedClaims {
+    pub sub: String,
+    pub company: String,
+}
+
+pub fn check_auth_grpc_offchain_mlfeed(req: Request<()>) -> Result<Request<()>, Status> {
+    let token = req
+        .metadata()
+        .get("authorization")
+        .ok_or(Status::unauthenticated("No valid auth token"))?
+        .to_str()
+        .map_err(|_| Status::unauthenticated("Invalid auth token"))?
+        .trim_start_matches("Bearer ");
+
+    let mlfeed_public_key =
+        env::var("MLFEED_JWT_PUBLIC_KEY").expect("MLFEED_JWT_PUBLIC_KEY is required");
+
+    let decoding_key = DecodingKey::from_ed_pem(mlfeed_public_key.as_bytes())
+        .expect("failed to create decoding key");
+
+    let mut validation = Validation::new(Algorithm::EdDSA);
+    validation.required_spec_claims = HashSet::new();
+    validation.validate_exp = false;
+
+    let token_message =
+        decode::<MLFeedClaims>(token, &decoding_key, &validation).expect("failed to decode token");
+
+    let claims = token_message.claims;
+    if claims.sub != "yral-ml-feed-server" || claims.company != "gobazzinga" {
+        return Err(Status::unauthenticated("Invalid auth token"));
+    }
+
+    Ok(req)
+}