Merge pull request #100 from yral-dapp/refactor/remove-deps

refactor: add secrets file to gitignore, remove deps on external services

.gitignore

@@ -1,4 +1,5 @@
 /target
 /.direnv
 gha-creds-*.json
-.DS_Store
+.DS_Store
+google_service_account.json

Dockerfile

@@ -2,8 +2,6 @@ FROM debian:bookworm-20240211
 
 WORKDIR /app
 
-COPY ./target/x86_64-unknown-linux-musl/release/icp-off-chain-agent .
-
 RUN apt-get update \
     && apt-get install -y ca-certificates \
     && apt-get -y install curl \
@@ -25,4 +23,5 @@ EXPOSE 50051
 # # You might need to change this depending on where your crontab is located
 # COPY crontab crontab
 
+COPY ./target/x86_64-unknown-linux-musl/release/icp-off-chain-agent .
 CMD ["./icp-off-chain-agent"]

Dockerfile-local

@@ -0,0 +1,33 @@
+FROM debian:bookworm-20240211
+
+WORKDIR /app
+
+RUN apt-get update \
+    && apt-get install -y ca-certificates \
+    && apt-get -y install curl \
+    && apt-get -y install ffmpeg
+
+EXPOSE 50051
+
+ENV GOOGLE_SA_KEY=test
+ENV YRAL_METADATA_TOKEN="test_token"
+ENV QSTASH_CURRENT_SIGNING_KEY="test_key"
+ENV QSTASH_AUTH_TOKEN="test_token"
+
+
+# Latest releases available at https://github.com/aptible/supercronic/releases
+# ENV SUPERCRONIC_URL=https://github.com/aptible/supercronic/releases/download/v0.2.29/supercronic-linux-amd64 \
+#     SUPERCRONIC=supercronic-linux-amd64 \
+#     SUPERCRONIC_SHA1SUM=cd48d45c4b10f3f0bfdd3a57d054cd05ac96812b
+
+# RUN curl -fsSLO "$SUPERCRONIC_URL" \
+#     && echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - \
+#     && chmod +x "$SUPERCRONIC" \
+#     && mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" \
+#     && ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic
+
+# # You might need to change this depending on where your crontab is located
+# COPY crontab crontab
+
+COPY ./target/x86_64-unknown-linux-musl/release/icp-off-chain-agent .
+CMD ["./icp-off-chain-agent"]

src/app_state.rs

@@ -20,12 +20,16 @@ use yup_oauth2::{authenticator::Authenticator, ServiceAccountAuthenticator};
 pub struct AppState {
     pub agent: ic_agent::Agent,
     pub yral_metadata_client: MetadataClient<true>,
+    #[cfg(not(feature = "local-bin"))]
     pub auth: Authenticator<HttpsConnector<HttpConnector>>,
+    #[cfg(not(feature = "local-bin"))]
     pub firestoredb: FirestoreDb,
     pub qstash: QStashState,
+    #[cfg(not(feature = "local-bin"))]
     pub bigquery_client: Client,
     pub nsfw_detect_channel: Channel,
     pub qstash_client: QStashClient,
+    #[cfg(not(feature = "local-bin"))]
     pub gcs_client: Arc<cloud_storage::Client>,
 }
 
@@ -34,24 +38,36 @@ impl AppState {
         AppState {
             yral_metadata_client: init_yral_metadata_client(&app_config),
             agent: init_agent().await,
+            #[cfg(not(feature = "local-bin"))]
             auth: init_auth().await,
             // ml_server_grpc_channel: init_ml_server_grpc_channel().await,
+            #[cfg(not(feature = "local-bin"))]
             firestoredb: init_firestoredb().await,
             qstash: init_qstash(),
+            #[cfg(not(feature = "local-bin"))]
             bigquery_client: init_bigquery_client().await,
             nsfw_detect_channel: init_nsfw_detect_channel().await,
             qstash_client: init_qstash_client().await,
+            #[cfg(not(feature = "local-bin"))]
             gcs_client: Arc::new(cloud_storage::Client::default()),
         }
     }
 
     pub async fn get_access_token(&self, scopes: &[&str]) -> String {
-        let auth = &self.auth;
-        let token = auth.token(scopes).await.unwrap();
+        #[cfg(feature = "local-bin")]
+        {
+            "localtoken".into()
+        }
+
+        #[cfg(not(feature = "local-bin"))]
+        {
+            let auth = &self.auth;
+            let token = auth.token(scopes).await.unwrap();
 
-        match token.token() {
-            Some(t) => t.to_string(),
-            _ => panic!("No access token found"),
+            match token.token() {
+                Some(t) => t.to_string(),
+                _ => panic!("No access token found"),
+            }
         }
     }
 

src/events/event.rs

@@ -230,6 +230,7 @@ impl Event {
         });
     }
 
+    #[cfg(not(feature = "local-bin"))]
     pub fn stream_to_firestore(&self, app_state: &AppState) {
         if self.event.event == "token_creation_completed" {
             let app_state = app_state.clone();
@@ -305,6 +306,15 @@ fn system_time_to_custom(time: std::time::SystemTime) -> SystemTime {
     }
 }
 
+#[cfg(feature = "local-bin")]
+pub async fn stream_to_bigquery_token_metadata_impl_v2(
+    app_state: &AppState,
+    data: ICPumpTokenMetadata,
+) -> Result<(), anyhow::Error> {
+    Ok(())
+}
+
+#[cfg(not(feature = "local-bin"))]
 pub async fn stream_to_bigquery_token_metadata_impl_v2(
     app_state: &AppState,
     data: ICPumpTokenMetadata,

src/events/mod.rs

@@ -39,6 +39,7 @@ impl WarehouseEvents for WarehouseEventsService {
         let params: Value = serde_json::from_str(&event.event.params).expect("Invalid JSON");
         let event_type: &str = &event.event.event;
 
+        #[cfg(not(feature = "local-bin"))]
         event.stream_to_bigquery(&shared_state.clone());
 
         event.upload_to_gcs(&shared_state.clone());
@@ -47,8 +48,10 @@ impl WarehouseEvents for WarehouseEventsService {
 
         event.update_success_history(&shared_state.clone());
 
+        #[cfg(not(feature = "local-bin"))]
         event.stream_to_firestore(&shared_state.clone());
 
+        #[cfg(not(feature = "local-bin"))]
         event.stream_to_bigquery_token_metadata(&shared_state.clone());
 
         let _ = dispatch_notif(event_type, params, &shared_state.clone()).await;

src/events/nsfw.rs

@@ -114,6 +114,7 @@ pub async fn extract_frames_and_upload(
     );
     let output_dir = create_output_directory(&video_id)?;
     let frames = extract_frames(&video_path, output_dir.clone())?;
+    #[cfg(not(feature = "local-bin"))]
     upload_frames_to_gcs(&state.gcs_client, frames, &video_id).await?;
     // delete output directory
     fs::remove_dir_all(output_dir)?;
@@ -176,6 +177,15 @@ struct VideoNSFWData {
     nsfw_gore: String,
 }
 
+#[cfg(feature = "local-bin")]
+pub async fn nsfw_job(
+    State(state): State<Arc<AppState>>,
+    Json(payload): Json<VideoRequest>,
+) -> Result<Json<serde_json::Value>, AppError> {
+    Err(anyhow::anyhow!("not implemented for local binary").into())
+}
+
+#[cfg(not(feature = "local-bin"))]
 pub async fn nsfw_job(
     State(state): State<Arc<AppState>>,
     Json(payload): Json<VideoRequest>,