General STT crate

Cargo.toml

@@ -20,9 +20,10 @@ time = "0.3.37"
 pin-project = "1"
 sentry = "0.35.0"
 bytes = "1.9.0"
+dotenvy = "0.15.7"
+dotenvy_macro = "0.15.7"
 
 ndarray = "0.16.1"
-ort = "2.0.0-rc.9"
 tokenizers = "0.19.1"
 candle-examples = { git = "https://github.com/huggingface/candle", tag = "0.8.1" }
 candle-core = "0.8.1"

apps/desktop/src-tauri/Cargo.toml

@@ -47,7 +47,7 @@ thiserror = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 uuid = { workspace = true }
-dotenvy_macro = "0.15.7"
+dotenvy_macro = { workspace = true }
 log = "0.4.22"
 
 reqwest = { workspace = true, features = ["json", "stream"] }

apps/server/Cargo.toml

@@ -7,13 +7,13 @@ edition = "2021"
 hypr-bridge = { path = "../../crates/bridge", package = "bridge" }
 hypr-db-server = { path = "../../crates/db-server", package = "db-server" }
 hypr-s3 = { path = "../../crates/s3", package = "s3" }
+hypr-stt = { path = "../../crates/stt", package = "stt" }
 
 shuttle-axum = "0.49.0"
 shuttle-runtime = "0.49.0"
 shuttle-shared-db = { version = "0.49.0", features = ["postgres", "sqlx"] }
 shuttle-posthog = { path = "../../crates/shuttle-posthog" }
 shuttle-clerk = { path = "../../crates/shuttle-clerk" }
-shuttle-stt = { path = "../../crates/shuttle-stt" }
 
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 futures = { workspace = true }

apps/server/src/main.rs

@@ -9,7 +9,6 @@ use axum::{
 use shuttle_clerk::{ClerkClient as Clerk, ClerkLayer, MemoryCacheJwksProvider};
 use shuttle_posthog::posthog::Client as Posthog;
 use shuttle_runtime::SecretStore;
-use shuttle_stt::STTClient as STT;
 
 use sqlx::PgPool;
 use std::time::Duration;
@@ -34,11 +33,6 @@ async fn main(
         api_key = "{secrets.POSTHOG_API_KEY}"
     )]
     posthog: Posthog,
-    #[shuttle_stt::STT(
-        deepgram_api_key = "{secrets.DEEPGRAM_API_KEY}",
-        clova_api_key = "{secrets.CLOVA_API_KEY}"
-    )]
-    _stt: STT,
 ) -> shuttle_axum::ShuttleAxum {
     hypr_db_server::migrate(&db).await.unwrap();
 

crates/aec/Cargo.toml

@@ -7,4 +7,4 @@ edition = "2021"
 anyhow = { workspace = true }
 tokio = { workspace = true, features = ["rt", "macros"] }
 ndarray = { workspace = true }
-ort = { workspace = true, features = ["coreml"] }
+ort = { version = "2.0.0-rc.9", features = ["coreml"] }

crates/clova/Cargo.toml

@@ -16,4 +16,4 @@ tokio = { workspace = true }
 futures = { workspace = true }
 
 prost = "0.13.4"
-tonic = { version = "0.12.3", features = ["tls"] }
+tonic = { version = "0.12.3", features = ["channel", "tls-native-roots"] }

crates/clova/build.rs

@@ -1,7 +1,7 @@
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     tonic_build::configure()
         .build_server(false)
-        .out_dir("./src")
+        .out_dir("./src/interface")
         .compile_protos(&["proto/nest.proto"], &["proto"])?;
 
     Ok(())

crates/clova/src/interface/mod.rs

@@ -0,0 +1,7 @@
+mod types;
+mod nest {
+    include!("./com.nbp.cdncp.nest.grpc.proto.v1.rs");
+}
+
+pub use nest::*;
+pub use types::*;

crates/clova/src/interface.rs → crates/clova/src/interface/types.rs

@@ -1,3 +1,4 @@
+use serde::de::Error as _;
 use serde::{Deserialize, Serialize};
 
 // https://api.ncloud-docs.com/docs/en/ai-application-service-clovaspeech-grpc#3-request-config-json
@@ -21,98 +22,74 @@ pub enum Language {
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct ConfigResponse {
+    pub uid: String,
     pub config: ConfigResponseInner,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct ConfigResponseInner {
-    pub status: ConfigResponseStatus,
-}
-
-#[derive(Debug, PartialEq, Deserialize, Serialize)]
-pub enum ConfigResponseStatus {
-    Success,
-    Failure,
+    pub status: String,
 }
 
 // https://api.ncloud-docs.com/docs/ai-application-service-clovaspeech-grpc#%EC%9D%91%EB%8B%B5-%EC%98%88%EC%8B%9C1
 #[derive(Debug, Deserialize, Serialize)]
+#[serde(try_from = "StreamResponseRaw")]
 pub enum StreamResponse {
-    Success(StreamResponseSuccess),
-    Failure(StreamResponseFailure),
+    Config(ConfigResponse),
+    TranscribeSuccess(StreamResponseSuccess),
+    TranscribeFailure(StreamResponseFailure),
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct StreamResponseRaw {
+    response_type: Vec<String>,
+    #[serde(flatten)]
+    raw: serde_json::Value,
+}
+
+impl TryFrom<StreamResponseRaw> for StreamResponse {
+    type Error = serde_json::Error;
+
+    fn try_from(raw: StreamResponseRaw) -> Result<Self, Self::Error> {
+        let response_type = raw
+            .response_type
+            .first()
+            .ok_or_else(|| serde_json::Error::custom("missing response_type"))?;
+
+        match response_type.as_str() {
+            "config" => serde_json::from_value(raw.raw).map(StreamResponse::Config),
+            "recognize" => serde_json::from_value(raw.raw).map(StreamResponse::TranscribeFailure),
+            "transcription" => {
+                serde_json::from_value(raw.raw).map(StreamResponse::TranscribeSuccess)
+            }
+            _ => Err(serde_json::Error::custom("invalid response_type")),
+        }
+    }
 }
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct StreamResponseSuccess {
     pub uid: String,
-    pub response_type: Vec<String>,
     pub transcription: TranscriptionResponse,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
+#[serde(rename_all = "camelCase")]
 pub struct TranscriptionResponse {
     pub text: String,
-    pub position: i32,
-    pub period_positions: Vec<i32>,
-    pub period_align_indices: Vec<i32>,
-    pub ep_flag: bool,
-    pub seq_id: i32,
-    pub epd_type: EpdType,
     pub start_timestamp: i64,
     pub end_timestamp: i64,
     pub confidence: f64,
-    pub align_infos: Vec<AlignInfo>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct AlignInfo {
-    pub word: String,
-    pub start: i64,
-    pub end: i64,
-    pub confidence: f64,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct StreamResponseFailure {
     pub uid: String,
-    pub response_type: Vec<String>,
     pub recognize: RecognizeError,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct RecognizeError {
     pub status: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub ep_flag: Option<StatusInfo>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub seq_id: Option<StatusInfo>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub audio: Option<StatusInfo>,
-}
-
-#[derive(Debug, Deserialize, Serialize)]
-pub struct StatusInfo {
-    pub status: String,
-}
-
-mod nest {
-    include!("./com.nbp.cdncp.nest.grpc.proto.v1.rs");
-}
-
-pub use nest::*;
-
-#[derive(Debug, Deserialize, Serialize)]
-pub enum EpdType {
-    #[serde(rename = "gap")]
-    Gap,
-    #[serde(rename = "endPoint")]
-    EndPoint,
-    #[serde(rename = "durationThreshold")]
-    DurationThreshold,
-    #[serde(rename = "period")]
-    Period,
-    #[serde(rename = "syllableThreshold")]
-    SyllableThreshold,
-    #[serde(rename = "unvoice")]
-    Unvoice,
 }

crates/clova/src/lib.rs

@@ -1,28 +1,27 @@
-use anyhow::Result;
-use bytes::Bytes;
-
 mod handle;
 pub mod interface;
 
+use anyhow::Result;
+use bytes::Bytes;
 use futures::{Stream, StreamExt};
 use serde::{Deserialize, Serialize};
-use std::{error::Error, str::FromStr};
-use tonic::{
-    metadata::{MetadataMap, MetadataValue},
-    Request,
-};
+use std::error::Error;
 
 use interface::nest_service_client::NestServiceClient;
+use tonic::{service::interceptor::InterceptedService, transport::Channel, Request, Status};
+
+// https://docs.rs/tonic/latest/tonic/service/trait.Interceptor.html
+type Interceptor = Box<dyn FnMut(Request<()>) -> Result<Request<()>, Status>>;
 
 pub struct Client {
-    inner: NestServiceClient<tonic::transport::Channel>,
+    inner: NestServiceClient<InterceptedService<Channel, Interceptor>>,
     config: Config,
 }
 
 #[derive(Debug, Deserialize, Serialize)]
 pub struct Config {
-    secret_key: String,
-    config: interface::ConfigRequest,
+    pub secret_key: String,
+    pub config: interface::ConfigRequest,
 }
 
 impl Config {
@@ -40,60 +39,23 @@ impl Client {
     pub async fn new(config: Config) -> Result<Self> {
         let channel =
             tonic::transport::Channel::from_static("https://clovaspeech-gw.ncloud.com:50051")
-                .tls_config(tonic::transport::ClientTlsConfig::new())?
+                .tls_config(tonic::transport::ClientTlsConfig::new().with_native_roots())?
                 .connect()
                 .await?;
 
-        let inner = NestServiceClient::new(channel);
+        let key = config.secret_key.clone();
+        let inner = NestServiceClient::with_interceptor(channel, Self::make_interceptor(key));
 
         Ok(Self { inner, config })
     }
 
-    fn auth<T>(&self, request: T) -> Request<T> {
-        let mut req = Request::new(request);
-        let mut metadata = MetadataMap::new();
-
-        let auth_header = format!("Bearer {}", self.config.secret_key);
-        let auth_value = MetadataValue::from_str(&auth_header).unwrap();
-
-        // never capitalize authorization
-        metadata.insert("authorization", auth_value);
-
-        *req.metadata_mut() = metadata;
-        req
-    }
-
-    async fn config_request(&mut self) -> Result<()> {
-        let config = self.config.config.clone();
-
-        let request = interface::NestRequest {
-            r#type: interface::RequestType::Config.into(),
-            part: Some(interface::nest_request::Part::Config(
-                interface::NestConfig {
-                    config: serde_json::to_string(&config).unwrap(),
-                },
-            )),
-        };
-
-        let request = self.auth(request);
-        let response = self
-            .inner
-            .recognize(tonic::Request::new(futures::stream::once(async {
-                request.into_inner()
-            })))
-            .await?;
-
-        let mut stream = response.into_inner();
-
-        while let Some(message) = stream.message().await? {
-            let res: interface::ConfigResponse = serde_json::from_str(&message.contents)?;
-
-            if res.config.status != interface::ConfigResponseStatus::Success {
-                return Err(anyhow::anyhow!("config request failed"));
-            }
-        }
-
-        Ok(())
+    fn make_interceptor(secret_key: String) -> Interceptor {
+        Box::new(move |mut req: Request<()>| {
+            req.metadata_mut()
+                // lowercase is required
+                .insert("authorization", secret_key.parse().unwrap());
+            Ok(req)
+        })
     }
 
     pub async fn stream<S, E>(
@@ -104,15 +66,22 @@ impl Client {
         S: Stream<Item = Result<Bytes, E>> + Send + Unpin + 'static,
         E: Error + Send + Sync + 'static,
     {
-        self.config_request().await?;
+        let config = serde_json::to_string(&self.config.config).unwrap();
+        let config_request = interface::NestRequest {
+            r#type: interface::RequestType::Config.into(),
+            part: Some(interface::nest_request::Part::Config(
+                interface::NestConfig { config },
+            )),
+        };
+        let config_stream = futures::stream::once(async move { config_request });
 
-        let request_stream = stream.filter_map(|chunk| async {
+        let audio_request_stream = stream.filter_map(|chunk| async {
             if let Ok(chunk) = chunk {
                 Some(interface::NestRequest {
                     r#type: interface::RequestType::Data.into(),
                     part: Some(interface::nest_request::Part::Data(interface::NestData {
-                        chunk: chunk.to_vec(),
-                        extra_contents: "".to_string(),
+                        chunk: chunk.into(),
+                        extra_contents: r#"{"seqId": 0, "epFlag": false}"#.to_string(),
                     })),
                 })
             } else {
@@ -122,10 +91,11 @@ impl Client {
 
         let response = self
             .inner
-            .recognize(request_stream)
+            .recognize(config_stream.chain(audio_request_stream))
             .await?
             .into_inner()
             .map(|message| {
+                println!("message: {:?}", message);
                 let res = serde_json::from_str::<interface::StreamResponse>(&message?.contents)?;
                 Ok(res)
             });