feature: add fetch endpoint to work around local CORs issue

lib/api/Cargo.toml

@@ -9,15 +9,17 @@ edition = "2021"
 chrono = { workspace = true }
 dotenv = { workspace = true }
 dotenv_codegen = { workspace = true }
-log = { workspace = true }
 jsonschema = "0.17.1"
+log = { workspace = true }
+qdrant-client = "1.2.0"
 sea-orm = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 thiserror = "1.0"
 tokio = { workspace = true }
+reqwest = { version = "0.11", features = ["json"] }
+url = "2.4.1"
 uuid = { version = "1.3.3", features = ["v4", "fast-rng"] }
 warp = "0.3.3"
-qdrant-client = "1.2.0"
 
 libmemex = { path = "../libmemex" }

lib/api/src/endpoints/fetch/filters.rs

@@ -0,0 +1,20 @@
+use serde::{Deserialize, Serialize};
+use warp::Filter;
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct FetchRequest {
+    /// Url to fetch
+    pub url: Option<String>,
+}
+
+fn fetch_url() -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
+    warp::path!("fetch")
+        .and(warp::get())
+        .and(warp::query::<FetchRequest>())
+        .and_then(super::handlers::handle_fetch)
+}
+
+pub fn build() -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
+    fetch_url()
+}

lib/api/src/endpoints/fetch/handlers.rs

@@ -0,0 +1,24 @@
+use crate::{schema::ApiResponse, ServerError};
+use warp::reject::Rejection;
+
+use super::filters;
+
+pub async fn handle_fetch(query: filters::FetchRequest) -> Result<impl warp::Reply, Rejection> {
+    let time = std::time::Instant::now();
+
+    if let Some(url) = query.url {
+        let content = reqwest::get(url)
+            .await
+            .map_err(|err| ServerError::Other(err.to_string()))?
+            .text()
+            .await
+            .map_err(|err| ServerError::Other(err.to_string()))?;
+
+        Ok(warp::reply::json(&ApiResponse::success(
+            &time.elapsed(),
+            Some(serde_json::json!({ "content": content })),
+        )))
+    } else {
+        Err(warp::reject())
+    }
+}

lib/api/src/endpoints/fetch/mod.rs

@@ -0,0 +1,2 @@
+pub mod filters;
+pub mod handlers;

lib/api/src/endpoints/mod.rs

@@ -5,6 +5,7 @@ use warp::Filter;
 
 mod actions;
 mod collections;
+mod fetch;
 mod tasks;
 
 const LIMIT_1_MB: u64 = 1000 * 1024;
@@ -22,5 +23,6 @@ pub fn build(
 ) -> impl Filter<Extract = (impl warp::Reply,), Error = warp::Rejection> + Clone {
     actions::filters::build(llm, db)
         .or(collections::filters::build(db))
+        .or(fetch::filters::build())
         .or(tasks::filters::build(db))
 }