From 02b9abdef5dea4b99af4e7d347c356cc0e0f818e Mon Sep 17 00:00:00 2001 From: Anish Sinha Date: Sun, 13 Oct 2024 02:48:04 -0400 Subject: [PATCH] Add a schema validation utility to airtable service. Also, refactor airtable.rs in data_imports to a flat file module instead of directory since the code size was reduced by a factor of 4 --- .../api/v1/data_exports/workspace/policies.rs | 2 + .../{airtable/mod.rs => airtable.rs} | 0 src/app/api/v1/data_imports/controllers.rs | 7 ++ src/main.rs | 2 +- src/services/airtable/mod.rs | 80 ++++++++++++++----- 5 files changed, 72 insertions(+), 19 deletions(-) rename src/app/api/v1/data_imports/{airtable/mod.rs => airtable.rs} (100%) diff --git a/src/app/api/v1/data_exports/workspace/policies.rs b/src/app/api/v1/data_exports/workspace/policies.rs index dea7729..1f23650 100644 --- a/src/app/api/v1/data_exports/workspace/policies.rs +++ b/src/app/api/v1/data_exports/workspace/policies.rs @@ -1,5 +1,6 @@ use rand::distributions::Alphanumeric; use rand::Rng; +use serde::{Deserialize, Serialize}; use crate::app::api::v1::data_exports::requests::ExportUsersToWorkspaceRequest; @@ -36,6 +37,7 @@ impl EmailPolicy { } } +#[derive(Debug, Serialize, Deserialize)] pub struct PasswordPolicy { pub change_password_at_next_login: bool, pub generated_password_length: u8, diff --git a/src/app/api/v1/data_imports/airtable/mod.rs b/src/app/api/v1/data_imports/airtable.rs similarity index 100% rename from src/app/api/v1/data_imports/airtable/mod.rs rename to src/app/api/v1/data_imports/airtable.rs diff --git a/src/app/api/v1/data_imports/controllers.rs b/src/app/api/v1/data_imports/controllers.rs index dd56b5d..915b4e0 100644 --- a/src/app/api/v1/data_imports/controllers.rs +++ b/src/app/api/v1/data_imports/controllers.rs @@ -56,6 +56,13 @@ pub async fn import_airtable_base( ) -> Result { let storage_layer = &services.storage_layer; + if !services.airtable.validate_schema(&base_id).await? { + return Ok(api_response::error( + StatusCode::BAD_REQUEST, + "Invalid schema for airtable base", + )); + } + let current_time = Utc::now(); let time_only = current_time.format("%H:%M:%S").to_string(); diff --git a/src/main.rs b/src/main.rs index d436c30..c9824fc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -49,7 +49,7 @@ async fn main() -> Result<()> { Err(_) => println!("no .env file found"), }; - tracing_subscriber::fmt().with_max_level(tracing::Level::INFO).init(); + tracing_subscriber::fmt().with_max_level(tracing::Level::DEBUG).init(); let args = Args::parse(); diff --git a/src/services/airtable/mod.rs b/src/services/airtable/mod.rs index 2453450..09de79c 100644 --- a/src/services/airtable/mod.rs +++ b/src/services/airtable/mod.rs @@ -99,10 +99,48 @@ pub trait AirtableClient: Send + Sync { async fn get_mentor_mentee_linkages(&self, base_id: &str) -> Result> { unimplemented!() } + + async fn validate_schema(&self, base_id: &str) -> Result { + unimplemented!() + } } #[async_trait] impl AirtableClient for Airtable { + async fn validate_schema(&self, base_id: &str) -> Result { + let schema = self.get_base_schema(base_id, vec![]).await?; + + let (Some(volunteers_table), Some(_)) = ( + schema.tables.iter().find(|table| table.name == "Volunteers"), + schema.tables.iter().find(|table| table.name == "Nonprofits"), + ) else { + return Ok(false); + }; + + let v_has_required_fields = REQUIRED_VOLUNTEER_FIELDS.iter().all(|required_field| { + volunteers_table + .fields + .iter() + .map(|f| f.name.as_ref()) + .collect::>() + .contains(required_field) + }); + + if !v_has_required_fields { + return Ok(false); + } + + let (Some(_), Some(_), Some(_)) = ( + volunteers_table.views.iter().find(|view| view.name == VOLUNTEERS_VIEW), + volunteers_table.views.iter().find(|view| view.name == MENTORS_VIEW), + volunteers_table.views.iter().find(|view| view.name == MENTOR_MENTEE_LINKAGE_VIEW), + ) else { + return Ok(false); + }; + + Ok(true) + } + async fn list_available_bases(&self) -> Result> { let mut offset = Option::::None; @@ -128,15 +166,15 @@ impl AirtableClient for Airtable { let mut volunteers = Vec::::with_capacity(300); - while let Ok(res) = - self.list_records::(base_id, "Volunteers", query.clone()).await - { + loop { + let res = self.list_records::(base_id, "Volunteers", query.clone()).await?; + volunteers .append(&mut res.records.into_iter().map(|data| data.fields).collect::>()); match res.offset { Some(next_offset) => query.offset = Some(next_offset), - _ => break, + None => break, } } @@ -152,7 +190,6 @@ impl AirtableClient for Airtable { let mut mentors = Vec::::with_capacity(100); loop { - println!("HERE"); let res = self.list_records::(base_id, "Volunteers", query.clone()).await?; mentors @@ -188,15 +225,15 @@ impl AirtableClient for Airtable { let mut nonprofits = Vec::::with_capacity(100); - while let Ok(res) = - self.list_records::(base_id, "Nonprofits", query.clone()).await - { + loop { + let res = self.list_records::(base_id, "Nonprofits", query.clone()).await?; + nonprofits .append(&mut res.records.into_iter().map(|data| data.fields).collect::>()); match res.offset { Some(next_offset) => query.offset = Some(next_offset), - _ => break, + None => break, } } @@ -204,22 +241,29 @@ impl AirtableClient for Airtable { } async fn get_mentor_mentee_linkages(&self, base_id: &str) -> Result> { - let query = ListRecordsQueryBuilder::default() + let mut query = ListRecordsQueryBuilder::default() .view(MENTOR_MENTEE_LINKAGE_VIEW.to_owned()) .fields(REQUIRED_MENTOR_MENTEE_LINKAGE_FIELDS.map(ToString::to_string).to_vec()) .build()?; - let data = self - .list_records::(base_id, "Volunteers", query) - .await? - .records - .into_iter() - .map(|data| data.fields) - .collect::>(); + let mut linkages = Vec::::with_capacity(100); + loop { + let res = self + .list_records::(base_id, "Volunteers", query.clone()) + .await?; + + linkages + .append(&mut res.records.into_iter().map(|data| data.fields).collect::>()); + + match res.offset { + Some(next_offset) => query.offset = Some(next_offset), + None => break, + } + } log::info!("Retrieved mentor-mentee linkages from Airtable"); - Ok(data) + Ok(linkages) } }