From 1d53952c8efd4513e614692417889e723645afb9 Mon Sep 17 00:00:00 2001 From: boxbeam Date: Fri, 26 Apr 2024 13:47:27 -0400 Subject: [PATCH] feat(webserver, db): track github/gitlab provider status with synced_at --- .../migrations/0022_github-provider.up.sql | 4 +- ...down.sql => 0027_gitlab-provider.down.sql} | 0 ...der.up.sql => 0027_gitlab-provider.up.sql} | 3 +- ...ithub-repository-columns-on-oauth.down.sql | 0 ...-github-repository-columns-on-oauth.up.sql | 7 --- ee/tabby-db/schema.sqlite | Bin 176128 -> 176128 bytes ee/tabby-db/src/github_repository_provider.rs | 17 +++++++- ee/tabby-db/src/gitlab_repository_provider.rs | 17 +++++++- ee/tabby-webserver/src/cron/github.rs | 5 ++- ee/tabby-webserver/src/cron/gitlab.rs | 5 ++- .../src/schema/github_repository_provider.rs | 10 +++-- .../src/schema/gitlab_repository_provider.rs | 10 +++-- ee/tabby-webserver/src/schema/types.rs | 19 +++++++- ee/tabby-webserver/src/service/dao.rs | 11 ++++- .../src/service/github_repository_provider.rs | 41 ++++++++---------- .../src/service/gitlab_repository_provider.rs | 37 ++++++---------- 16 files changed, 112 insertions(+), 74 deletions(-) rename ee/tabby-db/migrations/{0028_gitlab-provider.down.sql => 0027_gitlab-provider.down.sql} (100%) rename ee/tabby-db/migrations/{0028_gitlab-provider.up.sql => 0027_gitlab-provider.up.sql} (94%) delete mode 100644 ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.down.sql delete mode 100644 ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql diff --git a/ee/tabby-db/migrations/0022_github-provider.up.sql b/ee/tabby-db/migrations/0022_github-provider.up.sql index 0634b42cb7ac..b72697de3795 100644 --- a/ee/tabby-db/migrations/0022_github-provider.up.sql +++ b/ee/tabby-db/migrations/0022_github-provider.up.sql @@ -1,8 +1,6 @@ CREATE TABLE github_repository_provider( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, display_name TEXT NOT NULL, - application_id TEXT NOT NULL, - secret TEXT NOT NULL, access_token TEXT, - CONSTRAINT `idx_application_id` UNIQUE (`application_id`) + synced_at TIMESTAMP ); diff --git a/ee/tabby-db/migrations/0028_gitlab-provider.down.sql b/ee/tabby-db/migrations/0027_gitlab-provider.down.sql similarity index 100% rename from ee/tabby-db/migrations/0028_gitlab-provider.down.sql rename to ee/tabby-db/migrations/0027_gitlab-provider.down.sql diff --git a/ee/tabby-db/migrations/0028_gitlab-provider.up.sql b/ee/tabby-db/migrations/0027_gitlab-provider.up.sql similarity index 94% rename from ee/tabby-db/migrations/0028_gitlab-provider.up.sql rename to ee/tabby-db/migrations/0027_gitlab-provider.up.sql index 4c1103d69d07..f0e387f91ca8 100644 --- a/ee/tabby-db/migrations/0028_gitlab-provider.up.sql +++ b/ee/tabby-db/migrations/0027_gitlab-provider.up.sql @@ -1,7 +1,8 @@ CREATE TABLE gitlab_repository_provider( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, display_name TEXT NOT NULL, - access_token TEXT + access_token TEXT, + synced_at TIMESTAMP ); CREATE TABLE gitlab_provided_repositories( diff --git a/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.down.sql b/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.down.sql deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql b/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql deleted file mode 100644 index 424f642b519d..000000000000 --- a/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP TABLE github_repository_provider; - -CREATE TABLE github_repository_provider( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - display_name TEXT NOT NULL, - access_token TEXT -); diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite index 325abece2c60ba6cf0326220d74ca06681fe9f69..666a0bcbe2d842b1162706bb3a60e5a4c4536cfd 100644 GIT binary patch delta 1412 zcmbV~ZA@Eb6vyv*ZrckN@U~gj(ii$h%b>t$J5uI04Al``B4e2wukC0{OJTIdFv3O@ zM+t0dbm@GEPw*4eFKkORtO+$vaPxyRTgHswj4>v28Z$DpY&sT~DfrYkZa?_Nn|sf_ z`JHqB=YP(&`ki)Xb)~(!(&;ifYIZob?{L&Sy>>2L{vCS)Z(6$%n(__>N-Kv( z`@c~)DkonZsoK-@-nMH0c3H*ZoAoQ*p_5;I-nxGo%avw+LjmtnI?&E}+^5uwDN)*4 zPwzu~MOiAwzh&_19bBcV7UQVO&bBNE@RZ72yCHr3!v2G|&OboWQ+Eb`6Ju~5-w2a-iy>{^n*3!_Qzc=~F-S+h`URP!Ha}?Uy_V+)*!9t_B-zkbc z-*@9ng;}OoC_Xgrz&drAxVlkoXJ1zWHQQYbY;Kw?Vhzr|_9O&9D>@5-*ap99iRAbX|{9#d+#$AeA3spUP1Q6 zy9KHIRlYd)O0J4q;y(Z{FNK5PJw5FsulwLCFSE+16vH$ZlMX*Lfh9fX2ZaO#yv}LJ zy&%%hb-@y_qi!qtv>VJ&l3wWsBY=hQhoKRyr0p>H;jgzPmhnb2QTD=aGVVhPESE^q zeZ6oDfr;FX!UZ87HF7Gq8%9F|{y-!|=6gXyR-;Hmter?fj4>3~RPaW692tsrhJqdb z0po!^4fQV^a5wDZ)5#bdx|FPwpRbe1(~>$^6KuB`9?dXZm+YSOS=l-EzLaO7z=LM; z&-^*ozX8pp-13(ZYTlZe(?BMos3y-Ak5>Hchx`;kPZA+UUAQ_?T>sxL%0`v8^hcdY z1yGKE=X9ARnQ}@wRaqH~4dahxwdtl1(gT#y>`n!@#N=dL3~ZsDiXpBbRbjN5EOyb* z6h`{Y6HoHONCzdPG>o=FDQOO)T@r?o+#r{`!SsJtZbL%cUd5@}>mlCI6AOlpk<(FB kL~5h7*~17LX2(sH#O*`7L7qP0LmmWt3he6!x?Oty0f|tMjsO4v delta 1449 zcmb7@drVtZ9LM`RZRu!%woY35egNgQLR;8?xeBON9NV%)rh6FYm>ER3`Pc$k_J^BpL?@ocsJ)5*nB3pV%_raU z{eI6qx%+Mi_uUWf!|i8wxy>}Cxx2yK z;cjO-yi* zTlM^=jD)K6sj`NPTYtH3Dmd^`&&jLt2WwG3mdK34f4b-4EixcYq5#;yNakMk2f;*qbG_Uv3gxH0IQT^XJfe|mXB zHP1Y@`>neVn!SN4%MWRL78DnBcgmNqG}n|?;59ib&zeAu1M|2sK_8hnNb%0H1$-uf zReG7?n)hDCYDHS4x>8}`w)S7geTtMw9HiI|BlwmgITFV!srP5E;}Rtc_fTwqCkY={ z8Y0yTVSG-5*Qrk4F7848Z6M~B5_TyNUs#* z`Yaw&tAq7f$4(Cyilm8D5*{NRB^^ajYWYzd7?K2r3a6g~o+MQSa~z(8`z6eYTg6dv zW3fMp)(dBZf^}!(mSWCFuY!Yae zGD7!VNDi<8KN6_J{SA;7dd7_!0MaRXBDyP;?oNbWDPcS)p2X1w(h2b(P550%$~;I7 zDWuthGD)HbX~~oa>3DHtd^7pR1BU;9>=>g!-fD*4R&TGxN~H7 pNqV=N|31)3Kjf2Q7m{KLk4Hpp@LH5GEqq_0@iwp<0)-O#{0Uf5s~`XX diff --git a/ee/tabby-db/src/github_repository_provider.rs b/ee/tabby-db/src/github_repository_provider.rs index f6c666783a0c..8c9f191f662d 100644 --- a/ee/tabby-db/src/github_repository_provider.rs +++ b/ee/tabby-db/src/github_repository_provider.rs @@ -9,6 +9,7 @@ pub struct GithubRepositoryProviderDAO { pub id: i64, pub display_name: String, pub access_token: Option, + pub synced_at: Option, } #[derive(FromRow)] @@ -36,7 +37,7 @@ impl DbConn { pub async fn get_github_provider(&self, id: i64) -> Result { let provider = query_as!( GithubRepositoryProviderDAO, - "SELECT id, display_name, access_token FROM github_repository_provider WHERE id = ?;", + r#"SELECT id, display_name, access_token, synced_at AS "synced_at: DateTimeUtc" FROM github_repository_provider WHERE id = ?;"#, id ) .fetch_one(&self.pool) @@ -95,6 +96,18 @@ impl DbConn { Ok(()) } + pub async fn update_github_provider_synced_at(&self, id: i64, success: bool) -> Result<()> { + let time = success.then_some(DateTimeUtc::now()); + query!( + "UPDATE github_repository_provider SET synced_at = ? WHERE id = ?", + time, + id + ) + .execute(&self.pool) + .await?; + Ok(()) + } + pub async fn list_github_repository_providers( &self, ids: Vec, @@ -113,7 +126,7 @@ impl DbConn { let providers = query_paged_as!( GithubRepositoryProviderDAO, "github_repository_provider", - ["id", "display_name", "access_token"], + ["id", "display_name", "access_token", "synced_at" as "synced_at: DateTimeUtc"], limit, skip_id, backwards, diff --git a/ee/tabby-db/src/gitlab_repository_provider.rs b/ee/tabby-db/src/gitlab_repository_provider.rs index 509498333185..f26d70e4f961 100644 --- a/ee/tabby-db/src/gitlab_repository_provider.rs +++ b/ee/tabby-db/src/gitlab_repository_provider.rs @@ -9,6 +9,7 @@ pub struct GitlabRepositoryProviderDAO { pub id: i64, pub display_name: String, pub access_token: Option, + pub synced_at: Option, } #[derive(FromRow)] @@ -36,7 +37,7 @@ impl DbConn { pub async fn get_gitlab_provider(&self, id: i64) -> Result { let provider = query_as!( GitlabRepositoryProviderDAO, - "SELECT id, display_name, access_token FROM gitlab_repository_provider WHERE id = ?;", + r#"SELECT id, display_name, access_token, synced_at AS "synced_at: DateTimeUtc" FROM gitlab_repository_provider WHERE id = ?;"#, id ) .fetch_one(&self.pool) @@ -95,6 +96,18 @@ impl DbConn { Ok(()) } + pub async fn update_gitlab_provider_synced_at(&self, id: i64, success: bool) -> Result<()> { + let time = success.then_some(DateTimeUtc::now()); + query!( + "UPDATE gitlab_repository_provider SET synced_at = ? WHERE id = ?", + time, + id + ) + .execute(&self.pool) + .await?; + Ok(()) + } + pub async fn list_gitlab_repository_providers( &self, ids: Vec, @@ -113,7 +126,7 @@ impl DbConn { let providers = query_paged_as!( GitlabRepositoryProviderDAO, "gitlab_repository_provider", - ["id", "display_name", "access_token"], + ["id", "display_name", "access_token", "synced_at" as "synced_at: DateTimeUtc"], limit, skip_id, backwards, diff --git a/ee/tabby-webserver/src/cron/github.rs b/ee/tabby-webserver/src/cron/github.rs index 4d49a3a1e3fa..bef73e9f4b3b 100644 --- a/ee/tabby-webserver/src/cron/github.rs +++ b/ee/tabby-webserver/src/cron/github.rs @@ -38,7 +38,7 @@ async fn refresh_repositories_for_provider( .. }) if source.status_code.is_client_error() => { service - .reset_github_repository_provider_access_token(provider.id.clone()) + .update_github_repository_provider_sync_status(provider.id.clone(), false) .await?; warn!( "GitHub credentials for provider {} are expired or invalid", @@ -66,6 +66,9 @@ async fn refresh_repositories_for_provider( .upsert_github_provided_repository(provider.id.clone(), id, repo.name, url) .await?; } + service + .update_github_repository_provider_sync_status(provider.id.clone(), true) + .await?; Ok(()) } diff --git a/ee/tabby-webserver/src/cron/gitlab.rs b/ee/tabby-webserver/src/cron/gitlab.rs index 1639c673d14b..6c7e27ec28fd 100644 --- a/ee/tabby-webserver/src/cron/gitlab.rs +++ b/ee/tabby-webserver/src/cron/gitlab.rs @@ -39,7 +39,7 @@ async fn refresh_repositories_for_provider( Ok(repos) => repos, Err(e) if e.to_string().contains("401 Unauthorized") => { service - .reset_gitlab_repository_provider_access_token(provider.id.clone()) + .update_gitlab_repository_provider_sync_status(provider.id.clone(), false) .await?; warn!( "GitLab credentials for provider {} are expired or invalid", @@ -64,6 +64,9 @@ async fn refresh_repositories_for_provider( ) .await?; } + service + .update_gitlab_repository_provider_sync_status(provider.id.clone(), true) + .await?; Ok(()) } diff --git a/ee/tabby-webserver/src/schema/github_repository_provider.rs b/ee/tabby-webserver/src/schema/github_repository_provider.rs index fdf4ebaf319e..3412729f2b6e 100644 --- a/ee/tabby-webserver/src/schema/github_repository_provider.rs +++ b/ee/tabby-webserver/src/schema/github_repository_provider.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use juniper::{GraphQLObject, ID}; -use super::{repository::RepositoryProvider, Context}; +use super::{repository::RepositoryProvider, types::RepositoryProviderStatus, Context}; use crate::{juniper::relay::NodeType, schema::Result}; #[derive(GraphQLObject, Debug, PartialEq)] @@ -11,7 +11,7 @@ pub struct GithubRepositoryProvider { pub id: ID, pub display_name: String, - pub connected: bool, + pub status: RepositoryProviderStatus, #[graphql(skip)] pub access_token: Option, @@ -75,7 +75,11 @@ pub trait GithubRepositoryProviderService: Send + Sync + RepositoryProvider { display_name: String, access_token: String, ) -> Result<()>; - async fn reset_github_repository_provider_access_token(&self, id: ID) -> Result<()>; + async fn update_github_repository_provider_sync_status( + &self, + id: ID, + success: bool, + ) -> Result<()>; async fn list_github_repository_providers( &self, diff --git a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs index 71316f44aa6f..b446ddbe80ca 100644 --- a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs +++ b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use juniper::{GraphQLObject, ID}; -use super::{repository::RepositoryProvider, Context}; +use super::{repository::RepositoryProvider, types::RepositoryProviderStatus, Context}; use crate::{juniper::relay::NodeType, schema::Result}; #[derive(GraphQLObject, Debug, PartialEq)] @@ -11,7 +11,7 @@ pub struct GitlabRepositoryProvider { pub id: ID, pub display_name: String, - pub connected: bool, + pub status: RepositoryProviderStatus, #[graphql(skip)] pub access_token: Option, @@ -75,7 +75,11 @@ pub trait GitlabRepositoryProviderService: Send + Sync + RepositoryProvider { display_name: String, access_token: String, ) -> Result<()>; - async fn reset_gitlab_repository_provider_access_token(&self, id: ID) -> Result<()>; + async fn update_gitlab_repository_provider_sync_status( + &self, + id: ID, + success: bool, + ) -> Result<()>; async fn list_gitlab_repository_providers( &self, diff --git a/ee/tabby-webserver/src/schema/types.rs b/ee/tabby-webserver/src/schema/types.rs index 06ff4ceed7c7..49599e00f7ee 100644 --- a/ee/tabby-webserver/src/schema/types.rs +++ b/ee/tabby-webserver/src/schema/types.rs @@ -1,4 +1,4 @@ -use juniper::{GraphQLInputObject, ID}; +use juniper::{GraphQLEnum, GraphQLInputObject, ID}; use validator::Validate; #[derive(GraphQLInputObject, Validate)] @@ -23,3 +23,20 @@ pub struct UpdateRepositoryProviderInput { #[validate(length(code = "access_token", min = 10))] pub access_token: String, } + +#[derive(GraphQLEnum, Debug, PartialEq)] +pub enum RepositoryProviderStatus { + Ready, + Pending, + Error, +} + +impl RepositoryProviderStatus { + pub fn get_status(access_token_set: bool, synced_at_set: bool) -> Self { + match (access_token_set, synced_at_set) { + (true, true) => RepositoryProviderStatus::Ready, + (true, false) => RepositoryProviderStatus::Pending, + _ => RepositoryProviderStatus::Error, + } + } +} diff --git a/ee/tabby-webserver/src/service/dao.rs b/ee/tabby-webserver/src/service/dao.rs index 6d8b0dca8765..19ab7787d7ef 100644 --- a/ee/tabby-webserver/src/service/dao.rs +++ b/ee/tabby-webserver/src/service/dao.rs @@ -16,6 +16,7 @@ use crate::{ gitlab_repository_provider::{GitlabProvidedRepository, GitlabRepositoryProvider}, job, setting::{NetworkSetting, SecuritySetting}, + types::RepositoryProviderStatus, user_event::{EventKind, UserEvent}, CoreError, }, @@ -130,7 +131,10 @@ impl From for GithubRepositoryProvider { Self { display_name: value.display_name, id: value.id.as_id(), - connected: value.access_token.is_some(), + status: RepositoryProviderStatus::get_status( + value.access_token.is_some(), + value.synced_at.is_some(), + ), access_token: value.access_token, } } @@ -154,7 +158,10 @@ impl From for GitlabRepositoryProvider { Self { display_name: value.display_name, id: value.id.as_id(), - connected: value.access_token.is_some(), + status: RepositoryProviderStatus::get_status( + value.access_token.is_some(), + value.synced_at.is_some(), + ), access_token: value.access_token, } } diff --git a/ee/tabby-webserver/src/service/github_repository_provider.rs b/ee/tabby-webserver/src/service/github_repository_provider.rs index fb78553415fb..7dbf57ed4f32 100644 --- a/ee/tabby-webserver/src/service/github_repository_provider.rs +++ b/ee/tabby-webserver/src/service/github_repository_provider.rs @@ -50,13 +50,6 @@ impl GithubRepositoryProviderService for GithubRepositoryProviderServiceImpl { Ok(()) } - async fn reset_github_repository_provider_access_token(&self, id: ID) -> Result<()> { - self.db - .reset_github_provider_access_token(id.as_rowid()?) - .await?; - Ok(()) - } - async fn list_github_repository_providers( &self, ids: Vec, @@ -183,6 +176,21 @@ impl GithubRepositoryProviderService for GithubRepositoryProviderServiceImpl { .await?; Ok(()) } + + async fn update_github_repository_provider_sync_status( + &self, + id: ID, + success: bool, + ) -> Result<()> { + let id = id.as_rowid()?; + if !success { + self.db.reset_github_provider_access_token(id).await?; + } + self.db + .update_github_provider_synced_at(id, success) + .await?; + Ok(()) + } } #[async_trait] @@ -217,7 +225,7 @@ mod tests { use chrono::Duration; use super::*; - use crate::service::AsID; + use crate::{schema::types::RepositoryProviderStatus, service::AsID}; #[tokio::test] async fn test_github_provided_repositories() { @@ -318,7 +326,7 @@ mod tests { id: id.clone(), display_name: "id".into(), access_token: Some("secret".into()), - connected: true, + status: RepositoryProviderStatus::Pending, } ); @@ -330,21 +338,6 @@ mod tests { assert_eq!(providers.len(), 1); assert_eq!(providers[0].access_token, Some("secret".into())); - // Test resetgithub provider tokens - service - .reset_github_repository_provider_access_token(id.clone()) - .await - .unwrap(); - - assert_eq!( - service - .get_github_repository_provider(id.clone()) - .await - .unwrap() - .access_token, - None - ); - // Test deleting github provider service .delete_github_repository_provider(id.clone()) diff --git a/ee/tabby-webserver/src/service/gitlab_repository_provider.rs b/ee/tabby-webserver/src/service/gitlab_repository_provider.rs index 5c4d328d75cd..4e75a187ce34 100644 --- a/ee/tabby-webserver/src/service/gitlab_repository_provider.rs +++ b/ee/tabby-webserver/src/service/gitlab_repository_provider.rs @@ -50,13 +50,6 @@ impl GitlabRepositoryProviderService for GitlabRepositoryProviderServiceImpl { Ok(()) } - async fn reset_gitlab_repository_provider_access_token(&self, id: ID) -> Result<()> { - self.db - .reset_gitlab_provider_access_token(id.as_rowid()?) - .await?; - Ok(()) - } - async fn list_gitlab_repository_providers( &self, ids: Vec, @@ -186,6 +179,17 @@ impl GitlabRepositoryProviderService for GitlabRepositoryProviderServiceImpl { .await?; Ok(()) } + + async fn update_gitlab_repository_provider_sync_status( + &self, + id: ID, + success: bool, + ) -> Result<()> { + self.db + .update_gitlab_provider_synced_at(id.as_rowid()?, success) + .await?; + Ok(()) + } } fn deduplicate_gitlab_repositories(repositories: &mut Vec) { @@ -220,7 +224,7 @@ mod tests { use chrono::Duration; use super::*; - use crate::service::AsID; + use crate::{schema::types::RepositoryProviderStatus, service::AsID}; #[tokio::test] async fn test_gitlab_provided_repositories() { @@ -321,7 +325,7 @@ mod tests { id: id.clone(), display_name: "id".into(), access_token: Some("secret".into()), - connected: true, + status: RepositoryProviderStatus::Pending } ); @@ -333,21 +337,6 @@ mod tests { assert_eq!(providers.len(), 1); assert_eq!(providers[0].access_token, Some("secret".into())); - // Test resetgitlab provider tokens - service - .reset_gitlab_repository_provider_access_token(id.clone()) - .await - .unwrap(); - - assert_eq!( - service - .get_gitlab_repository_provider(id.clone()) - .await - .unwrap() - .access_token, - None - ); - // Test deleting gitlab provider service .delete_gitlab_repository_provider(id.clone())