From 7a576b50c3261d30c32e1cdde86ccf273e19f4fa Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 15:34:10 -0400 Subject: [PATCH 1/9] Copy code --- Cargo.lock | 135 +++++- .../migrations/0028_gitlab-provider.down.sql | 1 + .../migrations/0028_gitlab-provider.up.sql | 19 + ee/tabby-db/schema.sqlite | Bin 172032 -> 176128 bytes ee/tabby-db/src/gitlab_repository_provider.rs | 224 +++++++++ ee/tabby-db/src/lib.rs | 2 + ee/tabby-webserver/Cargo.toml | 1 + ee/tabby-webserver/src/cron/gitlab.rs | 103 +++++ ee/tabby-webserver/src/cron/mod.rs | 1 + .../src/schema/gitlab_repository_provider.rs | 136 ++++++ ee/tabby-webserver/src/schema/mod.rs | 1 + ee/tabby-webserver/src/service/dao.rs | 30 +- .../src/service/gitlab_repository_provider.rs | 426 ++++++++++++++++++ ee/tabby-webserver/src/service/mod.rs | 1 + 14 files changed, 1072 insertions(+), 8 deletions(-) create mode 100644 ee/tabby-db/migrations/0028_gitlab-provider.down.sql create mode 100644 ee/tabby-db/migrations/0028_gitlab-provider.up.sql create mode 100644 ee/tabby-db/src/gitlab_repository_provider.rs create mode 100644 ee/tabby-webserver/src/cron/gitlab.rs create mode 100644 ee/tabby-webserver/src/schema/gitlab_repository_provider.rs create mode 100644 ee/tabby-webserver/src/service/gitlab_repository_provider.rs diff --git a/Cargo.lock b/Cargo.lock index 70edc68b5347..8f32518d469a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -204,7 +204,7 @@ dependencies = [ "backoff", "base64 0.21.5", "bytes", - "derive_builder", + "derive_builder 0.12.0", "futures", "rand 0.8.5", "reqwest", @@ -1007,13 +1007,34 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_builder" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3" +dependencies = [ + "derive_builder_macro 0.11.2", +] + [[package]] name = "derive_builder" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" dependencies = [ - "derive_builder_macro", + "derive_builder_macro 0.12.0", +] + +[[package]] +name = "derive_builder_core" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -1028,13 +1049,23 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_macro" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68" +dependencies = [ + "derive_builder_core 0.11.2", + "syn 1.0.109", +] + [[package]] name = "derive_builder_macro" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ - "derive_builder_core", + "derive_builder_core 0.12.0", "syn 1.0.109", ] @@ -1486,6 +1517,32 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +[[package]] +name = "gitlab" +version = "0.1610.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c802fc7eb82ff5ba2e4447c5acd0f18ec1b7bb95dbe95b6d77639e25be7cbe" +dependencies = [ + "async-trait", + "base64 0.13.1", + "bytes", + "chrono", + "cron", + "derive_builder 0.11.2", + "futures-util", + "graphql_client", + "http 0.2.11", + "itertools 0.10.5", + "log", + "percent-encoding", + "reqwest", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "url", +] + [[package]] name = "glob" version = "0.3.1" @@ -1515,6 +1572,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "graphql-introspection-query" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2a4732cf5140bd6c082434494f785a19cfb566ab07d1382c3671f5812fed6d" +dependencies = [ + "serde", +] + [[package]] name = "graphql-parser" version = "0.3.0" @@ -1525,6 +1591,55 @@ dependencies = [ "thiserror", ] +[[package]] +name = "graphql-parser" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ebc8013b4426d5b81a4364c419a95ed0b404af2b82e2457de52d9348f0e474" +dependencies = [ + "combine", + "thiserror", +] + +[[package]] +name = "graphql_client" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc16d75d169fddb720d8f1c7aed6413e329e1584079b9734ff07266a193f5bc" +dependencies = [ + "graphql_query_derive", + "serde", + "serde_json", +] + +[[package]] +name = "graphql_client_codegen" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f290ecfa3bea3e8a157899dc8a1d96ee7dd6405c18c8ddd213fc58939d18a0e9" +dependencies = [ + "graphql-introspection-query", + "graphql-parser 0.4.0", + "heck", + "lazy_static", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 1.0.109", +] + +[[package]] +name = "graphql_query_derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a755cc59cda2641ea3037b4f9f7ef40471c329f55c1fa2db6fa0bb7ae6c1f7ce" +dependencies = [ + "graphql_client_codegen", + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "h2" version = "0.3.19" @@ -2149,7 +2264,7 @@ dependencies = [ "fnv", "futures", "futures-enum", - "graphql-parser", + "graphql-parser 0.3.0", "indexmap 1.9.3", "juniper_codegen", "serde", @@ -2324,7 +2439,7 @@ dependencies = [ "cmake", "cxx", "cxx-build", - "derive_builder", + "derive_builder 0.12.0", "futures", "tabby-inference", "tokio", @@ -3616,6 +3731,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", + "webpki-roots", "winreg", ] @@ -4852,7 +4968,7 @@ dependencies = [ "async-stream", "async-trait", "dashmap", - "derive_builder", + "derive_builder 0.12.0", "futures", "tabby-common", "trie-rs", @@ -4921,6 +5037,7 @@ dependencies = [ "chrono", "fs_extra", "futures", + "gitlab", "hash-ids", "hyper 0.14.27", "jsonwebtoken", @@ -6304,6 +6421,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "which" version = "4.4.0" diff --git a/ee/tabby-db/migrations/0028_gitlab-provider.down.sql b/ee/tabby-db/migrations/0028_gitlab-provider.down.sql new file mode 100644 index 000000000000..d2f607c5b8bd --- /dev/null +++ b/ee/tabby-db/migrations/0028_gitlab-provider.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/ee/tabby-db/migrations/0028_gitlab-provider.up.sql b/ee/tabby-db/migrations/0028_gitlab-provider.up.sql new file mode 100644 index 000000000000..4c1103d69d07 --- /dev/null +++ b/ee/tabby-db/migrations/0028_gitlab-provider.up.sql @@ -0,0 +1,19 @@ +CREATE TABLE gitlab_repository_provider( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + display_name TEXT NOT NULL, + access_token TEXT +); + +CREATE TABLE gitlab_provided_repositories( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + gitlab_repository_provider_id INTEGER NOT NULL, + -- vendor_id from https://docs.gitlab.com/ee/api/repositories.html + vendor_id TEXT NOT NULL, + name TEXT NOT NULL, + git_url TEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMP NOT NULL DEFAULT (DATETIME('now')), + FOREIGN KEY (gitlab_repository_provider_id) REFERENCES gitlab_repository_provider(id) ON DELETE CASCADE, + CONSTRAINT `idx_vendor_id_provider_id` UNIQUE (vendor_id, gitlab_repository_provider_id) +); +CREATE INDEX gitlab_provided_repositories_updated_at ON gitlab_provided_repositories(updated_at); diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite index 649193aaeb4ae0b771f648569572029d1557eb69..8eaacca689da1d5cc4a3318c7da8ad5fa0a779fc 100644 GIT binary patch delta 756 zcmYjOZ%7ki9KQG6-A-?RZZdVc&O9>=nftFy4P+~?KTsC2${e;$J>A*eHntIcoIxL= z!r6+0K15EaPlY<-Xwn+=F(c?pBvfidAw|+3G$`z}D|+F5U!LFl{CJ<|`K3!ldXdO% zt3VM1vB>-HvQ4zO0Xh@9LF$_1T6MKfJ}JzT#SI>adGbEP8X7uH;lSc9=TT~eOYz|W zYdjG<%MGy!m&4U+b+lStZM3t!)6v%Hb}cPf##ingSon1%`(@?&OWO70{mtV2p(D

RRW=+$i6I2dm!1D_t9oD8?vL2qVgg8 ztf+y1VVSuWnpda)?6@r;Wp${aaZ&9EHQhP6$$ITj0L{Nk387PuxNVR@S_fBIZ z3ItyESlS)}w=p?3xRKq!&C)UpmY_t9z(q`&io*VeT*i>g(7B8w=QYfFQ9~Y0|E!>oB6EsE8HJ>5*NVpYyyi@5cv?0YA0h`}(Z-_#(#QOa!s260}1FXSnKe(6pW Jo^r~=_ziH@@3{Z~ delta 481 zcmZ`#&ubGw6rMLT`;*;+{iCj3%L_GUyx4md+ShY?I zXK+|AJG73x=S;f3cg^Y1x|2C;=4I=LwQYYL0}tV+y=5mTWryw8F;5raBOk25)oj5k zWCJ>>@+oYWWcenzAo}D1$odVqg)ux4J9>l2!V-Mf@+S03D7 zieXEm)^Dg}e~i2uj;G!=ppRv#2A6;$M4!otCX~s4m-A!^9*!??pc~SnpsmsDW*Tl* z{ce@|9U3Yt92L4%@MrW2Vq_k%Ceyu+vtX@T$IaL{t-sN)&{R7h^R(vf(VB?-V?^#O z<6qcxM#xkjL;P(6f0O++*w=I=Ki`isFemwkCInC!r@Vg_A-FyfMewxr=, +} + +#[derive(FromRow)] +pub struct GitlabProvidedRepositoryDAO { + pub id: i64, + pub vendor_id: String, + pub gitlab_repository_provider_id: i64, + pub name: String, + pub git_url: String, + pub active: bool, +} + +impl DbConn { + pub async fn create_gitlab_provider(&self, name: String, access_token: String) -> Result { + let res = query!( + "INSERT INTO gitlab_repository_provider (display_name, access_token) VALUES ($1, $2);", + name, + access_token + ) + .execute(&self.pool) + .await?; + Ok(res.last_insert_rowid()) + } + + pub async fn get_gitlab_provider(&self, id: i64) -> Result { + let provider = query_as!( + GitlabRepositoryProviderDAO, + "SELECT id, display_name, access_token FROM gitlab_repository_provider WHERE id = ?;", + id + ) + .fetch_one(&self.pool) + .await?; + Ok(provider) + } + + pub async fn delete_gitlab_provider(&self, id: i64) -> Result<()> { + let res = query!("DELETE FROM gitlab_repository_provider WHERE id = ?;", id) + .execute(&self.pool) + .await?; + if res.rows_affected() != 1 { + return Err(anyhow!("No gitlab provider details to delete")); + } + Ok(()) + } + + pub async fn reset_gitlab_provider_access_token(&self, id: i64) -> Result<()> { + let res = query!( + "UPDATE gitlab_repository_provider SET access_token = NULL WHERE id = ?", + id + ) + .execute(&self.pool) + .await?; + + if res.rows_affected() != 1 { + return Err(anyhow!( + "The specified gitlab repository provider does not exist" + )); + } + + Ok(()) + } + + pub async fn update_gitlab_provider( + &self, + id: i64, + display_name: String, + access_token: String, + ) -> Result<()> { + let res = query!( + "UPDATE gitlab_repository_provider SET display_name = ?, access_token=? WHERE id = ?;", + display_name, + access_token, + id + ) + .execute(&self.pool) + .await?; + + if res.rows_affected() != 1 { + return Err(anyhow!( + "The specified gitlab repository provider does not exist" + )); + } + + Ok(()) + } + + pub async fn list_gitlab_repository_providers( + &self, + ids: Vec, + limit: Option, + skip_id: Option, + backwards: bool, + ) -> Result> { + let condition = (!ids.is_empty()).then(|| { + let ids = ids + .into_iter() + .map(|id| id.to_string()) + .collect::>() + .join(", "); + format!("id in ({ids})") + }); + let providers = query_paged_as!( + GitlabRepositoryProviderDAO, + "gitlab_repository_provider", + ["id", "display_name", "access_token"], + limit, + skip_id, + backwards, + condition + ) + .fetch_all(&self.pool) + .await?; + Ok(providers) + } + + pub async fn upsert_gitlab_provided_repository( + &self, + gitlab_provider_id: i64, + vendor_id: String, + name: String, + git_url: String, + ) -> Result { + let res = query!( + "INSERT INTO gitlab_provided_repositories (gitlab_repository_provider_id, vendor_id, name, git_url) VALUES ($1, $2, $3, $4) + ON CONFLICT(gitlab_repository_provider_id, vendor_id) DO UPDATE SET name = $3, git_url = $4, updated_at = DATETIME('now')", + gitlab_provider_id, + vendor_id, + name, + git_url + ).execute(&self.pool).await?; + Ok(res.last_insert_rowid()) + } + + pub async fn delete_gitlab_provided_repository(&self, id: i64) -> Result<()> { + let res = query!("DELETE FROM gitlab_provided_repositories WHERE id = ?", id) + .execute(&self.pool) + .await?; + + if res.rows_affected() != 1 { + return Err(anyhow!("Repository not found")); + } + Ok(()) + } + + pub async fn delete_outdated_gitlab_repositories( + &self, + gitlab_provider_id: i64, + cutoff_timestamp: DateTimeUtc, + ) -> Result<()> { + query!( + "DELETE FROM gitlab_provided_repositories WHERE gitlab_repository_provider_id = ? AND updated_at < ?;", + gitlab_provider_id, + cutoff_timestamp + ).execute(&self.pool).await?; + Ok(()) + } + + pub async fn list_gitlab_provided_repositories( + &self, + provider_ids: Vec, + limit: Option, + skip_id: Option, + backwards: bool, + ) -> Result> { + let provider_ids = provider_ids + .into_iter() + .map(|id| id.to_string()) + .collect::>() + .join(", "); + let repos = query_paged_as!( + GitlabProvidedRepositoryDAO, + "gitlab_provided_repositories", + [ + "id", + "vendor_id", + "name", + "git_url", + "active", + "gitlab_repository_provider_id" + ], + limit, + skip_id, + backwards, + (!provider_ids.is_empty()) + .then(|| format!("gitlab_repository_provider_id IN ({provider_ids})")) + ) + .fetch_all(&self.pool) + .await?; + Ok(repos) + } + + pub async fn update_gitlab_provided_repository_active( + &self, + id: i64, + active: bool, + ) -> Result<()> { + let not_active = !active; + let res = query!( + "UPDATE gitlab_provided_repositories SET active = ? WHERE id = ? AND active = ?", + active, + id, + not_active + ) + .execute(&self.pool) + .await?; + + if res.rows_affected() != 1 { + return Err(anyhow!("Repository active status was not changed")); + } + + Ok(()) + } +} diff --git a/ee/tabby-db/src/lib.rs b/ee/tabby-db/src/lib.rs index 31d803546bad..f855953582a0 100644 --- a/ee/tabby-db/src/lib.rs +++ b/ee/tabby-db/src/lib.rs @@ -11,6 +11,7 @@ use cached::TimedSizedCache; use chrono::{DateTime, Duration, NaiveDateTime, Utc}; pub use email_setting::EmailSettingDAO; pub use github_repository_provider::{GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO}; +pub use gitlab_repository_provider::{GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO}; pub use invitations::InvitationDAO; pub use job_runs::JobRunDAO; pub use oauth_credential::OAuthCredentialDAO; @@ -28,6 +29,7 @@ pub use users::UserDAO; pub mod cache; mod email_setting; mod github_repository_provider; +mod gitlab_repository_provider; mod invitations; mod job_runs; mod oauth_credential; diff --git a/ee/tabby-webserver/Cargo.toml b/ee/tabby-webserver/Cargo.toml index 842d4e3dad75..8205406e730b 100644 --- a/ee/tabby-webserver/Cargo.toml +++ b/ee/tabby-webserver/Cargo.toml @@ -52,6 +52,7 @@ regex.workspace = true tabby-search = { path = "../tabby-search" } octocrab = "0.38.0" fs_extra = "1.3.0" +gitlab = "0.1610.0" [dev-dependencies] assert_matches = "1.5.0" diff --git a/ee/tabby-webserver/src/cron/gitlab.rs b/ee/tabby-webserver/src/cron/gitlab.rs new file mode 100644 index 000000000000..d38bb5cbf366 --- /dev/null +++ b/ee/tabby-webserver/src/cron/gitlab.rs @@ -0,0 +1,103 @@ +use std::sync::Arc; + +use anyhow::Result; +use chrono::Utc; +use gitlab::AsyncGitlab; +use juniper::ID; +use octocrab::{models::Repository, GitHubError, Octocrab}; +use tracing::warn; + +use crate::schema::{ + github_repository_provider::{GithubRepositoryProvider, GithubRepositoryProviderService}, + gitlab_repository_provider::{GitlabRepositoryProvider, GitlabRepositoryProviderService}, +}; + +pub async fn refresh_all_repositories( + service: Arc, +) -> Result<()> { + for provider in service + .list_gitlab_repository_providers(vec![], None, None, None, None) + .await? + { + let start = Utc::now(); + refresh_repositories_for_provider(service.clone(), provider.id.clone()).await?; + service + .delete_outdated_gitlab_provided_repositories(provider.id, start) + .await?; + } + Ok(()) +} + +async fn refresh_repositories_for_provider( + service: Arc, + provider_id: ID, +) -> Result<()> { + let provider = service.get_gitlab_repository_provider(provider_id).await?; + let repos = match fetch_all_repos(&provider).await { + Ok(repos) => repos, + Err(octocrab::Error::GitHub { + source: source @ GitHubError { .. }, + .. + }) if source.status_code.is_client_error() => { + service + .reset_gitlab_repository_provider_access_token(provider.id.clone()) + .await?; + warn!( + "GitLab credentials for provider {} are expired or invalid", + provider.display_name + ); + return Err(source.into()); + } + Err(e) => { + warn!("Failed to fetch repositories from github: {e}"); + return Err(e.into()); + } + }; + for repo in repos { + let id = repo.id.to_string(); + let Some(url) = repo.git_url else { + continue; + }; + let url = url.to_string(); + + service + .upsert_gitlab_provided_repository(provider.id.clone(), id, repo.name, url) + .await?; + } + + Ok(()) +} + +// FIXME(wsxiaoys): Convert to async stream +async fn fetch_all_repos( + provider: &GitlabRepositoryProvider, +) -> Result, octocrab::Error> { + let Some(token) = &provider.access_token else { + return Ok(vec![]); + }; + let octocrab = Octocrab::builder() + .user_access_token(token.to_string()) + .build()?; + + let mut page = 1; + let mut repos = vec![]; + + loop { + let response = octocrab + .current() + .list_repos_for_authenticated_user() + .visibility("all") + .page(page) + .send() + .await?; + + let pages = response.number_of_pages().unwrap_or_default() as u8; + repos.extend(response.items); + + page += 1; + if page > pages { + break; + } + } + Ok(repos) +} diff --git a/ee/tabby-webserver/src/cron/mod.rs b/ee/tabby-webserver/src/cron/mod.rs index ccb1de92f43b..982e3885f6a2 100644 --- a/ee/tabby-webserver/src/cron/mod.rs +++ b/ee/tabby-webserver/src/cron/mod.rs @@ -1,5 +1,6 @@ mod db; mod github; +mod gitlab; mod scheduler; use std::sync::Arc; diff --git a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs new file mode 100644 index 000000000000..2cff5cd7f7aa --- /dev/null +++ b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs @@ -0,0 +1,136 @@ +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use juniper::{GraphQLInputObject, GraphQLObject, ID}; +use lazy_static::lazy_static; +use regex::Regex; +use validator::Validate; + +use super::Context; +use crate::{juniper::relay::NodeType, schema::Result}; + +lazy_static! { + static ref GITHUB_REPOSITORY_PROVIDER_NAME_REGEX: Regex = Regex::new("^[\\w-]+$").unwrap(); +} + +#[derive(GraphQLInputObject, Validate)] +pub struct CreateGitlabRepositoryProviderInput { + #[validate(regex(code = "displayName", path = "GITHUB_REPOSITORY_PROVIDER_NAME_REGEX"))] + pub display_name: String, + #[validate(length(code = "access_token", min = 10))] + pub access_token: String, +} + +#[derive(GraphQLInputObject, Validate)] +pub struct UpdateGitlabRepositoryProviderInput { + pub id: ID, + #[validate(regex(code = "displayName", path = "GITHUB_REPOSITORY_PROVIDER_NAME_REGEX"))] + pub display_name: String, + #[validate(length(code = "access_token", min = 10))] + pub access_token: String, +} + +#[derive(GraphQLObject, Debug, PartialEq)] +#[graphql(context = Context)] +pub struct GitlabRepositoryProvider { + pub id: ID, + pub display_name: String, + + pub connected: bool, + + #[graphql(skip)] + pub access_token: Option, +} + +impl NodeType for GitlabRepositoryProvider { + type Cursor = String; + + fn cursor(&self) -> Self::Cursor { + self.id.to_string() + } + + fn connection_type_name() -> &'static str { + "GitlabRepositoryProviderConnection" + } + + fn edge_type_name() -> &'static str { + "GitlabRepositoryProviderEdge" + } +} + +#[derive(GraphQLObject, Debug)] +#[graphql(context = Context)] +pub struct GitlabProvidedRepository { + pub id: ID, + pub vendor_id: String, + pub gitlab_repository_provider_id: ID, + pub name: String, + pub git_url: String, + pub active: bool, +} + +impl NodeType for GitlabProvidedRepository { + type Cursor = String; + + fn cursor(&self) -> Self::Cursor { + self.id.to_string() + } + + fn connection_type_name() -> &'static str { + "GitlabProvidedRepositoryConnection" + } + + fn edge_type_name() -> &'static str { + "GitlabProvidedRepositoryEdge" + } +} + +#[async_trait] +pub trait GitlabRepositoryProviderService: Send + Sync { + async fn create_gitlab_repository_provider( + &self, + display_name: String, + access_token: String, + ) -> Result; + async fn get_gitlab_repository_provider(&self, id: ID) -> Result; + async fn delete_gitlab_repository_provider(&self, id: ID) -> Result<()>; + async fn update_gitlab_repository_provider( + &self, + id: ID, + display_name: String, + access_token: String, + ) -> Result<()>; + async fn reset_gitlab_repository_provider_access_token(&self, id: ID) -> Result<()>; + + async fn list_gitlab_repository_providers( + &self, + ids: Vec, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result>; + + async fn list_gitlab_provided_repositories_by_provider( + &self, + provider: Vec, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result>; + + async fn upsert_gitlab_provided_repository( + &self, + provider_id: ID, + vendor_id: String, + display_name: String, + git_url: String, + ) -> Result<()>; + async fn update_gitlab_provided_repository_active(&self, id: ID, active: bool) -> Result<()>; + async fn list_provided_git_urls(&self) -> Result>; + async fn delete_outdated_gitlab_provided_repositories( + &self, + provider_id: ID, + cutoff_timestamp: DateTime, + ) -> Result<()>; +} diff --git a/ee/tabby-webserver/src/schema/mod.rs b/ee/tabby-webserver/src/schema/mod.rs index efe69d5935da..decaa117fbc7 100644 --- a/ee/tabby-webserver/src/schema/mod.rs +++ b/ee/tabby-webserver/src/schema/mod.rs @@ -3,6 +3,7 @@ pub mod auth; pub mod email; pub mod git_repository; pub mod github_repository_provider; +pub mod gitlab_repository_provider; pub mod job; pub mod license; pub mod repository; diff --git a/ee/tabby-webserver/src/service/dao.rs b/ee/tabby-webserver/src/service/dao.rs index a65bf9a616ca..6d8b0dca8765 100644 --- a/ee/tabby-webserver/src/service/dao.rs +++ b/ee/tabby-webserver/src/service/dao.rs @@ -1,8 +1,9 @@ use hash_ids::HashIds; use lazy_static::lazy_static; use tabby_db::{ - EmailSettingDAO, GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO, InvitationDAO, - JobRunDAO, OAuthCredentialDAO, RepositoryDAO, ServerSettingDAO, UserDAO, UserEventDAO, + EmailSettingDAO, GithubProvidedRepositoryDAO, GithubRepositoryProviderDAO, + GitlabProvidedRepositoryDAO, GitlabRepositoryProviderDAO, InvitationDAO, JobRunDAO, + OAuthCredentialDAO, RepositoryDAO, ServerSettingDAO, UserDAO, UserEventDAO, }; use crate::{ @@ -12,6 +13,7 @@ use crate::{ email::{AuthMethod, EmailSetting, Encryption}, git_repository::GitRepository, github_repository_provider::{GithubProvidedRepository, GithubRepositoryProvider}, + gitlab_repository_provider::{GitlabProvidedRepository, GitlabRepositoryProvider}, job, setting::{NetworkSetting, SecuritySetting}, user_event::{EventKind, UserEvent}, @@ -147,6 +149,30 @@ impl From for GithubProvidedRepository { } } +impl From for GitlabRepositoryProvider { + fn from(value: GitlabRepositoryProviderDAO) -> Self { + Self { + display_name: value.display_name, + id: value.id.as_id(), + connected: value.access_token.is_some(), + access_token: value.access_token, + } + } +} + +impl From for GitlabProvidedRepository { + fn from(value: GitlabProvidedRepositoryDAO) -> Self { + Self { + id: value.id.as_id(), + gitlab_repository_provider_id: value.gitlab_repository_provider_id.as_id(), + name: value.name, + git_url: value.git_url, + vendor_id: value.vendor_id, + active: value.active, + } + } +} + impl TryFrom for UserEvent { type Error = anyhow::Error; fn try_from(value: UserEventDAO) -> Result { diff --git a/ee/tabby-webserver/src/service/gitlab_repository_provider.rs b/ee/tabby-webserver/src/service/gitlab_repository_provider.rs new file mode 100644 index 000000000000..d63bc43f4bc7 --- /dev/null +++ b/ee/tabby-webserver/src/service/gitlab_repository_provider.rs @@ -0,0 +1,426 @@ +use std::collections::{HashMap, HashSet}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use juniper::ID; +use tabby_db::DbConn; +use url::Url; + +use super::{AsID, AsRowid}; +use crate::{ + schema::{ + gitlab_repository_provider::{ + GitlabProvidedRepository, GitlabRepositoryProvider, GitlabRepositoryProviderService, + }, + Result, + }, + service::graphql_pagination_to_filter, +}; + +struct GitlabRepositoryProviderServiceImpl { + db: DbConn, +} + +pub fn create(db: DbConn) -> impl GitlabRepositoryProviderService { + GitlabRepositoryProviderServiceImpl { db } +} + +#[async_trait] +impl GitlabRepositoryProviderService for GitlabRepositoryProviderServiceImpl { + async fn create_gitlab_repository_provider( + &self, + display_name: String, + access_token: String, + ) -> Result { + let id = self + .db + .create_gitlab_provider(display_name, access_token) + .await?; + Ok(id.as_id()) + } + + async fn get_gitlab_repository_provider(&self, id: ID) -> Result { + let provider = self.db.get_gitlab_provider(id.as_rowid()?).await?; + Ok(provider.into()) + } + + async fn delete_gitlab_repository_provider(&self, id: ID) -> Result<()> { + self.db.delete_gitlab_provider(id.as_rowid()?).await?; + Ok(()) + } + + async fn reset_gitlab_repository_provider_access_token(&self, id: ID) -> Result<()> { + self.db + .reset_gitlab_provider_access_token(id.as_rowid()?) + .await?; + Ok(()) + } + + async fn list_gitlab_repository_providers( + &self, + ids: Vec, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result> { + let (limit, skip_id, backwards) = graphql_pagination_to_filter(after, before, first, last)?; + + let ids = ids + .into_iter() + .map(|id| id.as_rowid()) + .collect::, _>>()?; + + let providers = self + .db + .list_gitlab_repository_providers(ids, limit, skip_id, backwards) + .await?; + Ok(providers + .into_iter() + .map(GitlabRepositoryProvider::from) + .collect()) + } + + async fn list_gitlab_provided_repositories_by_provider( + &self, + providers: Vec, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result> { + let providers = providers + .into_iter() + .map(|i| i.as_rowid()) + .collect::, _>>()?; + let (limit, skip_id, backwards) = graphql_pagination_to_filter(after, before, first, last)?; + let repos = self + .db + .list_gitlab_provided_repositories(providers, limit, skip_id, backwards) + .await?; + + Ok(repos + .into_iter() + .map(GitlabProvidedRepository::from) + .collect()) + } + + async fn upsert_gitlab_provided_repository( + &self, + provider_id: ID, + vendor_id: String, + display_name: String, + git_url: String, + ) -> Result<()> { + self.db + .upsert_gitlab_provided_repository( + provider_id.as_rowid()?, + vendor_id, + display_name, + git_url, + ) + .await?; + Ok(()) + } + + async fn update_gitlab_provided_repository_active(&self, id: ID, active: bool) -> Result<()> { + self.db + .update_gitlab_provided_repository_active(id.as_rowid()?, active) + .await?; + Ok(()) + } + + async fn update_gitlab_repository_provider( + &self, + id: ID, + display_name: String, + access_token: String, + ) -> Result<()> { + self.db + .update_gitlab_provider(id.as_rowid()?, display_name, access_token) + .await?; + Ok(()) + } + + async fn list_provided_git_urls(&self) -> Result> { + let tokens: HashMap = self + .list_gitlab_repository_providers(vec![], None, None, None, None) + .await? + .into_iter() + .filter_map(|provider| Some((provider.id.to_string(), provider.access_token?))) + .collect(); + + let mut repos = self + .list_gitlab_provided_repositories_by_provider(vec![], None, None, None, None) + .await?; + + deduplicate_gitlab_repositories(&mut repos); + + let urls = repos + .into_iter() + .filter_map(|repo| { + if !repo.active { + return None; + } + let mut url = Url::parse(&repo.git_url).ok()?; + url.set_username("oauth2").ok()?; + url.set_password(Some( + tokens.get(&repo.gitlab_repository_provider_id.to_string())?, + )) + .ok()?; + Some(url.to_string()) + }) + .collect(); + + Ok(urls) + } + + async fn delete_outdated_gitlab_provided_repositories( + &self, + provider_id: ID, + cutoff_timestamp: DateTime, + ) -> Result<()> { + self.db + .delete_outdated_gitlab_repositories(provider_id.as_rowid()?, cutoff_timestamp.into()) + .await?; + Ok(()) + } +} + +fn deduplicate_gitlab_repositories(repositories: &mut Vec) { + let mut vendor_ids = HashSet::new(); + repositories.retain(|repo| vendor_ids.insert(repo.vendor_id.clone())); +} + +#[cfg(test)] +mod tests { + use chrono::Duration; + + use super::*; + use crate::service::AsID; + + #[tokio::test] + async fn test_gitlab_provided_repositories() { + let db = DbConn::new_in_memory().await.unwrap(); + let service = create(db.clone()); + + let provider_id1 = db + .create_gitlab_provider("test_id1".into(), "test_secret".into()) + .await + .unwrap(); + + let provider_id2 = db + .create_gitlab_provider("test_id2".into(), "test_secret".into()) + .await + .unwrap(); + + let repo_id1 = db + .upsert_gitlab_provided_repository( + provider_id1, + "vendor_id1".into(), + "test_repo1".into(), + "https://gitlab.com/test/test1".into(), + ) + .await + .unwrap(); + + let repo_id2 = db + .upsert_gitlab_provided_repository( + provider_id2, + "vendor_id2".into(), + "test_repo2".into(), + "https://gitlab.com/test/test2".into(), + ) + .await + .unwrap(); + + // Test listing with no filter on providers + let repos = service + .list_gitlab_provided_repositories_by_provider(vec![], None, None, None, None) + .await + .unwrap(); + + assert_eq!(repos.len(), 2); + assert_eq!(repos[0].name, "test_repo1"); + assert_eq!(repos[1].name, "test_repo2"); + + // Test listing with a filter on providers + let repos = service + .list_gitlab_provided_repositories_by_provider( + vec![provider_id1.as_id()], + None, + None, + None, + None, + ) + .await + .unwrap(); + + assert_eq!(repos.len(), 1); + assert_eq!(repos[0].name, "test_repo1"); + + // Test deletion and toggling active status + db.delete_gitlab_provided_repository(repo_id1) + .await + .unwrap(); + + db.update_gitlab_provided_repository_active(repo_id2, true) + .await + .unwrap(); + + let repos = service + .list_gitlab_provided_repositories_by_provider(vec![], None, None, None, None) + .await + .unwrap(); + + assert_eq!(repos.len(), 1); + assert!(repos[0].active); + } + + #[tokio::test] + async fn test_gitlab_repository_provider_crud() { + let db = DbConn::new_in_memory().await.unwrap(); + let service = super::create(db.clone()); + + let id = service + .create_gitlab_repository_provider("id".into(), "secret".into()) + .await + .unwrap(); + + // Test retrieving gitlab provider by ID + let provider1 = service + .get_gitlab_repository_provider(id.clone()) + .await + .unwrap(); + assert_eq!( + provider1, + GitlabRepositoryProvider { + id: id.clone(), + display_name: "id".into(), + access_token: Some("secret".into()), + connected: true, + } + ); + + // Test listing gitlab providers + let providers = service + .list_gitlab_repository_providers(vec![], None, None, None, None) + .await + .unwrap(); + assert_eq!(providers.len(), 1); + assert_eq!(providers[0].access_token, Some("secret".into())); + + // Test resetgitlab provider tokens + service + .reset_gitlab_repository_provider_access_token(id.clone()) + .await + .unwrap(); + + assert_eq!( + service + .get_gitlab_repository_provider(id.clone()) + .await + .unwrap() + .access_token, + None + ); + + // Test deleting gitlab provider + service + .delete_gitlab_repository_provider(id.clone()) + .await + .unwrap(); + + assert_eq!( + 0, + service + .list_gitlab_repository_providers(vec![], None, None, None, None) + .await + .unwrap() + .len() + ); + } + + #[tokio::test] + async fn test_provided_git_urls() { + let db = DbConn::new_in_memory().await.unwrap(); + let service = create(db.clone()); + + let provider_id = db + .create_gitlab_provider("provider1".into(), "token".into()) + .await + .unwrap(); + + let repo_id = db + .upsert_gitlab_provided_repository( + provider_id, + "vendor_id1".into(), + "test_repo".into(), + "https://gitlab.com/TabbyML/tabby".into(), + ) + .await + .unwrap(); + + db.update_gitlab_provided_repository_active(repo_id, true) + .await + .unwrap(); + + let git_urls = service.list_provided_git_urls().await.unwrap(); + assert_eq!( + git_urls, + ["https://token@gitlab.com/TabbyML/tabby".to_string()] + ); + } + + #[tokio::test] + async fn test_delete_outdated_repos() { + let db = DbConn::new_in_memory().await.unwrap(); + let service = create(db.clone()); + let time = Utc::now(); + + let provider_id = db + .create_gitlab_provider("provider1".into(), "secret1".into()) + .await + .unwrap(); + + let _repo_id = db + .upsert_gitlab_provided_repository( + provider_id, + "vendor_id1".into(), + "test_repo".into(), + "https://gitlab.com/TabbyML/tabby".into(), + ) + .await + .unwrap(); + + service + .delete_outdated_gitlab_provided_repositories(provider_id.as_id(), time) + .await + .unwrap(); + + assert_eq!( + 1, + service + .list_gitlab_provided_repositories_by_provider(vec![], None, None, None, None) + .await + .unwrap() + .len() + ); + + let time = time + Duration::minutes(1); + + service + .delete_outdated_gitlab_provided_repositories(provider_id.as_id(), time) + .await + .unwrap(); + + assert_eq!( + 0, + service + .list_gitlab_provided_repositories_by_provider(vec![], None, None, None, None) + .await + .unwrap() + .len() + ); + } +} diff --git a/ee/tabby-webserver/src/service/mod.rs b/ee/tabby-webserver/src/service/mod.rs index 7a3885b83b9c..b851a0f100f5 100644 --- a/ee/tabby-webserver/src/service/mod.rs +++ b/ee/tabby-webserver/src/service/mod.rs @@ -5,6 +5,7 @@ mod email; pub mod event_logger; mod git_repository; mod github_repository_provider; +mod gitlab_repository_provider; mod job; mod license; mod proxy; From 422460563231709cc6f9ce4fed26ada18786483b Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:21:40 -0400 Subject: [PATCH 2/9] Finish implementing --- .../migrations/0028_gitlab-provider.down.sql | 4 +- ee/tabby-webserver/src/cron/db.rs | 20 ++++- ee/tabby-webserver/src/cron/gitlab.rs | 73 ++++++++----------- ee/tabby-webserver/src/cron/mod.rs | 13 +++- ee/tabby-webserver/src/handler.rs | 2 +- ee/tabby-webserver/src/schema/repository.rs | 2 + ee/tabby-webserver/src/service/repository.rs | 11 ++- 7 files changed, 74 insertions(+), 51 deletions(-) diff --git a/ee/tabby-db/migrations/0028_gitlab-provider.down.sql b/ee/tabby-db/migrations/0028_gitlab-provider.down.sql index d2f607c5b8bd..8065696f2fa8 100644 --- a/ee/tabby-db/migrations/0028_gitlab-provider.down.sql +++ b/ee/tabby-db/migrations/0028_gitlab-provider.down.sql @@ -1 +1,3 @@ --- Add down migration script here +DROP TABLE gitlab_provided_repositories; +DROP TABLE gitlab_repository_provider; + diff --git a/ee/tabby-webserver/src/cron/db.rs b/ee/tabby-webserver/src/cron/db.rs index 122f84483e66..32fd9026754b 100644 --- a/ee/tabby-webserver/src/cron/db.rs +++ b/ee/tabby-webserver/src/cron/db.rs @@ -7,7 +7,9 @@ use futures::Future; use tokio_cron_scheduler::Job; use tracing::{debug, error}; -use super::github::refresh_all_repositories; +use super::github; +use super::gitlab; +use crate::schema::gitlab_repository_provider::GitlabRepositoryProviderService; use crate::schema::{ auth::AuthenticationService, github_repository_provider::GithubRepositoryProviderService, job::JobService, @@ -60,7 +62,21 @@ pub async fn update_integrated_github_repositories_job( github_repository_provider, |github_repository_provider| async move { debug!("Syncing github repositories..."); - refresh_all_repositories(github_repository_provider).await + github::refresh_all_repositories(github_repository_provider).await + }, + ) + .await +} + +pub async fn update_integrated_gitlab_repositories_job( + gitlab_repository_provider: Arc, +) -> Result { + service_job( + "0 * * * * *", + gitlab_repository_provider, + |gitlab_repository_provider| async move { + debug!("Syncing gitlab repositories..."); + gitlab::refresh_all_repositories(gitlab_repository_provider).await }, ) .await diff --git a/ee/tabby-webserver/src/cron/gitlab.rs b/ee/tabby-webserver/src/cron/gitlab.rs index d38bb5cbf366..d76cb1ba26e8 100644 --- a/ee/tabby-webserver/src/cron/gitlab.rs +++ b/ee/tabby-webserver/src/cron/gitlab.rs @@ -2,14 +2,16 @@ use std::sync::Arc; use anyhow::Result; use chrono::Utc; -use gitlab::AsyncGitlab; +use gitlab::{ + api::{projects::Projects, AsyncQuery, Pagination}, + GitlabBuilder, +}; use juniper::ID; -use octocrab::{models::Repository, GitHubError, Octocrab}; +use serde::Deserialize; use tracing::warn; -use crate::schema::{ - github_repository_provider::{GithubRepositoryProvider, GithubRepositoryProviderService}, - gitlab_repository_provider::{GitlabRepositoryProvider, GitlabRepositoryProviderService}, +use crate::schema::gitlab_repository_provider::{ + GitlabRepositoryProvider, GitlabRepositoryProviderService, }; pub async fn refresh_all_repositories( @@ -35,10 +37,7 @@ async fn refresh_repositories_for_provider( let provider = service.get_gitlab_repository_provider(provider_id).await?; let repos = match fetch_all_repos(&provider).await { Ok(repos) => repos, - Err(octocrab::Error::GitHub { - source: source @ GitHubError { .. }, - .. - }) if source.status_code.is_client_error() => { + Err(e) if e.to_string().contains("401 Unauthorized") => { service .reset_gitlab_repository_provider_access_token(provider.id.clone()) .await?; @@ -46,7 +45,7 @@ async fn refresh_repositories_for_provider( "GitLab credentials for provider {} are expired or invalid", provider.display_name ); - return Err(source.into()); + return Err(e); } Err(e) => { warn!("Failed to fetch repositories from github: {e}"); @@ -55,49 +54,41 @@ async fn refresh_repositories_for_provider( }; for repo in repos { let id = repo.id.to_string(); - let Some(url) = repo.git_url else { - continue; - }; - let url = url.to_string(); service - .upsert_gitlab_provided_repository(provider.id.clone(), id, repo.name, url) + .upsert_gitlab_provided_repository( + provider.id.clone(), + id, + repo.name, + repo.http_url_to_repo, + ) .await?; } Ok(()) } -// FIXME(wsxiaoys): Convert to async stream +#[derive(Deserialize)] +struct Repository { + id: u128, + name: String, + http_url_to_repo: String, +} + async fn fetch_all_repos( provider: &GitlabRepositoryProvider, -) -> Result, octocrab::Error> { +) -> Result, anyhow::Error> { let Some(token) = &provider.access_token else { return Ok(vec![]); }; - let octocrab = Octocrab::builder() - .user_access_token(token.to_string()) - .build()?; + let gitlab = GitlabBuilder::new("gitlab.com", token) + .build_async() + .await?; - let mut page = 1; - let mut repos = vec![]; - - loop { - let response = octocrab - .current() - .list_repos_for_authenticated_user() - .visibility("all") - .page(page) - .send() - .await?; - - let pages = response.number_of_pages().unwrap_or_default() as u8; - repos.extend(response.items); - - page += 1; - if page > pages { - break; - } - } - Ok(repos) + Ok(gitlab::api::paged( + Projects::builder().membership(true).build()?, + Pagination::All, + ) + .query_async(&gitlab) + .await?) } diff --git a/ee/tabby-webserver/src/cron/mod.rs b/ee/tabby-webserver/src/cron/mod.rs index 982e3885f6a2..a3410eca25b6 100644 --- a/ee/tabby-webserver/src/cron/mod.rs +++ b/ee/tabby-webserver/src/cron/mod.rs @@ -8,8 +8,8 @@ use std::sync::Arc; use tokio_cron_scheduler::{Job, JobScheduler}; use crate::schema::{ - auth::AuthenticationService, github_repository_provider::GithubRepositoryProviderService, - job::JobService, worker::WorkerService, + auth::AuthenticationService, job::JobService, repository::RepositoryService, + worker::WorkerService, }; async fn new_job_scheduler(jobs: Vec) -> anyhow::Result { @@ -25,7 +25,7 @@ pub async fn run_cron( auth: Arc, job: Arc, worker: Arc, - github_repository_provider: Arc, + repository: Arc, local_port: u16, ) { let mut jobs = vec![]; @@ -50,11 +50,16 @@ pub async fn run_cron( .expect("failed to create stale job runs cleanup job"); jobs.push(job4); - let job5 = db::update_integrated_github_repositories_job(github_repository_provider) + let job5 = db::update_integrated_github_repositories_job(repository.github()) .await .expect("Failed to create github repository refresh job"); jobs.push(job5); + let job6 = db::update_integrated_gitlab_repositories_job(repository.gitlab()) + .await + .expect("Failed to create gitlab repository refresh job"); + jobs.push(job6); + new_job_scheduler(jobs) .await .expect("failed to start job scheduler"); diff --git a/ee/tabby-webserver/src/handler.rs b/ee/tabby-webserver/src/handler.rs index b6547ca99371..98d536682a9d 100644 --- a/ee/tabby-webserver/src/handler.rs +++ b/ee/tabby-webserver/src/handler.rs @@ -87,7 +87,7 @@ impl WebserverHandle { ctx.auth(), ctx.job(), ctx.worker(), - ctx.repository().github(), + ctx.repository(), local_port, ) .await; diff --git a/ee/tabby-webserver/src/schema/repository.rs b/ee/tabby-webserver/src/schema/repository.rs index 5b1feeefd304..1d16993e744e 100644 --- a/ee/tabby-webserver/src/schema/repository.rs +++ b/ee/tabby-webserver/src/schema/repository.rs @@ -8,6 +8,7 @@ use tabby_search::FileSearch; use super::{ git_repository::GitRepositoryService, github_repository_provider::GithubRepositoryProviderService, + gitlab_repository_provider::GitlabRepositoryProviderService, }; #[derive(GraphQLObject, Debug)] @@ -33,5 +34,6 @@ impl From for FileEntrySearchResult { pub trait RepositoryService: Send + Sync + RepositoryAccess { fn git(&self) -> Arc; fn github(&self) -> Arc; + fn gitlab(&self) -> Arc; fn access(self: Arc) -> Arc; } diff --git a/ee/tabby-webserver/src/service/repository.rs b/ee/tabby-webserver/src/service/repository.rs index 4a2bf9fcfb1f..e978952e2408 100644 --- a/ee/tabby-webserver/src/service/repository.rs +++ b/ee/tabby-webserver/src/service/repository.rs @@ -4,21 +4,24 @@ use async_trait::async_trait; use tabby_common::config::{RepositoryAccess, RepositoryConfig}; use tabby_db::DbConn; -use super::github_repository_provider; +use super::{github_repository_provider, gitlab_repository_provider}; use crate::schema::{ git_repository::GitRepositoryService, - github_repository_provider::GithubRepositoryProviderService, repository::RepositoryService, + github_repository_provider::GithubRepositoryProviderService, + gitlab_repository_provider::GitlabRepositoryProviderService, repository::RepositoryService, }; struct RepositoryServiceImpl { git: Arc, github: Arc, + gitlab: Arc, } pub fn create(db: DbConn) -> Arc { Arc::new(RepositoryServiceImpl { git: Arc::new(db.clone()), github: Arc::new(github_repository_provider::create(db.clone())), + gitlab: Arc::new(gitlab_repository_provider::create(db.clone())), }) } @@ -58,6 +61,10 @@ impl RepositoryService for RepositoryServiceImpl { fn github(&self) -> Arc { self.github.clone() } + + fn gitlab(&self) -> Arc { + self.gitlab.clone() + } } #[cfg(test)] From 7b3759bc0d3dda1f03bf75247a7cbf1d580e0b35 Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:24:53 -0400 Subject: [PATCH 3/9] Update RepositoryAccess implementation --- ee/tabby-webserver/src/service/repository.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ee/tabby-webserver/src/service/repository.rs b/ee/tabby-webserver/src/service/repository.rs index e978952e2408..00dbf2def023 100644 --- a/ee/tabby-webserver/src/service/repository.rs +++ b/ee/tabby-webserver/src/service/repository.rs @@ -45,6 +45,15 @@ impl RepositoryAccess for RepositoryServiceImpl { .map(RepositoryConfig::new), ); + repos.extend( + self.gitlab + .list_provided_git_urls() + .await + .unwrap_or_default() + .into_iter() + .map(RepositoryConfig::new), + ); + Ok(repos) } } From 82e24502277a5829c8526a2e5e4a1735f18baf2a Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:25:46 -0400 Subject: [PATCH 4/9] Fix debug value --- ee/tabby-webserver/src/cron/db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/tabby-webserver/src/cron/db.rs b/ee/tabby-webserver/src/cron/db.rs index 32fd9026754b..96c70eee0b3c 100644 --- a/ee/tabby-webserver/src/cron/db.rs +++ b/ee/tabby-webserver/src/cron/db.rs @@ -72,7 +72,7 @@ pub async fn update_integrated_gitlab_repositories_job( gitlab_repository_provider: Arc, ) -> Result { service_job( - "0 * * * * *", + EVERY_TEN_MINUTES, gitlab_repository_provider, |gitlab_repository_provider| async move { debug!("Syncing gitlab repositories..."); From a282126d3c6fd5992e41b66719e73916a4ca0723 Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:27:15 -0400 Subject: [PATCH 5/9] Fix const name --- ee/tabby-webserver/src/schema/gitlab_repository_provider.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs index 2cff5cd7f7aa..28e475d6b308 100644 --- a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs +++ b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs @@ -9,12 +9,12 @@ use super::Context; use crate::{juniper::relay::NodeType, schema::Result}; lazy_static! { - static ref GITHUB_REPOSITORY_PROVIDER_NAME_REGEX: Regex = Regex::new("^[\\w-]+$").unwrap(); + static ref GITLAB_REPOSITORY_PROVIDER_NAME_REGEX: Regex = Regex::new("^[\\w-]+$").unwrap(); } #[derive(GraphQLInputObject, Validate)] pub struct CreateGitlabRepositoryProviderInput { - #[validate(regex(code = "displayName", path = "GITHUB_REPOSITORY_PROVIDER_NAME_REGEX"))] + #[validate(regex(code = "displayName", path = "GITLAB_REPOSITORY_PROVIDER_NAME_REGEX"))] pub display_name: String, #[validate(length(code = "access_token", min = 10))] pub access_token: String, @@ -23,7 +23,7 @@ pub struct CreateGitlabRepositoryProviderInput { #[derive(GraphQLInputObject, Validate)] pub struct UpdateGitlabRepositoryProviderInput { pub id: ID, - #[validate(regex(code = "displayName", path = "GITHUB_REPOSITORY_PROVIDER_NAME_REGEX"))] + #[validate(regex(code = "displayName", path = "GITLAB_REPOSITORY_PROVIDER_NAME_REGEX"))] pub display_name: String, #[validate(length(code = "access_token", min = 10))] pub access_token: String, From c757eae9ed19adcf6ba4662721b63d9fbb25dc79 Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:27:49 -0400 Subject: [PATCH 6/9] Reapply migrations to local schema --- ee/tabby-db/schema.sqlite | Bin 176128 -> 176128 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite index 8eaacca689da1d5cc4a3318c7da8ad5fa0a779fc..325abece2c60ba6cf0326220d74ca06681fe9f69 100644 GIT binary patch delta 691 zcmY+>KS%;$7{_tW3p5*Y(NphE&oC+*vg_X^B1KD5($HWQQf38F;Sw|yBn~DGVmyXK z6#b*Yt=w87f|97F93mvAqNSyVpr{@p=zZ4@Kfdn+Z?vS1mb59JQLPC*jOu9Yu$oao zty35!*DZZiR0ah#)7Vlu%R-l)*T}LWBM*Ch@WWA&W+Sfm zE@8qbk&z_wP+tMgjMODB$ioLqAeb7+^?=FA&LkcoW~w7Oha6zi@MNkbS#QSnMI3s~ z6fYnLHtmo#+sJjH%y|(sJawSV<0b1pnB--0y^B104sgd)ZB}ry)2HvyE>P_da!`4O zwBR5kd1U_s1D^sFafo<#j}L+(b;$~{Kbe3Nk&3*Qc`*Vu3-x5b1ta&XuuOBd%;UFE nwAjf@s;tQ0Uhr6{Z1H delta 692 zcmY+>ODIH990qXrG29u&lgN5skLDw`jv5=I#GErV>u#|;}Vki0(ilVTv zW0WEb3pGvbSZN9ir974vMx3)dXZ!2x`+uji@Z?%}a;<5Cysmc&^2y*)xzL6Zce~J* zNxHvfXG$*-8l%p%jHIe5&F!`sv!fPSJep9MA+{2>3j^zwtF{j|wU&FE*AUvBx zcYxy=!qWQ-jsul!`hlj)QcyH*prw(=pDP&AEH=GQSQ&f7y~cG*Iw=_+M61qq4^i&u z!G_+zmSic%r|_e5B@IC`(D8wRAh%?Na Date: Thu, 25 Apr 2024 22:37:14 +0000 Subject: [PATCH 7/9] [autofix.ci] apply automated fixes --- ee/tabby-db/schema/schema.sql | 20 + ee/tabby-db/schema/schema.svg | 660 ++++++++++++++------------ ee/tabby-webserver/src/cron/db.rs | 6 +- ee/tabby-webserver/src/cron/gitlab.rs | 2 +- 4 files changed, 382 insertions(+), 306 deletions(-) diff --git a/ee/tabby-db/schema/schema.sql b/ee/tabby-db/schema/schema.sql index 687ff70ece12..e855a24eb36d 100644 --- a/ee/tabby-db/schema/schema.sql +++ b/ee/tabby-db/schema/schema.sql @@ -149,3 +149,23 @@ CREATE TABLE github_repository_provider( display_name TEXT NOT NULL, access_token TEXT ); +CREATE TABLE gitlab_repository_provider( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + display_name TEXT NOT NULL, + access_token TEXT +); +CREATE TABLE gitlab_provided_repositories( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + gitlab_repository_provider_id INTEGER NOT NULL, + -- vendor_id from https://docs.gitlab.com/ee/api/repositories.html + vendor_id TEXT NOT NULL, + name TEXT NOT NULL, + git_url TEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + FOREIGN KEY(gitlab_repository_provider_id) REFERENCES gitlab_repository_provider(id) ON DELETE CASCADE, + CONSTRAINT `idx_vendor_id_provider_id` UNIQUE(vendor_id, gitlab_repository_provider_id) +); +CREATE INDEX gitlab_provided_repositories_updated_at ON gitlab_provided_repositories( + updated_at +); diff --git a/ee/tabby-db/schema/schema.svg b/ee/tabby-db/schema/schema.svg index ea8762a071b4..096664001962 100644 --- a/ee/tabby-db/schema/schema.svg +++ b/ee/tabby-db/schema/schema.svg @@ -4,11 +4,11 @@ - - + + structs - + _sqlx_migrations @@ -130,352 +130,410 @@ access_token - + github_provided_repositories:e->github_repository_provider:w - + +gitlab_provided_repositories + +gitlab_provided_repositories + +🔑 + +id + +  + +gitlab_repository_provider_id + +  + +vendor_id + +  + +name + +  + +git_url + +  + +active + +  + +updated_at + + + +gitlab_repository_provider + +gitlab_repository_provider + +🔑 + +id + +  + +display_name + +  + +access_token + + + +gitlab_provided_repositories:e->gitlab_repository_provider:w + + + + + invitations - -invitations - -🔑 - -id - -  - -email - -  - -code - -  - -created_at + +invitations + +🔑 + +id + +  + +email + +  + +code + +  + +created_at - + job_runs - -job_runs - -🔑 - -id - -  - -job - -  - -start_ts - -  - -end_ts - -  - -exit_code - -  - -stdout - -  - -stderr - -  - -created_at - -  - -updated_at + +job_runs + +🔑 + +id + +  + +job + +  + +start_ts + +  + +end_ts + +  + +exit_code + +  + +stdout + +  + +stderr + +  + +created_at + +  + +updated_at - + oauth_credential - -oauth_credential - -🔑 - -id - -  - -provider - -  - -client_id - -  - -client_secret - -  - -created_at - -  - -updated_at + +oauth_credential + +🔑 + +id + +  + +provider + +  + +client_id + +  + +client_secret + +  + +created_at + +  + +updated_at - + password_reset - -password_reset - -🔑 - -id - -  - -user_id - -  - -code - -  - -created_at + +password_reset + +🔑 + +id + +  + +user_id + +  + +code + +  + +created_at - + users - -users - -🔑 - -id - -  - -email - -  - -is_admin - -  - -created_at - -  - -updated_at - -  - -auth_token - -  - -active - -  - -password_encrypted - -  - -avatar + +users + +🔑 + +id + +  + +email + +  + +is_admin + +  + +created_at + +  + +updated_at + +  + +auth_token + +  + +active + +  + +password_encrypted + +  + +avatar - + password_reset:e->users:w - - + + - + refresh_tokens - -refresh_tokens - -🔑 - -id - -  - -user_id - -  - -token - -  - -expires_at - -  - -created_at + +refresh_tokens + +🔑 + +id + +  + +user_id + +  + +token + +  + +expires_at + +  + +created_at - + refresh_tokens:e->users:w - - + + - + registration_token - -registration_token - -🔑 - -id - -  - -token - -  - -created_at - -  - -updated_at + +registration_token + +🔑 + +id + +  + +token + +  + +created_at + +  + +updated_at - + repositories - -repositories - -🔑 - -id - -  - -name - -  - -git_url + +repositories + +🔑 + +id + +  + +name + +  + +git_url - + server_setting - -server_setting - -🔑 - -id - -  - -security_allowed_register_domain_list - -  - -security_disable_client_side_telemetry - -  - -network_external_url - -  - -billing_enterprise_license + +server_setting + +🔑 + +id + +  + +security_allowed_register_domain_list + +  + +security_disable_client_side_telemetry + +  + +network_external_url + +  + +billing_enterprise_license - + user_completions - -user_completions - -🔑 - -id - -  - -user_id - -  - -completion_id - -  - -language - -  - -views - -  - -selects - -  - -dismisses - -  - -created_at - -  - -updated_at + +user_completions + +🔑 + +id + +  + +user_id + +  + +completion_id + +  + +language + +  + +views + +  + +selects + +  + +dismisses + +  + +created_at + +  + +updated_at - + user_completions:e->users:w - - + + - + user_events - -user_events - -🔑 - -id - -  - -user_id - -  - -kind - -  - -created_at - -  - -payload + +user_events + +🔑 + +id + +  + +user_id + +  + +kind + +  + +created_at + +  + +payload user_events:e->users:w - - + + diff --git a/ee/tabby-webserver/src/cron/db.rs b/ee/tabby-webserver/src/cron/db.rs index 96c70eee0b3c..eca9082e1e3b 100644 --- a/ee/tabby-webserver/src/cron/db.rs +++ b/ee/tabby-webserver/src/cron/db.rs @@ -7,12 +7,10 @@ use futures::Future; use tokio_cron_scheduler::Job; use tracing::{debug, error}; -use super::github; -use super::gitlab; -use crate::schema::gitlab_repository_provider::GitlabRepositoryProviderService; +use super::{github, gitlab}; use crate::schema::{ auth::AuthenticationService, github_repository_provider::GithubRepositoryProviderService, - job::JobService, + gitlab_repository_provider::GitlabRepositoryProviderService, job::JobService, }; const EVERY_TWO_HOURS: &str = "0 0 1/2 * * * *"; diff --git a/ee/tabby-webserver/src/cron/gitlab.rs b/ee/tabby-webserver/src/cron/gitlab.rs index d76cb1ba26e8..1639c673d14b 100644 --- a/ee/tabby-webserver/src/cron/gitlab.rs +++ b/ee/tabby-webserver/src/cron/gitlab.rs @@ -49,7 +49,7 @@ async fn refresh_repositories_for_provider( } Err(e) => { warn!("Failed to fetch repositories from github: {e}"); - return Err(e.into()); + return Err(e); } }; for repo in repos { From 8df12dfc45f5d913e79fbd42d33d8ef9638b3586 Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:55:57 -0400 Subject: [PATCH 8/9] Extract constant for repository name regex --- ee/tabby-webserver/src/schema/constants.rs | 6 ++++++ ee/tabby-webserver/src/schema/git_repository.rs | 8 +------- .../src/schema/github_repository_provider.rs | 16 ++++++++-------- .../src/schema/gitlab_repository_provider.rs | 16 ++++++++-------- ee/tabby-webserver/src/schema/mod.rs | 1 + 5 files changed, 24 insertions(+), 23 deletions(-) create mode 100644 ee/tabby-webserver/src/schema/constants.rs diff --git a/ee/tabby-webserver/src/schema/constants.rs b/ee/tabby-webserver/src/schema/constants.rs new file mode 100644 index 000000000000..412b6c16af16 --- /dev/null +++ b/ee/tabby-webserver/src/schema/constants.rs @@ -0,0 +1,6 @@ +use lazy_static::lazy_static; +use regex::Regex; + +lazy_static! { + pub static ref REPOSITORY_NAME_REGEX: Regex = Regex::new("^[\\w-]+$").unwrap(); +} diff --git a/ee/tabby-webserver/src/schema/git_repository.rs b/ee/tabby-webserver/src/schema/git_repository.rs index b70ddb8e5905..9f41e2732029 100644 --- a/ee/tabby-webserver/src/schema/git_repository.rs +++ b/ee/tabby-webserver/src/schema/git_repository.rs @@ -1,21 +1,15 @@ use async_trait::async_trait; use juniper::{GraphQLObject, ID}; -use lazy_static::lazy_static; -use regex::Regex; use validator::Validate; use super::{repository::FileEntrySearchResult, Context, Result}; use crate::juniper::relay::NodeType; -lazy_static! { - static ref REPOSITORY_NAME_REGEX: Regex = Regex::new("^[a-zA-Z][\\w.-]+$").unwrap(); -} - #[derive(Validate)] pub struct CreateGitRepositoryInput { #[validate(regex( code = "name", - path = "self::REPOSITORY_NAME_REGEX", + path = "crate::schema::constants::REPOSITORY_NAME_REGEX", message = "Invalid repository name" ))] pub name: String, diff --git a/ee/tabby-webserver/src/schema/github_repository_provider.rs b/ee/tabby-webserver/src/schema/github_repository_provider.rs index 573e194b8c30..2251126cd215 100644 --- a/ee/tabby-webserver/src/schema/github_repository_provider.rs +++ b/ee/tabby-webserver/src/schema/github_repository_provider.rs @@ -1,20 +1,17 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use juniper::{GraphQLInputObject, GraphQLObject, ID}; -use lazy_static::lazy_static; -use regex::Regex; use validator::Validate; use super::Context; use crate::{juniper::relay::NodeType, schema::Result}; -lazy_static! { - static ref GITHUB_REPOSITORY_PROVIDER_NAME_REGEX: Regex = Regex::new("^[\\w-]+$").unwrap(); -} - #[derive(GraphQLInputObject, Validate)] pub struct CreateGithubRepositoryProviderInput { - #[validate(regex(code = "displayName", path = "GITHUB_REPOSITORY_PROVIDER_NAME_REGEX"))] + #[validate(regex( + code = "displayName", + path = "crate::schema::constants::REPOSITORY_NAME_REGEX" + ))] pub display_name: String, #[validate(length(code = "access_token", min = 10))] pub access_token: String, @@ -23,7 +20,10 @@ pub struct CreateGithubRepositoryProviderInput { #[derive(GraphQLInputObject, Validate)] pub struct UpdateGithubRepositoryProviderInput { pub id: ID, - #[validate(regex(code = "displayName", path = "GITHUB_REPOSITORY_PROVIDER_NAME_REGEX"))] + #[validate(regex( + code = "displayName", + path = "crate::schema::constants::REPOSITORY_NAME_REGEX" + ))] pub display_name: String, #[validate(length(code = "access_token", min = 10))] pub access_token: String, diff --git a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs index 28e475d6b308..6c33570c3860 100644 --- a/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs +++ b/ee/tabby-webserver/src/schema/gitlab_repository_provider.rs @@ -1,20 +1,17 @@ use async_trait::async_trait; use chrono::{DateTime, Utc}; use juniper::{GraphQLInputObject, GraphQLObject, ID}; -use lazy_static::lazy_static; -use regex::Regex; use validator::Validate; use super::Context; use crate::{juniper::relay::NodeType, schema::Result}; -lazy_static! { - static ref GITLAB_REPOSITORY_PROVIDER_NAME_REGEX: Regex = Regex::new("^[\\w-]+$").unwrap(); -} - #[derive(GraphQLInputObject, Validate)] pub struct CreateGitlabRepositoryProviderInput { - #[validate(regex(code = "displayName", path = "GITLAB_REPOSITORY_PROVIDER_NAME_REGEX"))] + #[validate(regex( + code = "displayName", + path = "crate::schema::constants::REPOSITORY_NAME_REGEX" + ))] pub display_name: String, #[validate(length(code = "access_token", min = 10))] pub access_token: String, @@ -23,7 +20,10 @@ pub struct CreateGitlabRepositoryProviderInput { #[derive(GraphQLInputObject, Validate)] pub struct UpdateGitlabRepositoryProviderInput { pub id: ID, - #[validate(regex(code = "displayName", path = "GITLAB_REPOSITORY_PROVIDER_NAME_REGEX"))] + #[validate(regex( + code = "displayName", + path = "crate::schema::constants::REPOSITORY_NAME_REGEX" + ))] pub display_name: String, #[validate(length(code = "access_token", min = 10))] pub access_token: String, diff --git a/ee/tabby-webserver/src/schema/mod.rs b/ee/tabby-webserver/src/schema/mod.rs index decaa117fbc7..a0845c8b258a 100644 --- a/ee/tabby-webserver/src/schema/mod.rs +++ b/ee/tabby-webserver/src/schema/mod.rs @@ -1,5 +1,6 @@ pub mod analytic; pub mod auth; +pub mod constants; pub mod email; pub mod git_repository; pub mod github_repository_provider; From 0b3a7396bf1e9e71c59ff86ee1cfe752fb4cc98f Mon Sep 17 00:00:00 2001 From: boxbeam Date: Thu, 25 Apr 2024 18:57:48 -0400 Subject: [PATCH 9/9] Fix tests --- ee/tabby-webserver/src/service/gitlab_repository_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/tabby-webserver/src/service/gitlab_repository_provider.rs b/ee/tabby-webserver/src/service/gitlab_repository_provider.rs index d63bc43f4bc7..a9b5b0c78705 100644 --- a/ee/tabby-webserver/src/service/gitlab_repository_provider.rs +++ b/ee/tabby-webserver/src/service/gitlab_repository_provider.rs @@ -368,7 +368,7 @@ mod tests { let git_urls = service.list_provided_git_urls().await.unwrap(); assert_eq!( git_urls, - ["https://token@gitlab.com/TabbyML/tabby".to_string()] + ["https://oauth2:token@gitlab.com/TabbyML/tabby".to_string()] ); }