feat(webserver, db): track github/gitlab provider status with synced_…

…at (#1980) * feat(webserver, db): track github/gitlab provider status with synced_at * [autofix.ci] apply automated fixes * Fix migrations * [autofix.ci] apply automated fixes * Perform status update logic in db layer * Remove unused methods * update ui * [autofix.ci] apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Meng Zhang <[email protected]>
TabbyML · Apr 26, 2024 · cb9a8a7 · cb9a8a7
1 parent d42bfb4
commit cb9a8a7
Show file tree

Hide file tree

Showing 20 changed files with 225 additions and 123 deletions.
diff --git a/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql b/ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql
@@ -3,5 +3,6 @@ DROP TABLE github_repository_provider;
 CREATE TABLE github_repository_provider(
     id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     display_name TEXT NOT NULL,
-    access_token TEXT
+    access_token TEXT,
+    synced_at TIMESTAMP
 );
diff --git a/ee/tabby-db/migrations/0028_gitlab-provider.up.sql b/ee/tabby-db/migrations/0028_gitlab-provider.up.sql
@@ -1,7 +1,8 @@
 CREATE TABLE gitlab_repository_provider(
     id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     display_name TEXT NOT NULL,
-    access_token TEXT
+    access_token TEXT,
+    synced_at TIMESTAMP
 );
 
 CREATE TABLE gitlab_provided_repositories(

diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite
diff --git a/ee/tabby-db/schema/schema.sql b/ee/tabby-db/schema/schema.sql
@@ -147,12 +147,14 @@ CREATE TABLE password_reset(
 CREATE TABLE github_repository_provider(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
   display_name TEXT NOT NULL,
-  access_token TEXT
+  access_token TEXT,
+  synced_at TIMESTAMP
 );
 CREATE TABLE gitlab_repository_provider(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
   display_name TEXT NOT NULL,
-  access_token TEXT
+  access_token TEXT,
+  synced_at TIMESTAMP
 );
 CREATE TABLE gitlab_provided_repositories(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,

diff --git a/ee/tabby-db/schema/schema.svg b/ee/tabby-db/schema/schema.svg
diff --git a/ee/tabby-db/src/github_repository_provider.rs b/ee/tabby-db/src/github_repository_provider.rs
@@ -9,6 +9,7 @@ pub struct GithubRepositoryProviderDAO {
     pub id: i64,
     pub display_name: String,
     pub access_token: Option<String>,
+    pub synced_at: Option<DateTimeUtc>,
 }
 
 #[derive(FromRow)]
@@ -36,7 +37,7 @@ impl DbConn {
     pub async fn get_github_provider(&self, id: i64) -> Result<GithubRepositoryProviderDAO> {
         let provider = query_as!(
             GithubRepositoryProviderDAO,
-            "SELECT id, display_name, access_token FROM github_repository_provider WHERE id = ?;",
+            r#"SELECT id, display_name, access_token, synced_at AS "synced_at: DateTimeUtc" FROM github_repository_provider WHERE id = ?;"#,
             id
         )
         .fetch_one(&self.pool)
@@ -54,23 +55,6 @@ impl DbConn {
         Ok(())
     }
 
-    pub async fn reset_github_provider_access_token(&self, id: i64) -> Result<()> {
-        let res = query!(
-            "UPDATE github_repository_provider SET access_token = NULL WHERE id = ?",
-            id
-        )
-        .execute(&self.pool)
-        .await?;
-
-        if res.rows_affected() != 1 {
-            return Err(anyhow!(
-                "The specified Github repository provider does not exist"
-            ));
-        }
-
-        Ok(())
-    }
-
     pub async fn update_github_provider(
         &self,
         id: i64,
@@ -95,6 +79,19 @@ impl DbConn {
         Ok(())
     }
 
+    pub async fn update_github_provider_sync_status(&self, id: i64, success: bool) -> Result<()> {
+        let time = success.then_some(DateTimeUtc::now());
+        query!(
+            "UPDATE github_repository_provider SET synced_at = ?, access_token = IIF(?, access_token, NULL) WHERE id = ?",
+            time,
+            success,
+            id
+        )
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
     pub async fn list_github_repository_providers(
         &self,
         ids: Vec<i64>,
@@ -113,7 +110,7 @@ impl DbConn {
         let providers = query_paged_as!(
             GithubRepositoryProviderDAO,
             "github_repository_provider",
-            ["id", "display_name", "access_token"],
+            ["id", "display_name", "access_token", "synced_at" as "synced_at: DateTimeUtc"],
             limit,
             skip_id,
             backwards,

diff --git a/ee/tabby-db/src/gitlab_repository_provider.rs b/ee/tabby-db/src/gitlab_repository_provider.rs
@@ -9,6 +9,7 @@ pub struct GitlabRepositoryProviderDAO {
     pub id: i64,
     pub display_name: String,
     pub access_token: Option<String>,
+    pub synced_at: Option<DateTimeUtc>,
 }
 
 #[derive(FromRow)]
@@ -36,7 +37,7 @@ impl DbConn {
     pub async fn get_gitlab_provider(&self, id: i64) -> Result<GitlabRepositoryProviderDAO> {
         let provider = query_as!(
             GitlabRepositoryProviderDAO,
-            "SELECT id, display_name, access_token FROM gitlab_repository_provider WHERE id = ?;",
+            r#"SELECT id, display_name, access_token, synced_at AS "synced_at: DateTimeUtc" FROM gitlab_repository_provider WHERE id = ?;"#,
             id
         )
         .fetch_one(&self.pool)
@@ -54,23 +55,6 @@ impl DbConn {
         Ok(())
     }
 
-    pub async fn reset_gitlab_provider_access_token(&self, id: i64) -> Result<()> {
-        let res = query!(
-            "UPDATE gitlab_repository_provider SET access_token = NULL WHERE id = ?",
-            id
-        )
-        .execute(&self.pool)
-        .await?;
-
-        if res.rows_affected() != 1 {
-            return Err(anyhow!(
-                "The specified gitlab repository provider does not exist"
-            ));
-        }
-
-        Ok(())
-    }
-
     pub async fn update_gitlab_provider(
         &self,
         id: i64,
@@ -95,6 +79,19 @@ impl DbConn {
         Ok(())
     }
 
+    pub async fn update_gitlab_provider_sync_status(&self, id: i64, success: bool) -> Result<()> {
+        let time = success.then_some(DateTimeUtc::now());
+        query!(
+            "UPDATE gitlab_repository_provider SET synced_at = ?, access_token = IIF(?, access_token, NULL) WHERE id = ?",
+            time,
+            success,
+            id
+        )
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
     pub async fn list_gitlab_repository_providers(
         &self,
         ids: Vec<i64>,
@@ -113,7 +110,7 @@ impl DbConn {
         let providers = query_paged_as!(
             GitlabRepositoryProviderDAO,
             "gitlab_repository_provider",
-            ["id", "display_name", "access_token"],
+            ["id", "display_name", "access_token", "synced_at" as "synced_at: DateTimeUtc"],
             limit,
             skip_id,
             backwards,

diff --git a/...y-ui/app/(dashboard)/settings/(integrations)/repository/github/components/github-form.tsx b/...y-ui/app/(dashboard)/settings/(integrations)/repository/github/components/github-form.tsx
@@ -34,13 +34,7 @@ import { IconExternalLink, IconSpinner } from '@/components/ui/icons'
 import { Input } from '@/components/ui/input'
 
 export const createGithubProviderSchema = z.object({
-  displayName: z
-    .string()
-    .trim()
-    .regex(
-      /^[\w-]+$/,
-      'Display name must contain only alphanumeric characters, underscores, and hyphens'
-    ),
+  displayName: z.string(),
   accessToken: z.string()
 })
 

diff --git a/ee/tabby-ui/app/(dashboard)/settings/(integrations)/repository/github/components/github.tsx b/ee/tabby-ui/app/(dashboard)/settings/(integrations)/repository/github/components/github.tsx
@@ -3,7 +3,10 @@
 import Link from 'next/link'
 import { useQuery } from 'urql'
 
-import { ListGithubRepositoryProvidersQuery } from '@/lib/gql/generates/graphql'
+import {
+  ListGithubRepositoryProvidersQuery,
+  RepositoryProviderStatus
+} from '@/lib/gql/generates/graphql'
 import { listGithubRepositoryProviders } from '@/lib/tabby/query'
 import { buttonVariants } from '@/components/ui/button'
 import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
@@ -65,11 +68,7 @@ const GitProvidersList: React.FC<GitProvidersTableProps> = ({ data }) => {
                 <span className="w-[30%] shrink-0 text-muted-foreground">
                   Status
                 </span>
-                <span>
-                  {item.node?.connected
-                    ? 'Connected'
-                    : 'access_token needs update'}
-                </span>
+                <span>{toStatusMessage(item.node.status)}</span>
               </div>
             </CardContent>
           </Card>
@@ -79,6 +78,17 @@ const GitProvidersList: React.FC<GitProvidersTableProps> = ({ data }) => {
   )
 }
 
+function toStatusMessage(status: RepositoryProviderStatus) {
+  switch (status) {
+    case RepositoryProviderStatus.Ready:
+      return 'Ready'
+    case RepositoryProviderStatus.Error:
+      return 'Processing error. Please check if the access token is still valid'
+    case RepositoryProviderStatus.Pending:
+      return 'Awaiting the next data synchronization'
+  }
+}
+
 const GitProvidersPlaceholder = () => {
   return (
     <div className="flex flex-col items-center gap-4 rounded-lg border-4 border-dashed py-8">

diff --git a/...ui/app/(dashboard)/settings/(integrations)/repository/github/detail/components/detail.tsx b/...ui/app/(dashboard)/settings/(integrations)/repository/github/detail/components/detail.tsx
@@ -5,6 +5,7 @@ import { useRouter, useSearchParams } from 'next/navigation'
 import { useQuery } from 'urql'
 
 import { DEFAULT_PAGE_SIZE } from '@/lib/constants'
+import { RepositoryProviderStatus } from '@/lib/gql/generates/graphql'
 import { QueryResponseData, QueryVariables, useMutation } from '@/lib/tabby/gql'
 import {
   listGithubRepositories,
@@ -68,7 +69,7 @@ const DetailPage: React.FC = () => {
 
   return (
     <LoadingWrapper loading={fetching}>
-      <CardTitle className="flex items-center justify-between">
+      <CardTitle className="flex items-center gap-4">
         <div className="-ml-1 flex items-center">
           <Button
             onClick={() => router.back()}
@@ -81,11 +82,7 @@ const DetailPage: React.FC = () => {
         </div>
         <div className="flex items-center gap-2 text-base">
           <div className="ml-1">
-            {provider?.connected ? (
-              <Badge variant="successful">Connected</Badge>
-            ) : (
-              <Badge variant="destructive">Not Connected</Badge>
-            )}
+            {provider && toStatusBadge(provider.status)}
           </div>
         </div>
       </CardTitle>
@@ -108,6 +105,17 @@ const DetailPage: React.FC = () => {
   )
 }
 
+function toStatusBadge(status: RepositoryProviderStatus) {
+  switch (status) {
+    case RepositoryProviderStatus.Ready:
+      return <Badge variant="successful">Ready</Badge>
+    case RepositoryProviderStatus.Error:
+      return <Badge variant="destructive">Error</Badge>
+    case RepositoryProviderStatus.Error:
+      return <Badge>Pending</Badge>
+  }
+}
+
 const LinkedRepoTable: React.FC<{
   data: GithubRepositories | undefined
   onDelete?: () => void

diff --git a/ee/tabby-ui/lib/tabby/query.ts b/ee/tabby-ui/lib/tabby/query.ts
@@ -186,7 +186,7 @@ export const listGithubRepositoryProviders = graphql(/* GraphQL */ `
         node {
           id
           displayName
-          connected
+          status
         }
         cursor
       }

diff --git a/ee/tabby-webserver/graphql/schema.graphql b/ee/tabby-webserver/graphql/schema.graphql
@@ -13,7 +13,7 @@ input UpdateRepositoryProviderInput {
 type GitlabRepositoryProvider {
   id: ID!
   displayName: String!
-  connected: Boolean!
+  status: RepositoryProviderStatus!
 }
 
 enum Language {
@@ -119,6 +119,12 @@ type UserConnection {
   pageInfo: PageInfo!
 }
 
+enum RepositoryProviderStatus {
+  READY
+  PENDING
+  ERROR
+}
+
 type DiskUsageStats {
   events: DiskUsage!
   indexedRepositories: DiskUsage!
@@ -142,7 +148,7 @@ type ServerInfo {
 type GithubRepositoryProvider {
   id: ID!
   displayName: String!
-  connected: Boolean!
+  status: RepositoryProviderStatus!
 }
 
 input PasswordChangeInput {

diff --git a/ee/tabby-webserver/src/cron/github.rs b/ee/tabby-webserver/src/cron/github.rs
@@ -38,7 +38,7 @@ async fn refresh_repositories_for_provider(
             ..
         }) if source.status_code.is_client_error() => {
             service
-                .reset_github_repository_provider_access_token(provider.id.clone())
+                .update_github_repository_provider_sync_status(provider.id.clone(), false)
                 .await?;
             warn!(
                 "GitHub credentials for provider {} are expired or invalid",
@@ -72,6 +72,9 @@ async fn refresh_repositories_for_provider(
             )
             .await?;
     }
+    service
+        .update_github_repository_provider_sync_status(provider.id.clone(), true)
+        .await?;
 
     Ok(())
 }

diff --git a/ee/tabby-webserver/src/cron/gitlab.rs b/ee/tabby-webserver/src/cron/gitlab.rs
@@ -39,7 +39,7 @@ async fn refresh_repositories_for_provider(
         Ok(repos) => repos,
         Err(e) if e.to_string().contains("401 Unauthorized") => {
             service
-                .reset_gitlab_repository_provider_access_token(provider.id.clone())
+                .update_gitlab_repository_provider_sync_status(provider.id.clone(), false)
                 .await?;
             warn!(
                 "GitLab credentials for provider {} are expired or invalid",
@@ -66,6 +66,9 @@ async fn refresh_repositories_for_provider(
             )
             .await?;
     }
+    service
+        .update_gitlab_repository_provider_sync_status(provider.id.clone(), true)
+        .await?;
 
     Ok(())
 }
-Original file line number
+Diff line change
@@ Expand Up @@
             node {
               id
               displayName
-              connected
+              status
             }
             cursor
           }
@@ Expand Down @@