feat(webserver, db): track github/gitlab provider status with synced_at

ee/tabby-db/migrations/0027_remove-github-repository-columns-on-oauth.up.sql

@@ -3,5 +3,6 @@ DROP TABLE github_repository_provider;
 CREATE TABLE github_repository_provider(
     id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     display_name TEXT NOT NULL,
-    access_token TEXT
+    access_token TEXT,
+    synced_at TIMESTAMP
 );

ee/tabby-db/migrations/0028_gitlab-provider.up.sql

@@ -1,7 +1,8 @@
 CREATE TABLE gitlab_repository_provider(
     id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
     display_name TEXT NOT NULL,
-    access_token TEXT
+    access_token TEXT,
+    synced_at TIMESTAMP
 );
 
 CREATE TABLE gitlab_provided_repositories(

ee/tabby-db/schema/schema.sql

@@ -147,12 +147,14 @@ CREATE TABLE password_reset(
 CREATE TABLE github_repository_provider(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
   display_name TEXT NOT NULL,
-  access_token TEXT
+  access_token TEXT,
+  synced_at TIMESTAMP
 );
 CREATE TABLE gitlab_repository_provider(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
   display_name TEXT NOT NULL,
-  access_token TEXT
+  access_token TEXT,
+  synced_at TIMESTAMP
 );
 CREATE TABLE gitlab_provided_repositories(
   id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,

ee/tabby-db/src/github_repository_provider.rs

@@ -9,6 +9,7 @@
     pub id: i64,
     pub display_name: String,
     pub access_token: Option<String>,
+    pub synced_at: Option<DateTimeUtc>,
 }
 
 #[derive(FromRow)]
@@ -36,7 +37,7 @@
     pub async fn get_github_provider(&self, id: i64) -> Result<GithubRepositoryProviderDAO> {
         let provider = query_as!(
             GithubRepositoryProviderDAO,
-            "SELECT id, display_name, access_token FROM github_repository_provider WHERE id = ?;",
+            r#"SELECT id, display_name, access_token, synced_at AS "synced_at: DateTimeUtc" FROM github_repository_provider WHERE id = ?;"#,
             id
         )
         .fetch_one(&self.pool)
@@ -54,23 +55,6 @@
         Ok(())
     }
 
-    pub async fn reset_github_provider_access_token(&self, id: i64) -> Result<()> {
-        let res = query!(
-            "UPDATE github_repository_provider SET access_token = NULL WHERE id = ?",
-            id
-        )
-        .execute(&self.pool)
-        .await?;
-
-        if res.rows_affected() != 1 {
-            return Err(anyhow!(
-                "The specified Github repository provider does not exist"
-            ));
-        }
-
-        Ok(())
-    }
-
     pub async fn update_github_provider(
         &self,
         id: i64,
@@ -95,6 +79,19 @@
         Ok(())
     }
 
+    pub async fn update_github_provider_sync_status(&self, id: i64, success: bool) -> Result<()> {
+        let time = success.then_some(DateTimeUtc::now());
+        query!(
+            "UPDATE github_repository_provider SET synced_at = ?, access_token = IIF(?, access_token, NULL) WHERE id = ?",
+            time,
+            success,
+            id
+        )
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
     pub async fn list_github_repository_providers(
         &self,
         ids: Vec<i64>,
@@ -113,7 +110,7 @@
         let providers = query_paged_as!(
             GithubRepositoryProviderDAO,
             "github_repository_provider",
-            ["id", "display_name", "access_token"],
+            ["id", "display_name", "access_token", "synced_at" as "synced_at: DateTimeUtc"],
             limit,
             skip_id,
             backwards,

ee/tabby-db/src/gitlab_repository_provider.rs

@@ -9,6 +9,7 @@
     pub id: i64,
     pub display_name: String,
     pub access_token: Option<String>,
+    pub synced_at: Option<DateTimeUtc>,
 }
 
 #[derive(FromRow)]
@@ -36,7 +37,7 @@
     pub async fn get_gitlab_provider(&self, id: i64) -> Result<GitlabRepositoryProviderDAO> {
         let provider = query_as!(
             GitlabRepositoryProviderDAO,
-            "SELECT id, display_name, access_token FROM gitlab_repository_provider WHERE id = ?;",
+            r#"SELECT id, display_name, access_token, synced_at AS "synced_at: DateTimeUtc" FROM gitlab_repository_provider WHERE id = ?;"#,
             id
         )
         .fetch_one(&self.pool)
@@ -54,23 +55,6 @@
         Ok(())
     }
 
-    pub async fn reset_gitlab_provider_access_token(&self, id: i64) -> Result<()> {
-        let res = query!(
-            "UPDATE gitlab_repository_provider SET access_token = NULL WHERE id = ?",
-            id
-        )
-        .execute(&self.pool)
-        .await?;
-
-        if res.rows_affected() != 1 {
-            return Err(anyhow!(
-                "The specified gitlab repository provider does not exist"
-            ));
-        }
-
-        Ok(())
-    }
-
     pub async fn update_gitlab_provider(
         &self,
         id: i64,
@@ -95,6 +79,19 @@
         Ok(())
     }
 
+    pub async fn update_gitlab_provider_sync_status(&self, id: i64, success: bool) -> Result<()> {
+        let time = success.then_some(DateTimeUtc::now());
+        query!(
+            "UPDATE gitlab_repository_provider SET synced_at = ?, access_token = IIF(?, access_token, NULL) WHERE id = ?",
+            time,
+            success,
+            id
+        )
+        .execute(&self.pool)
+        .await?;
+        Ok(())
+    }
+
     pub async fn list_gitlab_repository_providers(
         &self,
         ids: Vec<i64>,
@@ -113,7 +110,7 @@
         let providers = query_paged_as!(
             GitlabRepositoryProviderDAO,
             "gitlab_repository_provider",
-            ["id", "display_name", "access_token"],
+            ["id", "display_name", "access_token", "synced_at" as "synced_at: DateTimeUtc"],
             limit,
             skip_id,
             backwards,

ee/tabby-ui/app/(dashboard)/settings/(integrations)/repository/github/components/github-form.tsx

@@ -34,13 +34,7 @@ import { IconExternalLink, IconSpinner } from '@/components/ui/icons'
 import { Input } from '@/components/ui/input'
 
 export const createGithubProviderSchema = z.object({
-  displayName: z
-    .string()
-    .trim()
-    .regex(
-      /^[\w-]+$/,
-      'Display name must contain only alphanumeric characters, underscores, and hyphens'
-    ),
+  displayName: z.string(),
   accessToken: z.string()
 })
 

ee/tabby-ui/app/(dashboard)/settings/(integrations)/repository/github/components/github.tsx

@@ -3,7 +3,10 @@
 import Link from 'next/link'
 import { useQuery } from 'urql'
 
-import { ListGithubRepositoryProvidersQuery } from '@/lib/gql/generates/graphql'
+import {
+  ListGithubRepositoryProvidersQuery,
+  RepositoryProviderStatus
+} from '@/lib/gql/generates/graphql'
 import { listGithubRepositoryProviders } from '@/lib/tabby/query'
 import { buttonVariants } from '@/components/ui/button'
 import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'
@@ -65,11 +68,7 @@ const GitProvidersList: React.FC<GitProvidersTableProps> = ({ data }) => {
                 <span className="w-[30%] shrink-0 text-muted-foreground">
                   Status
                 </span>
-                <span>
-                  {item.node?.connected
-                    ? 'Connected'
-                    : 'access_token needs update'}
-                </span>
+                <span>{toStatusMessage(item.node.status)}</span>
               </div>
             </CardContent>
           </Card>
@@ -79,6 +78,17 @@ const GitProvidersList: React.FC<GitProvidersTableProps> = ({ data }) => {
   )
 }
 
+function toStatusMessage(status: RepositoryProviderStatus) {
+  switch (status) {
+    case RepositoryProviderStatus.Ready:
+      return 'Ready'
+    case RepositoryProviderStatus.Error:
+      return 'Processing error. Please check if the access token is still valid'
+    case RepositoryProviderStatus.Pending:
+      return 'Awaiting the next data synchronization'
+  }
+}
+
 const GitProvidersPlaceholder = () => {
   return (
     <div className="flex flex-col items-center gap-4 rounded-lg border-4 border-dashed py-8">

ee/tabby-ui/app/(dashboard)/settings/(integrations)/repository/github/detail/components/detail.tsx

@@ -5,6 +5,7 @@ import { useRouter, useSearchParams } from 'next/navigation'
 import { useQuery } from 'urql'
 
 import { DEFAULT_PAGE_SIZE } from '@/lib/constants'
+import { RepositoryProviderStatus } from '@/lib/gql/generates/graphql'
 import { QueryResponseData, QueryVariables, useMutation } from '@/lib/tabby/gql'
 import {
   listGithubRepositories,
@@ -68,7 +69,7 @@ const DetailPage: React.FC = () => {
 
   return (
     <LoadingWrapper loading={fetching}>
-      <CardTitle className="flex items-center justify-between">
+      <CardTitle className="flex items-center gap-4">
         <div className="-ml-1 flex items-center">
           <Button
             onClick={() => router.back()}
@@ -81,11 +82,7 @@ const DetailPage: React.FC = () => {
         </div>
         <div className="flex items-center gap-2 text-base">
           <div className="ml-1">
-            {provider?.connected ? (
-              <Badge variant="successful">Connected</Badge>
-            ) : (
-              <Badge variant="destructive">Not Connected</Badge>
-            )}
+            {provider && toStatusBadge(provider.status)}
           </div>
         </div>
       </CardTitle>
@@ -108,6 +105,17 @@ const DetailPage: React.FC = () => {
   )
 }
 
+function toStatusBadge(status: RepositoryProviderStatus) {
+  switch (status) {
+    case RepositoryProviderStatus.Ready:
+      return <Badge variant="successful">Ready</Badge>
+    case RepositoryProviderStatus.Error:
+      return <Badge variant="destructive">Error</Badge>
+    case RepositoryProviderStatus.Error:
+      return <Badge>Pending</Badge>
+  }
+}
+
 const LinkedRepoTable: React.FC<{
   data: GithubRepositories | undefined
   onDelete?: () => void

ee/tabby-ui/lib/tabby/query.ts

@@ -186,7 +186,7 @@ export const listGithubRepositoryProviders = graphql(/* GraphQL */ `
         node {
           id
           displayName
-          connected
+          status
         }
         cursor
       }

ee/tabby-webserver/graphql/schema.graphql

@@ -13,7 +13,7 @@ input UpdateRepositoryProviderInput {
 type GitlabRepositoryProvider {
   id: ID!
   displayName: String!
-  connected: Boolean!
+  status: RepositoryProviderStatus!
 }
 
 enum Language {
@@ -119,6 +119,12 @@ type UserConnection {
   pageInfo: PageInfo!
 }
 
+enum RepositoryProviderStatus {
+  READY
+  PENDING
+  ERROR
+}
+
 type DiskUsageStats {
   events: DiskUsage!
   indexedRepositories: DiskUsage!
@@ -142,7 +148,7 @@ type ServerInfo {
 type GithubRepositoryProvider {
   id: ID!
   displayName: String!
-  connected: Boolean!
+  status: RepositoryProviderStatus!
 }
 
 input PasswordChangeInput {

ee/tabby-webserver/src/cron/github.rs

@@ -38,7 +38,7 @@
             ..
         }) if source.status_code.is_client_error() => {
             service
-                .reset_github_repository_provider_access_token(provider.id.clone())
+                .update_github_repository_provider_sync_status(provider.id.clone(), false)
                 .await?;
             warn!(
                 "GitHub credentials for provider {} are expired or invalid",
@@ -66,6 +66,9 @@
             .upsert_github_provided_repository(provider.id.clone(), id, repo.name, url)
             .await?;
     }
+    service
+        .update_github_repository_provider_sync_status(provider.id.clone(), true)
+        .await?;
 
     Ok(())
 }

ee/tabby-webserver/src/cron/gitlab.rs

@@ -39,7 +39,7 @@
         Ok(repos) => repos,
         Err(e) if e.to_string().contains("401 Unauthorized") => {
             service
-                .reset_gitlab_repository_provider_access_token(provider.id.clone())
+                .update_gitlab_repository_provider_sync_status(provider.id.clone(), false)
                 .await?;
             warn!(
                 "GitLab credentials for provider {} are expired or invalid",
@@ -64,6 +64,9 @@
             )
             .await?;
     }
+    service
+        .update_gitlab_repository_provider_sync_status(provider.id.clone(), true)
+        .await?;
 
     Ok(())
 }