From f852da9735b117f51603b284d4d07672b40e4a8b Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Thu, 11 Apr 2024 08:35:36 -0700 Subject: [PATCH] refactor(webserver): remove /resolve/meta api endpoint (#1816) * refactor(webserver): remove /resolve/meta api endpoint * feat(ui): detect language by filename * update * move filename2prism to @/lib * [autofix.ci] apply automated fixes --------- Co-authored-by: liangfung <1098486429@qq.com> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- ee/tabby-db/src/repositories.rs | 12 +- .../app/files/components/code-editor-view.tsx | 11 +- .../files/components/source-code-browser.tsx | 18 +- .../app/files/components/text-file-view.tsx | 9 +- ee/tabby-ui/lib/filename2prism/index.ts | 40 + ee/tabby-ui/lib/filename2prism/languages.ts | 861 ++++++++++++++++++ ee/tabby-ui/lib/types/repositories.ts | 10 - ee/tabby-ui/package.json | 1 - ee/tabby-ui/yarn.lock | 5 - ee/tabby-webserver/src/cron/mod.rs | 68 +- ee/tabby-webserver/src/cron/scheduler.rs | 6 +- ee/tabby-webserver/src/handler.rs | 9 +- ee/tabby-webserver/src/repositories/mod.rs | 35 +- .../src/repositories/resolve.rs | 164 +--- ee/tabby-webserver/src/schema/repository.rs | 1 + ee/tabby-webserver/src/service/repository.rs | 4 + 16 files changed, 951 insertions(+), 303 deletions(-) create mode 100644 ee/tabby-ui/lib/filename2prism/index.ts create mode 100644 ee/tabby-ui/lib/filename2prism/languages.ts diff --git a/ee/tabby-db/src/repositories.rs b/ee/tabby-db/src/repositories.rs index eebf102aa682..6c75d3b7d590 100644 --- a/ee/tabby-db/src/repositories.rs +++ b/ee/tabby-db/src/repositories.rs @@ -68,11 +68,13 @@ impl DbConn { } pub async fn get_repository_by_name(&self, name: &str) -> Result { - let repository = - sqlx::query_as("SELECT id, name, git_url FROM repositories WHERE name = ?;") - .bind(name) - .fetch_one(&self.pool) - .await?; + let repository = sqlx::query_as!( + RepositoryDAO, + "SELECT id as 'id!: i64', name, git_url FROM repositories WHERE name = ?", + name + ) + .fetch_one(&self.pool) + .await?; Ok(repository) } } diff --git a/ee/tabby-ui/app/files/components/code-editor-view.tsx b/ee/tabby-ui/app/files/components/code-editor-view.tsx index 09d9caebe62b..78c881506254 100644 --- a/ee/tabby-ui/app/files/components/code-editor-view.tsx +++ b/ee/tabby-ui/app/files/components/code-editor-view.tsx @@ -5,7 +5,7 @@ import { useTheme } from 'next-themes' import { EXP_enable_code_browser_quick_action_bar } from '@/lib/experiment-flags' import { useIsChatEnabled } from '@/lib/hooks/use-server-info' -import { TFileMeta } from '@/lib/types' +import { TCodeTag } from '@/lib/types' import CodeEditor, { CodeMirrorEditorRef } from '@/components/codemirror/codemirror' @@ -17,17 +17,12 @@ import { ActionBarWidgetExtension } from './action-bar-widget/action-bar-widget- interface CodeEditorViewProps { value: string - meta: TFileMeta | undefined language: string } -const CodeEditorView: React.FC = ({ - value, - meta, - language -}) => { +const CodeEditorView: React.FC = ({ value, language }) => { const { theme } = useTheme() - const tags = meta?.tags + const tags: TCodeTag[] = [] const editorRef = React.useRef(null) const isChatEnabled = useIsChatEnabled() diff --git a/ee/tabby-ui/app/files/components/source-code-browser.tsx b/ee/tabby-ui/app/files/components/source-code-browser.tsx index 97a4a252db69..c8c758c586b3 100644 --- a/ee/tabby-ui/app/files/components/source-code-browser.tsx +++ b/ee/tabby-ui/app/files/components/source-code-browser.tsx @@ -1,13 +1,13 @@ 'use client' import React, { PropsWithChildren } from 'react' -import filename2prism from 'filename2prism' import { compact, findIndex, toNumber } from 'lodash-es' import { ImperativePanelHandle } from 'react-resizable-panels' import { toast } from 'sonner' import { SWRResponse } from 'swr' import useSWRImmutable from 'swr/immutable' +import filename2prism from '@/lib/filename2prism' import useRouterStuff from '@/lib/hooks/use-router-stuff' import fetcher from '@/lib/tabby/fetcher' import type { ResolveEntriesResponse, TFile } from '@/lib/types' @@ -254,14 +254,6 @@ const SourceCodeBrowserRenderer: React.FC = ({ const fileBlob = rawFileResponse?.blob const contentLength = rawFileResponse?.contentLength - // fetch active file meta - const { data: fileMeta } = useSWRImmutable( - isFileSelected - ? `/repositories/${activeRepoName}/meta/${activeBasename}` - : null, - fetcher - ) - // fetch active dir const { data: subTree, @@ -398,11 +390,7 @@ const SourceCodeBrowserRenderer: React.FC = ({ /> )} {showTextFileView && ( - + )} {showRawFileView && ( import('./markdown-view')) interface TextFileViewProps extends React.HTMLProps { blob: Blob | undefined - meta: TFileMeta | undefined contentLength?: number } export const TextFileView: React.FC = ({ className, blob, - meta, contentLength }) => { const { searchParams, updateSearchParams } = useRouterStuff() @@ -32,7 +29,7 @@ export const TextFileView: React.FC = ({ const detectedLanguage = activePath ? filename2prism(activePath)[0] : undefined - const language = (meta?.language ?? detectedLanguage) || '' + const language = detectedLanguage ?? 'plain' const isMarkdown = !!value && language === 'markdown' const isPlain = searchParams.get('plain')?.toString() === '1' const showMarkdown = isMarkdown && !isPlain @@ -86,7 +83,7 @@ export const TextFileView: React.FC = ({ {showMarkdown ? ( ) : ( - + )} diff --git a/ee/tabby-ui/lib/filename2prism/index.ts b/ee/tabby-ui/lib/filename2prism/index.ts new file mode 100644 index 000000000000..d6396da7b1b4 --- /dev/null +++ b/ee/tabby-ui/lib/filename2prism/index.ts @@ -0,0 +1,40 @@ +// Fork from +// https://github.com/TomerAberbach/filename2prism/ + +import path from 'path' +import { has } from 'lodash-es' + +import languages from './languages' + +const filenames: Record = {} +const extnames: Record = {} + +for (const [alias, associations] of Object.entries(languages)) { + for (const filename of associations.filenames) { + if (!has(filenames, filename)) { + filenames[filename] = [] + } + + filenames[filename].push(alias) + } + + for (const extname of associations.extnames) { + if (!has(extnames, extname)) { + extnames[extname] = [] + } + + extnames[extname].push(alias) + } +} + +const filename2prism: (filename: string) => Array = filename => { + const result: string[] = [] + return result + .concat( + filenames[path.basename(filename)], + extnames[path.extname(filename).substring(1)] + ) + .filter(Boolean) +} + +export default filename2prism diff --git a/ee/tabby-ui/lib/filename2prism/languages.ts b/ee/tabby-ui/lib/filename2prism/languages.ts new file mode 100644 index 000000000000..7a9eb118eada --- /dev/null +++ b/ee/tabby-ui/lib/filename2prism/languages.ts @@ -0,0 +1,861 @@ +const languages = { + abap: { + filenames: [], + extnames: [`abap`] + }, + actionscript: { + filenames: [], + extnames: [`as`] + }, + ada: { + filenames: [], + extnames: [`ada`, `adb`, `ads`] + }, + apacheconf: { + filenames: [`.htaccess`, `apache2.conf`, `httpd.conf`], + extnames: [`apacheconf`, `vhost`] + }, + apl: { + filenames: [], + extnames: [`apl`, `dyalog`] + }, + applescript: { + filenames: [], + extnames: [`applescript`, `scpt`] + }, + arff: { + filenames: [], + extnames: [`arff`] + }, + asciidoc: { + filenames: [], + extnames: [`asciidoc`, `adoc`, `asc`] + }, + asm6502: { + filenames: [], + extnames: [`asm`] + }, + autohotkey: { + filenames: [], + extnames: [`ahk`, `ahkl`] + }, + autoit: { + filenames: [], + extnames: [`au3`] + }, + bash: { + filenames: [ + `.bash_history`, + `.bash_logout`, + `.bash_profile`, + `.bashrc`, + `.cshrc`, + `.login`, + `.profile`, + `.zlogin`, + `.zlogout`, + `.zprofile`, + `.zshenv`, + `.zshrc`, + `9fs`, + `PKGBUILD`, + `bash_logout`, + `bash_profile`, + `bashrc`, + `cshrc`, + `gradlew`, + `login`, + `man`, + `profile`, + `zlogin`, + `zlogout`, + `zprofile`, + `zshenv`, + `zshrc` + ], + extnames: [ + `sh`, + `bash`, + `bats`, + `cgi`, + `command`, + `fcgi`, + `ksh`, + `tmux`, + `tool`, + `zsh` + ] + }, + basic: { + filenames: [], + extnames: [`vb`, `bas`, `cls`, `frm`, `frx`, `vba`, `vbhtml`, `vbs`] + }, + batch: { + filenames: [], + extnames: [`bat`, `cmd`] + }, + bison: { + filenames: [], + extnames: [`bison`] + }, + brainfuck: { + filenames: [], + extnames: [`b`, `bf`] + }, + bro: { + filenames: [], + extnames: [`bro`] + }, + c: { + filenames: [], + extnames: [`c`, `cats`, `h`, `idc`] + }, + csharp: { + filenames: [], + extnames: [`cs`, `cake`, `cshtml`, `csx`] + }, + cpp: { + filenames: [], + extnames: [ + `cpp`, + `c++`, + `cc`, + `cp`, + `cxx`, + `h`, + `h++`, + `hh`, + `hpp`, + `hxx`, + `inc`, + `inl`, + `ino`, + `ipp`, + `re`, + `tcc`, + `tpp` + ] + }, + coffeescript: { + filenames: [`Cakefile`], + extnames: [`coffee`, `_coffee`, `cake`, `cjsx`, `iced`] + }, + clojure: { + filenames: [`riemann.config`], + extnames: [ + `clj`, + `boot`, + `cl2`, + `cljc`, + `cljs`, + `cljs.hl`, + `cljscm`, + `cljx`, + `hic` + ] + }, + crystal: { + filenames: [], + extnames: [`cr`] + }, + css: { + filenames: [], + extnames: [`css`] + }, + d: { + filenames: [], + extnames: [`d`, `di`] + }, + dart: { + filenames: [], + extnames: [`dart`] + }, + diff: { + filenames: [], + extnames: [`diff`, `patch`] + }, + django: { + filenames: [], + extnames: [`jinja`, `jinja2`, `mustache`, `njk`] + }, + dockerfile: { + filenames: [`Dockerfile`], + extnames: [`dockerfile`] + }, + eiffel: { + filenames: [], + extnames: [`e`] + }, + elixir: { + filenames: [`mix.lock`], + extnames: [`ex`, `exs`] + }, + elm: { + filenames: [], + extnames: [`elm`] + }, + erb: { + filenames: [], + extnames: [`erb`] + }, + erlang: { + filenames: [`Emakefile`, `rebar.config`, `rebar.config.lock`, `rebar.lock`], + extnames: [`erl`, `app.src`, `es`, `escript`, `hrl`, `xrl`, `yrl`] + }, + fsharp: { + filenames: [], + extnames: [`fs`, `fsi`, `fsx`] + }, + fortran: { + filenames: [], + extnames: [`f90`, `f`, `f03`, `f08`, `f77`, `f95`, `for`, `fpp`] + }, + gedcom: { + filenames: [], + extnames: [`ged`] + }, + gherkin: { + filenames: [], + extnames: [`feature`] + }, + glsl: { + filenames: [], + extnames: [ + `glsl`, + `fp`, + `frag`, + `frg`, + `fs`, + `fsh`, + `fshader`, + `geo`, + `geom`, + `glslv`, + `gshader`, + `shader`, + `tesc`, + `tese`, + `vert`, + `vrx`, + `vsh`, + `vshader` + ] + }, + go: { + filenames: [], + extnames: [`go`] + }, + graphql: { + filenames: [], + extnames: [`graphql`, `gql`] + }, + groovy: { + filenames: [`Jenkinsfile`], + extnames: [`groovy`, `grt`, `gtpl`, `gvy`] + }, + haml: { + filenames: [], + extnames: [`haml`] + }, + handlebars: { + filenames: [], + extnames: [`handlebars`, `hbs`] + }, + haskell: { + filenames: [], + extnames: [`hs`, `hsc`] + }, + haxe: { + filenames: [], + extnames: [`hx`, `hxsl`] + }, + http: { + filenames: [], + extnames: [`http`] + }, + icon: { + filenames: [], + extnames: [`icn`] + }, + inform7: { + filenames: [], + extnames: [`ni`, `i7x`] + }, + ini: { + filenames: [`.editorconfig`, `.gitconfig`], + extnames: [`ini`, `cfg`, `lektorproject`, `prefs`, `pro`, `properties`] + }, + io: { + filenames: [], + extnames: [`io`] + }, + j: { + filenames: [], + extnames: [`ijs`] + }, + java: { + filenames: [], + extnames: [`java`] + }, + javascript: { + filenames: [`Jakefile`], + extnames: [ + `js`, + `_js`, + `cjs`, + `bones`, + `es`, + `es6`, + `frag`, + `gs`, + `jake`, + `jsb`, + `jscad`, + `jsfl`, + `jsm`, + `jss`, + `mjs`, + `njs`, + `pac`, + `sjs`, + `ssjs`, + `xsjs`, + `xsjslib` + ] + }, + jolie: { + filenames: [], + extnames: [`ol`, `iol`] + }, + json: { + filenames: [ + `.arcconfig`, + `.htmlhintrc`, + `.tern-config`, + `.tern-project`, + `composer.lock`, + `mcmod.info` + ], + extnames: [ + `json`, + `avsc`, + `geojson`, + `gltf`, + `JSON-tmLanguage`, + `jsonl`, + `tfstate`, + `topojson`, + `webapp`, + `webmanifest`, + `yy`, + `yyp` + ] + }, + julia: { + filenames: [], + extnames: [`jl`] + }, + keyman: { + filenames: [], + extnames: [`kmn`] + }, + kotlin: { + filenames: [], + extnames: [`kt`, `ktm`, `kts`] + }, + latex: { + filenames: [], + extnames: [ + `tex`, + `aux`, + `bbx`, + `bib`, + `cbx`, + `cls`, + `dtx`, + `ins`, + `lbx`, + `ltx`, + `mkii`, + `mkiv`, + `mkvi`, + `sty`, + `toc` + ] + }, + less: { + filenames: [], + extnames: [`less`] + }, + liquid: { + filenames: [], + extnames: [`liquid`] + }, + lisp: { + filenames: [], + extnames: [`lisp`, `asd`, `cl`, `l`, `lsp`, `ny`, `podsl`, `sexp`] + }, + livescript: { + filenames: [`Slakefile`], + extnames: [`ls`, `_ls`] + }, + lolcode: { + filenames: [], + extnames: [`lol`] + }, + lua: { + filenames: [], + extnames: [`lua`, `fcgi`, `nse`, `p8`, `pd_lua`, `rbxs`, `wlua`] + }, + cmake: { + filenames: [ + `BSDmakefile`, + `GNUmakefile`, + `Kbuild`, + `Makefile`, + `Makefile.am`, + `Makefile.boot`, + `Makefile.frag`, + `Makefile.in`, + `Makefile.inc`, + `Makefile.wat`, + `makefile`, + `makefile.sco`, + `mkfile` + ], + extnames: [`mak`, `d`, `make`, `mk`, `mkfile`] + }, + markdown: { + filenames: [`contents.lr`, `LICENSE`], + extnames: [ + `md`, + `markdown`, + `mdown`, + `mdwn`, + `mkd`, + `mkdn`, + `mkdown`, + `ronn`, + `workbook` + ] + }, + markup: { + filenames: [], + extnames: [ + `apib`, + `blade`, + `chem`, + `ecr`, + `eex`, + `ejs`, + `html`, + `htm`, + `ipynb`, + `kit`, + `latte`, + `marko`, + `mask`, + `mtml`, + `phtml`, + `pic`, + `raml`, + `rhtml`, + `vue`, + `xht`, + `xhtml` + ] + }, + matlab: { + filenames: [], + extnames: [`matlab`, `m`] + }, + mel: { + filenames: [], + extnames: [`mel`] + }, + mizar: { + filenames: [], + extnames: [`miz`, `voc`] + }, + monkey: { + filenames: [], + extnames: [`monkey`, `monkey2`] + }, + n4js: { + filenames: [], + extnames: [`n4jsd`] + }, + nasm: { + filenames: [], + extnames: [`nasm`] + }, + nginx: { + filenames: [`nginx.conf`], + extnames: [`nginxconf`, `vhost`] + }, + nim: { + filenames: [], + extnames: [`nim`, `nimrod`] + }, + nix: { + filenames: [], + extnames: [`nix`] + }, + nsis: { + filenames: [], + extnames: [`nsi`, `nsh`] + }, + objectivec: { + filenames: [], + extnames: [`m`, `h`] + }, + ocaml: { + filenames: [], + extnames: [`ml`, `eliom`, `eliomi`, `ml4`, `mli`, `mll`, `mly`] + }, + opencl: { + filenames: [], + extnames: [`opencl`, `cl`] + }, + oz: { + filenames: [], + extnames: [`oz`] + }, + pascal: { + filenames: [], + extnames: [`pas`, `dfm`, `dpr`, `inc`, `lpr`, `pascal`, `pp`] + }, + perl: { + filenames: [`Makefile.PL`, `Rexfile`, `ack`, `cpanfile`], + extnames: [ + `pl`, + `al`, + `cgi`, + `fcgi`, + `perl`, + `ph`, + `plx`, + `pm`, + `psgi`, + `t` + ] + }, + php: { + filenames: [`.php`, `.php_cs`, `.php_cs.dist`, `Phakefile`], + extnames: [ + `php`, + `aw`, + `ctp`, + `fcgi`, + `inc`, + `php3`, + `php4`, + `php5`, + `phps`, + `phpt` + ] + }, + plsql: { + filenames: [], + extnames: [ + `pls`, + `bdy`, + `ddl`, + `fnc`, + `pck`, + `pkb`, + `pks`, + `plb`, + `plsql`, + `prc`, + `spc`, + `tpb`, + `tps`, + `trg`, + `vw` + ] + }, + powershell: { + filenames: [], + extnames: [`ps1`, `psd1`, `psm1`] + }, + processing: { + filenames: [], + extnames: [`pde`] + }, + prolog: { + filenames: [], + extnames: [`pl`, `pro`, `prolog`, `yap`] + }, + properties: { + filenames: [], + extnames: [`properties`] + }, + protobuf: { + filenames: [], + extnames: [`proto`] + }, + pug: { + filenames: [], + extnames: [`jade`, `pug`] + }, + puppet: { + filenames: [`Modulefile`], + extnames: [`pp`] + }, + pure: { + filenames: [], + extnames: [`pure`] + }, + python: { + filenames: [ + `.gclient`, + `BUCK`, + `BUILD`, + `BUILD.bazel`, + `SConscript`, + `SConstruct`, + `Snakefile`, + `WORKSPACE`, + `wscript` + ], + extnames: [ + `py`, + `bzl`, + `cgi`, + `fcgi`, + `gyp`, + `gypi`, + `lmi`, + `py3`, + `pyde`, + `pyi`, + `pyp`, + `pyt`, + `pyw`, + `rpy`, + `spec`, + `tac`, + `wsgi`, + `xpy` + ] + }, + q: { + filenames: [], + extnames: [`q`] + }, + qore: { + filenames: [], + extnames: [`q`, `qm`, `qtest`] + }, + r: { + filenames: [`.Rprofile`, `expr-dist`], + extnames: [`r`, `rd`, `rsx`] + }, + jsx: { + filenames: [], + extnames: [`jsx`] + }, + toml: { + filenames: [], + extnames: [`toml`] + }, + tsx: { + filenames: [], + extnames: [`tsx`] + }, + renpy: { + filenames: [], + extnames: [`rpy`] + }, + reason: { + filenames: [], + extnames: [`re`, `rei`] + }, + rest: { + filenames: [], + extnames: [`rst`, `rest`] + }, + rip: { + filenames: [], + extnames: [`rip`] + }, + ruby: { + filenames: [ + `.irbrc`, + `.pryrc`, + `Appraisals`, + `Berksfile`, + `Brewfile`, + `Buildfile`, + `Capfile`, + `Dangerfile`, + `Deliverfile`, + `Fastfile`, + `Gemfile`, + `Gemfile.lock`, + `Guardfile`, + `Jarfile`, + `Mavenfile`, + `Podfile`, + `Puppetfile`, + `Rakefile`, + `Snapfile`, + `Thorfile`, + `Vagrantfile`, + `buildfile` + ], + extnames: [ + `rb`, + `builder`, + `eye`, + `fcgi`, + `gemspec`, + `god`, + `jbuilder`, + `mspec`, + `pluginspec`, + `podspec`, + `rabl`, + `rake`, + `rbuild`, + `rbw`, + `rbx`, + `ru`, + `ruby`, + `spec`, + `thor`, + `watchr` + ] + }, + rust: { + filenames: [], + extnames: [`rs`] + }, + sas: { + filenames: [], + extnames: [`sas`] + }, + sass: { + filenames: [], + extnames: [`sass`] + }, + scss: { + filenames: [], + extnames: [`scss`] + }, + scala: { + filenames: [], + extnames: [`scala`, `kojo`, `sbt`, `sc`] + }, + scheme: { + filenames: [], + extnames: [`scm`, `sch`, `sld`, `sls`, `sps`, `ss`] + }, + smalltalk: { + filenames: [], + extnames: [`st`, `cs`] + }, + smarty: { + filenames: [], + extnames: [`tpl`] + }, + sql: { + filenames: [], + extnames: [`sql`, `cql`, `ddl`, `inc`, `mysql`, `prc`, `tab`, `udf`, `viw`] + }, + soy: { + filenames: [], + extnames: [`soy`] + }, + stylus: { + filenames: [], + extnames: [`styl`] + }, + swift: { + filenames: [], + extnames: [`swift`] + }, + tcl: { + filenames: [`owh`, `starfield`], + extnames: [`tcl`, `adp`, `tm`] + }, + textile: { + filenames: [], + extnames: [`textile`] + }, + tt2: { + filenames: [], + extnames: [`pm`] + }, + twig: { + filenames: [], + extnames: [`twig`] + }, + typescript: { + filenames: [], + extnames: [`ts`] + }, + velocity: { + filenames: [], + extnames: [`vm`] + }, + verilog: { + filenames: [], + extnames: [`v`, `veo`] + }, + vhdl: { + filenames: [], + extnames: [`vhdl`, `vhd`, `vhf`, `vhi`, `vho`, `vhs`, `vht`, `vhw`] + }, + vim: { + filenames: [ + `.gvimrc`, + `.nvimrc`, + `.vimrc`, + `_vimrc`, + `gvimrc`, + `nvimrc`, + `vimrc` + ], + extnames: [`vim`] + }, + 'visual-basic': { + filenames: [], + extnames: [`vb`, `bas`, `cls`, `frm`, `frx`, `vba`, `vbhtml`, `vbs`] + }, + wasm: { + filenames: [], + extnames: [`wast`, `wat`] + }, + xojo: { + filenames: [], + extnames: [ + `xojo_code`, + `xojo_menu`, + `xojo_report`, + `xojo_script`, + `xojo_toolbar`, + `xojo_window` + ] + }, + xquery: { + filenames: [], + extnames: [`xquery`, `xq`, `xql`, `xqm`, `xqy`] + }, + yaml: { + filenames: [`.clang-format`, `.clang-tidy`, `.gemrc`, `glide.lock`], + extnames: [ + `yml`, + `mir`, + `reek`, + `rviz`, + `sublime-syntax`, + `syntax`, + `yaml`, + `yaml-tmlanguage` + ] + } +} + +export default languages diff --git a/ee/tabby-ui/lib/types/repositories.ts b/ee/tabby-ui/lib/types/repositories.ts index 52f23ae57b6c..4c1579236da1 100644 --- a/ee/tabby-ui/lib/types/repositories.ts +++ b/ee/tabby-ui/lib/types/repositories.ts @@ -17,14 +17,4 @@ export type TCodeTag = { span: TPointRange } -export type TFileMeta = { - git_url: string - filepath: string - language: string - max_line_length: number - avg_line_length: number - alphanum_fraction: number - tags: TCodeTag[] -} - export type ResolveEntriesResponse = { entries: TFile[] } diff --git a/ee/tabby-ui/package.json b/ee/tabby-ui/package.json index 3d76ed0481f7..eab5446e50f4 100644 --- a/ee/tabby-ui/package.json +++ b/ee/tabby-ui/package.json @@ -58,7 +58,6 @@ "copy-to-clipboard": "^3.3.3", "date-fns": "^3.6.0", "downshift": "^8.2.2", - "filename2prism": "^3.0.0", "focus-trap-react": "^10.1.1", "graphql": "^16.8.1", "humanize-duration": "^3.31.0", diff --git a/ee/tabby-ui/yarn.lock b/ee/tabby-ui/yarn.lock index 3d6dc0a6b8ed..9d649ac8d2b0 100644 --- a/ee/tabby-ui/yarn.lock +++ b/ee/tabby-ui/yarn.lock @@ -4599,11 +4599,6 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" -filename2prism@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/filename2prism/-/filename2prism-3.0.0.tgz#a578ca00ba6f491199ab0081ad5899ac646cae0f" - integrity sha512-s++aGNJvllroqr9KHed8ne8UqOcOtOnYYBnh7j5nVlhdRCn1ayXuT+bwkuDQu0yknlNA5ai3CzAbDu3+ka6IKQ== - fill-range@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" diff --git a/ee/tabby-webserver/src/cron/mod.rs b/ee/tabby-webserver/src/cron/mod.rs index 4992efee823c..554bca068826 100644 --- a/ee/tabby-webserver/src/cron/mod.rs +++ b/ee/tabby-webserver/src/cron/mod.rs @@ -3,45 +3,10 @@ mod scheduler; use std::sync::Arc; -use futures::Future; -use tokio::sync::broadcast::{self, error::RecvError, Receiver}; use tokio_cron_scheduler::{Job, JobScheduler}; use crate::schema::{auth::AuthenticationService, job::JobService, worker::WorkerService}; -pub(crate) struct CronEvents { - pub scheduler_job_succeeded: Receiver<()>, -} - -pub trait StartListener { - fn start_listener(&self, handler: F) - where - F: Fn(E) -> Fut + Send + 'static, - Fut: Future + Send, - E: Clone + Send + 'static; -} - -impl StartListener for Receiver { - fn start_listener(&self, handler: F) - where - F: Fn(E) -> Fut + Send + 'static, - Fut: Future + Send, - E: Clone + Send + 'static, - { - let mut recv = self.resubscribe(); - tokio::spawn(async move { - loop { - let event = match recv.recv().await { - Ok(event) => event, - Err(RecvError::Closed) => break, - Err(_) => continue, - }; - handler(event).await; - } - }); - } -} - async fn new_job_scheduler(jobs: Vec) -> anyhow::Result { let scheduler = JobScheduler::new().await?; for job in jobs { @@ -56,9 +21,8 @@ pub async fn run_cron( job: Arc, worker: Arc, local_port: u16, -) -> CronEvents { +) { let mut jobs = vec![]; - let (send_scheduler_complete, receive_scheduler_complete) = broadcast::channel::<()>(2); let job1 = db::refresh_token_job(auth.clone()) .await @@ -70,7 +34,7 @@ pub async fn run_cron( .expect("failed to create password reset token cleanup job"); jobs.push(job2); - let job3 = scheduler::scheduler_job(job.clone(), worker, send_scheduler_complete, local_port) + let job3 = scheduler::scheduler_job(job.clone(), worker, local_port) .await .expect("failed to create scheduler job"); jobs.push(job3); @@ -83,32 +47,4 @@ pub async fn run_cron( new_job_scheduler(jobs) .await .expect("failed to start job scheduler"); - CronEvents { - scheduler_job_succeeded: receive_scheduler_complete, - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use tokio::sync::Mutex; - - use super::*; - - #[tokio::test] - async fn test_receiver_events() { - let (send, receive) = broadcast::channel(1); - let counter = Arc::new(Mutex::new(0)); - let clone = counter.clone(); - receive.start_listener(move |_| { - let clone = clone.clone(); - async move { - *clone.lock().await += 1; - } - }); - send.send(()).unwrap(); - tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!(*counter.lock().await, 1); - } } diff --git a/ee/tabby-webserver/src/cron/scheduler.rs b/ee/tabby-webserver/src/cron/scheduler.rs index 2b869b210f7d..bbcc14c95cb8 100644 --- a/ee/tabby-webserver/src/cron/scheduler.rs +++ b/ee/tabby-webserver/src/cron/scheduler.rs @@ -2,7 +2,7 @@ use std::{pin::Pin, process::Stdio, sync::Arc}; use anyhow::{Context, Result}; use futures::Future; -use tokio::{io::AsyncBufReadExt, sync::broadcast}; +use tokio::io::AsyncBufReadExt; use tokio_cron_scheduler::{Job, JobScheduler}; use tracing::{error, info, warn}; @@ -11,7 +11,6 @@ use crate::schema::{job::JobService, worker::WorkerService}; pub async fn scheduler_job( job: Arc, worker: Arc, - events: broadcast::Sender<()>, local_port: u16, ) -> anyhow::Result { let scheduler_mutex = Arc::new(tokio::sync::Mutex::new(())); @@ -21,7 +20,6 @@ pub async fn scheduler_job( let worker = worker.clone(); let job = job.clone(); let scheduler_mutex = scheduler_mutex.clone(); - let events = events.clone(); Box::pin(async move { let Ok(_guard) = scheduler_mutex.try_lock() else { warn!("Scheduler job overlapped, skipping..."); @@ -30,8 +28,6 @@ pub async fn scheduler_job( if let Err(err) = run_scheduler_now(job, worker, local_port).await { error!("Failed to run scheduler job, reason: `{}`", err); - } else { - let _ = events.send(()); } if let Ok(Some(next_tick)) = scheduler.next_tick_for_job(uuid).await { diff --git a/ee/tabby-webserver/src/handler.rs b/ee/tabby-webserver/src/handler.rs index 4a0c412c7ed6..dc87d9b95f02 100644 --- a/ee/tabby-webserver/src/handler.rs +++ b/ee/tabby-webserver/src/handler.rs @@ -21,7 +21,7 @@ use tracing::{error, warn}; use crate::{ cron, hub, integrations, oauth, path::db_file, - repositories::{self, RepositoryCache}, + repositories::{self}, schema::{auth::AuthenticationService, create_schema, Schema, ServiceLocator}, service::{create_service_locator, event_logger::create_event_logger}, ui, @@ -56,12 +56,9 @@ impl WebserverHandle { ) -> (Router, Router) { let ctx = create_service_locator(self.logger(), code, self.db.clone(), is_chat_enabled).await; - let events = cron::run_cron(ctx.auth(), ctx.job(), ctx.worker(), local_port).await; - - let repository_cache = RepositoryCache::new_initialized(ctx.repository(), &events).await; + cron::run_cron(ctx.auth(), ctx.job(), ctx.worker(), local_port).await; let schema = Arc::new(create_schema()); - let rs = Arc::new(repository_cache); let api = api .route( @@ -85,7 +82,7 @@ impl WebserverHandle { ) .nest( "/repositories", - repositories::routes(rs.clone(), ctx.auth()), + repositories::routes(ctx.repository(), ctx.auth()), ) .nest( "/integrations/github", diff --git a/ee/tabby-webserver/src/repositories/mod.rs b/ee/tabby-webserver/src/repositories/mod.rs index 3fd02745e521..90d73a761c8b 100644 --- a/ee/tabby-webserver/src/repositories/mod.rs +++ b/ee/tabby-webserver/src/repositories/mod.rs @@ -8,20 +8,21 @@ use axum::{ http::StatusCode, middleware::from_fn_with_state, response::Response, - routing, Json, Router, + routing, Router, }; -pub use resolve::RepositoryCache; use tracing::{instrument, warn}; +use self::resolve::ResolveState; use crate::{ handler::require_login_middleware, - repositories::resolve::{RepositoryMeta, ResolveParams}, - schema::auth::AuthenticationService, + repositories::resolve::ResolveParams, + schema::{auth::AuthenticationService, repository::RepositoryService}, }; -pub type ResolveState = Arc; - -pub fn routes(rs: Arc, auth: Arc) -> Router { +pub fn routes( + repository: Arc, + auth: Arc, +) -> Router { Router::new() .route("/resolve", routing::get(resolve)) .route("/resolve/", routing::get(resolve)) @@ -29,9 +30,7 @@ pub fn routes(rs: Arc, auth: Arc) -> Ro .route("/:name/resolve/.git/*path", routing::get(not_found)) .route("/:name/resolve/", routing::get(resolve_path)) .route("/:name/resolve/*path", routing::get(resolve_path)) - .route("/:name/meta/", routing::get(meta)) - .route("/:name/meta/*path", routing::get(meta)) - .with_state(rs.clone()) + .with_state(Arc::new(ResolveState::new(repository))) .fallback(not_found) .layer(from_fn_with_state(auth, require_login_middleware)) } @@ -40,12 +39,12 @@ async fn not_found() -> StatusCode { StatusCode::NOT_FOUND } -#[instrument(skip(repo))] +#[instrument(skip(rs))] async fn resolve_path( State(rs): State>, Path(repo): Path, ) -> Result { - let Some(conf) = rs.find_repository(repo.name_str()) else { + let Some(conf) = rs.find_repository(repo.name_str()).await else { return Err(StatusCode::NOT_FOUND); }; let root = conf.dir(); @@ -74,18 +73,6 @@ async fn resolve_path( } } -#[instrument(skip(repo))] -async fn meta( - State(rs): State>, - Path(repo): Path, -) -> Result, StatusCode> { - let key = repo.dataset_key(); - if let Some(resp) = rs.resolve_meta(&key) { - return Ok(Json(resp)); - } - Err(StatusCode::NOT_FOUND) -} - async fn resolve(State(rs): State>) -> Result { rs.resolve_all().await.map_err(|_| StatusCode::NOT_FOUND) } diff --git a/ee/tabby-webserver/src/repositories/resolve.rs b/ee/tabby-webserver/src/repositories/resolve.rs index 6e2a6d53f143..041e2bea4864 100644 --- a/ee/tabby-webserver/src/repositories/resolve.rs +++ b/ee/tabby-webserver/src/repositories/resolve.rs @@ -1,10 +1,4 @@ -use std::{ - collections::HashMap, - ops::Deref, - path::PathBuf, - str::FromStr, - sync::{Arc, RwLock}, -}; +use std::{path::PathBuf, str::FromStr, sync::Arc}; use anyhow::Result; use axum::{ @@ -15,85 +9,14 @@ use axum::{ }; use hyper::Body; use serde::{Deserialize, Serialize}; -use tabby_common::{config::RepositoryConfig, SourceFile, Tag}; +use tabby_common::config::RepositoryConfig; use tower::ServiceExt; use tower_http::services::ServeDir; -use tracing::{debug, error, warn}; - -use crate::{ - cron::{CronEvents, StartListener}, - schema::repository::RepositoryService, -}; - -pub struct RepositoryCache { - repository_lookup: RwLock>, - service: Arc, -} - -impl std::fmt::Debug for RepositoryCache { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RepositoryCache") - .field("repository_lookup", &self.repository_lookup) - .finish() - } -} - -impl RepositoryCache { - pub async fn new_initialized( - service: Arc, - events: &CronEvents, - ) -> Arc { - let cache = RepositoryCache { - repository_lookup: Default::default(), - service, - }; - if let Err(e) = cache.reload().await { - error!("Failed to load repositories: {e}"); - }; - let cache = Arc::new(cache); - cache.start_reload_listener(events); - cache - } - - async fn reload(&self) -> Result<()> { - let new_repositories = self - .service - .list_repositories(None, None, None, None) - .await? - .into_iter() - .map(|repository| RepositoryConfig::new_named(repository.name, repository.git_url)) - .collect(); - let mut repository_lookup = self.repository_lookup.write().unwrap(); - debug!("Reloading repositoriy metadata..."); - *repository_lookup = load_meta(new_repositories); - Ok(()) - } - - fn start_reload_listener(self: &Arc, events: &CronEvents) { - let clone = self.clone(); - events.scheduler_job_succeeded.start_listener(move |_| { - let clone = clone.clone(); - async move { - if let Err(e) = clone.reload().await { - warn!("Error when reloading repository cache: {e}"); - }; - } - }); - } - fn repositories(&self) -> impl Deref> + '_ { - self.repository_lookup.read().unwrap() - } -} +use crate::schema::repository::RepositoryService; const DIRECTORY_MIME_TYPE: &str = "application/vnd.directory+json"; -#[derive(Hash, PartialEq, Eq, Debug)] -pub struct RepositoryKey { - repo_name: String, - rel_path: String, -} - #[derive(Deserialize, Debug)] pub struct ResolveParams { name: String, @@ -101,13 +24,6 @@ pub struct ResolveParams { } impl ResolveParams { - pub fn dataset_key(&self) -> RepositoryKey { - RepositoryKey { - repo_name: self.name.clone(), - rel_path: self.os_path(), - } - } - pub fn name_str(&self) -> &str { self.name.as_str() } @@ -143,56 +59,15 @@ struct DirEntry { basename: String, } -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct RepositoryMeta { - git_url: String, - filepath: String, - language: String, - max_line_length: usize, - avg_line_length: f32, - alphanum_fraction: f32, - tags: Vec, -} - -impl From for RepositoryMeta { - fn from(file: SourceFile) -> Self { - Self { - git_url: file.git_url, - filepath: file.filepath, - language: file.language, - max_line_length: file.max_line_length, - avg_line_length: file.avg_line_length, - alphanum_fraction: file.alphanum_fraction, - tags: file.tags, - } - } +pub(super) struct ResolveState { + service: Arc, } -fn load_meta(repositories: Vec) -> HashMap { - let mut dataset = HashMap::new(); - // Construct map of String -> &RepositoryConfig for lookup - let repo_conf = repositories - .iter() - .map(|repo| (repo.git_url.clone(), repo)) - .collect::>(); - let Ok(iter) = SourceFile::all() else { - return dataset; - }; - // Source files contain all metadata, read repository metadata from json - // (SourceFile can be converted into RepositoryMeta) - for file in iter { - if let Some(repo_name) = repo_conf.get(&file.git_url).map(|repo| repo.name()) { - let key = RepositoryKey { - repo_name, - rel_path: file.filepath.clone(), - }; - dataset.insert(key, file.into()); - } +impl ResolveState { + pub fn new(service: Arc) -> Self { + Self { service } } - dataset -} -impl RepositoryCache { /// Resolve a directory pub async fn resolve_dir( &self, @@ -216,10 +91,6 @@ impl RepositoryCache { let kind = if meta.is_dir() { DirEntryKind::Dir } else if meta.is_file() { - let _key = RepositoryKey { - repo_name: repo.name_str().to_string(), - rel_path: basename.clone(), - }; DirEntryKind::File } else { // Skip others. @@ -257,13 +128,6 @@ impl RepositoryCache { Ok(resp.map(boxed)) } - pub fn resolve_meta(&self, key: &RepositoryKey) -> Option { - if let Some(meta) = self.repositories().get(key) { - return Some(meta.clone()); - } - None - } - pub async fn resolve_all(&self) -> Result { let repositories = self .service @@ -286,15 +150,11 @@ impl RepositoryCache { Ok(resp) } - pub fn find_repository(&self, name: &str) -> Option { - let repository_lookup = self.repository_lookup.read().unwrap(); - let key = repository_lookup - .keys() - .find(|repo| repo.repo_name == name)?; - let value = repository_lookup.get(key)?; + pub async fn find_repository(&self, name: &str) -> Option { + let repository = self.service.get_repository_by_name(name).await.ok()?; Some(RepositoryConfig::new_named( - key.repo_name.clone(), - value.git_url.clone(), + name.into(), + repository.git_url.clone(), )) } } diff --git a/ee/tabby-webserver/src/schema/repository.rs b/ee/tabby-webserver/src/schema/repository.rs index 9e223b59618f..3d082c5f6281 100644 --- a/ee/tabby-webserver/src/schema/repository.rs +++ b/ee/tabby-webserver/src/schema/repository.rs @@ -71,6 +71,7 @@ pub trait RepositoryService: Send + Sync { ) -> Result>; async fn create_repository(&self, name: String, git_url: String) -> Result; + async fn get_repository_by_name(&self, name: &str) -> Result; async fn delete_repository(&self, id: &ID) -> Result; async fn update_repository(&self, id: &ID, name: String, git_url: String) -> Result; diff --git a/ee/tabby-webserver/src/service/repository.rs b/ee/tabby-webserver/src/service/repository.rs index 0a432a980d26..dfba3c32742c 100644 --- a/ee/tabby-webserver/src/service/repository.rs +++ b/ee/tabby-webserver/src/service/repository.rs @@ -32,6 +32,10 @@ impl RepositoryService for DbConn { Ok(self.create_repository(name, git_url).await?.as_id()) } + async fn get_repository_by_name(&self, name: &str) -> Result { + Ok(self.get_repository_by_name(name).await?.into()) + } + async fn delete_repository(&self, id: &ID) -> Result { Ok(self.delete_repository(id.as_rowid()?).await?) }