Use Helix's tree-sitter grammar registry

Cargo.toml

@@ -6,6 +6,9 @@ edition = "2021"
 authors = ["Jordan Doyle <[email protected]>"]
 license = "WTFPL"
 
+[workspace]
+members = ["tree-sitter-grammar-repository"]
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
@@ -55,32 +58,8 @@ tower-layer = "0.3"
 tower-service = "0.3"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-tree-sitter-bash = "0.23"
-tree-sitter-c = "0.23"
-tree-sitter-cpp = "0.23"
-tree-sitter-c-sharp = "0.23"
-tree-sitter-elixir = "0.3"
-tree-sitter-go = "0.23"
-tree-sitter-php = "0.23"
-tree-sitter-json = "0.23"
-tree-sitter-ocaml = "0.23"
-tree-sitter-python = "0.23"
-tree-sitter-regex = "0.23"
-tree-sitter-ruby = "0.23"
-tree-sitter-css = "0.23"
-tree-sitter-fortran = "0.1"
-tree-sitter-haskell = "0.23"
+tree-sitter-grammar-repository = { path = "./tree-sitter-grammar-repository" }
 tree-sitter-highlight = "0.23"
-tree-sitter-html = "0.23"
-tree-sitter-java = "0.23"
-tree-sitter-javascript = "0.23"
-tree-sitter-md = "0.3"
-tree-sitter-rust = "0.23"
-tree-sitter-scss = "1.0"
-tree-sitter-svelte-ng = "1.0"
-tree-sitter-toml-ng = "0.6.0"
-tree-sitter-typescript = "0.23"
-tree-sitter-yaml = "0.6"
 unix_mode = "0.1"
 uuid = { version = "1.7", features = ["v4"] }
 v_htmlescape = { version = "0.15", features = ["bytes-buf"] }

flake.nix

@@ -8,21 +8,36 @@
       url = "github:rustsec/advisory-db";
       flake = false;
     };
+
+    helix = {
+      url = "github:helix-editor/helix";
+      flake = false;
+    };
   };
 
-  outputs = { self, nixpkgs, utils, crane, advisory-db, treefmt-nix }:
+  outputs = { self, nixpkgs, utils, crane, advisory-db, treefmt-nix, helix }:
     utils.lib.eachDefaultSystem (system:
       let
         pkgs = import nixpkgs { inherit system; };
         craneLib = crane.mkLib pkgs;
         src = craneLib.cleanCargoSource ./.;
+        helix-grammar = pkgs.callPackage "${helix}/grammars.nix" { inherit pkgs; };
+        rgit-grammar = pkgs.runCommand "consolidated-rgit-grammars" { } ''
+          mkdir -p $out
+          for file in ${helix-grammar}/*; do
+            ln -s "$file" "$out/libtree-sitter-$(basename "$file")"
+          done
+          ln -s "${helix}/languages.toml" "$out/languages.toml"
+          ln -s "${helix}/runtime/queries" "$out/queries"
+        '';
         commonArgs = {
           inherit src;
           strictDeps = true;
           buildInputs = pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.libiconv ];
           nativeBuildInputs = with pkgs; [ cmake clang ];
           LIBCLANG_PATH = "${pkgs.clang.cc.lib}/lib";
           ROCKSDB_LIB_DIR = "${pkgs.rocksdb}/lib";
+          TREE_SITTER_GRAMMAR_LIB_DIR = "${rgit-grammar}";
         };
         cargoArtifacts = craneLib.buildDepsOnly commonArgs;
         rgit = craneLib.buildPackage (commonArgs // {
@@ -33,6 +48,7 @@
             fileset = pkgs.lib.fileset.unions [
               ./Cargo.toml
               ./Cargo.lock
+              ./tree-sitter-grammar-repository
               ./src
               ./statics
               ./templates

src/git.rs

@@ -1,3 +1,15 @@
+use std::{
+    borrow::Cow,
+    collections::{BTreeMap, VecDeque},
+    ffi::OsStr,
+    fmt::{self, Arguments, Write},
+    io::ErrorKind,
+    path::{Path, PathBuf},
+    str::FromStr,
+    sync::Arc,
+    time::Duration,
+};
+
 use anyhow::{anyhow, Context, Result};
 use axum::response::IntoResponse;
 use bytes::{buf::Writer, BufMut, Bytes, BytesMut};
@@ -7,26 +19,14 @@ use gix::{
     actor::SignatureRef,
     bstr::{BStr, BString, ByteSlice, ByteVec},
     diff::blob::{platform::prepare_diff::Operation, Sink},
-    object::tree::EntryKind,
-    object::Kind,
+    object::{tree::EntryKind, Kind},
     objs::tree::EntryRef,
     prelude::TreeEntryRefExt,
     traverse::tree::visit::Action,
     url::Scheme,
     ObjectId, ThreadSafeRepository, Url,
 };
 use moka::future::Cache;
-use std::{
-    borrow::Cow,
-    collections::{BTreeMap, VecDeque},
-    ffi::OsStr,
-    fmt::{self, Arguments, Write},
-    io::ErrorKind,
-    path::{Path, PathBuf},
-    str::FromStr,
-    sync::Arc,
-    time::Duration,
-};
 use tar::Builder;
 use time::{OffsetDateTime, UtcOffset};
 use tracing::{error, instrument, warn};
@@ -144,21 +144,15 @@ impl OpenRepository {
 
                 match object.kind {
                     Kind::Blob => {
-                        let path = path.join(item.filename().to_path_lossy());
                         let mut blob = object.into_blob();
 
                         let size = blob.data.len();
-                        let extension = path
-                            .extension()
-                            .or_else(|| path.file_name())
-                            .and_then(OsStr::to_str)
-                            .unwrap_or_default();
 
                         let content = match (formatted, simdutf8::basic::from_utf8(&blob.data)) {
                             (true, Err(_)) => Content::Binary(vec![]),
                             (true, Ok(data)) => Content::Text(Cow::Owned(format_file(
                                 data,
-                                FileIdentifier::Extension(extension),
+                                FileIdentifier::Path(path.as_path()),
                             )?)),
                             (false, Err(_)) => Content::Binary(blob.take_data()),
                             (false, Ok(_data)) => Content::Text(Cow::Owned(unsafe {
@@ -1091,29 +1085,17 @@ impl Callback for PlainDiffFormatter {
 }
 
 struct SyntaxHighlightedDiffFormatter<'a> {
-    extension: &'a str,
+    path: &'a Path,
 }
 
 impl<'a> SyntaxHighlightedDiffFormatter<'a> {
     fn new(path: &'a Path) -> Self {
-        let extension = path
-            .extension()
-            .or_else(|| path.file_name())
-            .and_then(OsStr::to_str)
-            .unwrap_or_default();
-
-        Self { extension }
+        Self { path }
     }
 
     fn write(&self, output: &mut String, class: &str, data: &str) {
         write!(output, r#"<span class="diff-{class}">"#).unwrap();
-        format_file_inner(
-            output,
-            data,
-            FileIdentifier::Extension(self.extension),
-            false,
-        )
-        .unwrap();
+        format_file_inner(output, data, FileIdentifier::Path(self.path), false).unwrap();
         write!(output, r#"</span>"#).unwrap();
     }
 }

src/main.rs

@@ -42,6 +42,7 @@ use crate::{
     },
     git::Git,
     layers::logger::LoggingMiddleware,
+    syntax_highlight::prime_highlighters,
     theme::Theme,
 };
 
@@ -190,6 +191,10 @@ async fn main() -> Result<(), anyhow::Error> {
         }
     };
 
+    info!("Priming highlighters...");
+    prime_highlighters();
+    info!("Server starting up...");
+
     let app = Router::new()
         .route("/", get(methods::index::handle))
         .route(