Add a Better Auto Docs Site Generator for Arbitrary claro_binary() Ta…

…rgets (#53)

This v2 of the auto docs generator is much more visually appealing and has more functionality than the previous. Unfortunatley, I still haven't figured out how to enable auto-reloading of the site when the dep graph changes in some way, so I really want to continue investigating that feature as that's what would really make this useful. But for now, this is at least a great way to do some static inspection of large Claro programs.

.bazelignore

@@ -0,0 +1 @@
+tools/clarodocs/node_modules

.bazelrc

@@ -1,2 +1,16 @@
 common --enable_bzlmod
-common --java_runtime_version=remotejdk_11
+common --java_runtime_version=remotejdk_11
+
+################################################################################
+# BEGIN: Flags required for ClaroDocs JS Development.
+################################################################################
+# passes an argument `--skipLibCheck` to *every* spawn of the TypeScript compiler, tsc
+build --@aspect_rules_ts//ts:skipLibCheck=always
+fetch --@aspect_rules_ts//ts:skipLibCheck=always
+query --@aspect_rules_ts//ts:skipLibCheck=always
+
+# Required for rules_js
+build --enable_runfiles
+################################################################################
+# BEGIN: Flags required for ClaroDocs JS Development.
+################################################################################

.gitignore

@@ -25,3 +25,5 @@
 .bazelversion
 # User-specific .bazelrc
 user.bazelrc
+# Created for ClaroDocs JS development.
+**/node_modules

MODULE.bazel

@@ -5,9 +5,17 @@ module(
 )
 
 bazel_dep(name = "aspect_bazel_lib", version = "2.0.1")
-bazel_dep(name = "bazel_skylib", version = "1.4.2")
+bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
 bazel_dep(name = "rules_jvm_external", version = "5.3")
+
+# Deps related to ClaroDocs
+bazel_dep(name = "aspect_rules_lint", version = "0.12.0", dev_dependency = True)
+bazel_dep(name = "aspect_rules_jest", version = "0.19.6", dev_dependency = True)
+bazel_dep(name = "aspect_rules_js", version = "1.38.0", dev_dependency = True)
+bazel_dep(name = "aspect_rules_swc", version = "1.2.2", dev_dependency = True)
+bazel_dep(name = "aspect_rules_ts", version = "2.1.1", dev_dependency = True)
+
 # -- bazel_dep definitions -- #
 
 # TODO(steving) For now, I'm re-exporting the maven module extension from @rules_jvm_external via my own .bzl file
@@ -77,3 +85,35 @@ non_module_deps = use_extension("//:non_module_deps.bzl", "non_module_deps")
 use_repo(non_module_deps, "jflex_rules")
 use_repo(non_module_deps, "bootstrapping_claro_compiler_tarfile")
 # End of extension `non_module_deps`
+
+
+
+############################################################################################################
+# BEGIN: ClaroDocs Deps Setup.
+############################################################################################################
+pnpm = use_extension("@aspect_rules_js//npm:extensions.bzl", "pnpm", dev_dependency = True)
+use_repo(pnpm, "pnpm")
+
+npm = use_extension("@aspect_rules_js//npm:extensions.bzl", "npm", dev_dependency = True)
+npm.npm_translate_lock(
+    name = "npm",
+    npmrc = "//tools/clarodocs:.npmrc",
+    pnpm_lock = "//tools/clarodocs:pnpm-lock.yaml",
+    public_hoist_packages = {
+        "eslint-config-react-app": ["tools/clarodocs"],
+        "[email protected]": ["tools/clarodocs"],
+    },
+    verify_node_modules_ignored = "//:.bazelignore",
+)
+use_repo(npm, "npm")
+
+rules_ts_ext = use_extension(
+    "@aspect_rules_ts//ts:extensions.bzl",
+    "ext",
+    dev_dependency = True,
+)
+rules_ts_ext.deps()
+use_repo(rules_ts_ext, "npm_typescript")
+############################################################################################################
+# END: ClaroDocs Deps Setup.
+############################################################################################################

examples/claro_programs/demo_server/buggy_buggies/BUILD

@@ -1,5 +1,5 @@
 load("//:rules.bzl", "claro_binary")
-load("//src/java/com/claro/module_system/clarodocs:clarodocs_rules.bzl", "clarodocs")
+load("//tools/clarodocs/generator:clarodocs_rules.bzl", "clarodocs")
 
 
 # This is an in-progress refactoring of Claro's first demo server into something that resembles a well-organized
@@ -13,6 +13,8 @@ claro_binary(
         "Utils": "//examples/claro_programs/demo_server/buggy_buggies/utils:utils",
     },
     optional_stdlib_deps = ["http"],
+    # This is just a great demo program for the new ClaroDocs generated site.
+    visibility = ["//tools/clarodocs/generator:__pkg__"],
 )
 
 clarodocs(

src/java/com/claro/claro_build_rules_internal.bzl

@@ -12,6 +12,7 @@ visibility([
     "//src/java/com/claro/stdlib/claro/...",
     "//src/java/com/claro/stdlib/utils/...",
     "//stdlib/...",
+    "//tools/clarodocs/generator/...",
     # TODO(steving) THIS IS A TEMPORARY HACK FOR ME TO COLLECT FEATURES THAT I DISCOVER ARE NECESSARY DURING AoC.
     "//examples/claro_programs/advent_of_code_2023/utils/..."
 ])
@@ -241,7 +242,8 @@ def _invoke_claro_compiler_impl(ctx):
                 files = depset(
                     direct = srcs + [ctx.outputs.compiler_out],
                     transitive = [dep[ClaroModuleInfo].info.files for dep in ctx.attr.deps]
-                )
+                ),
+                direct_deps = ctx.attr.deps,
             ),
             transitive_subgraph_dep_modules = depset(
                 direct = [dep[ClaroModuleInfo].info for dep in ctx.attr.deps],

src/java/com/claro/compiler_backends/java_source/BUILD

@@ -1,12 +1,21 @@
 package(default_visibility = ["//visibility:public"])
 
+java_library(
+    name = "java_source_compiler_backend_cli_options",
+    srcs = ["JavaSourceCompilerBackendCLIOptions.java"],
+    deps = [
+      "//:google-options",
+    ],
+    visibility = ["//tools/clarodocs/generator:__pkg__"],
+)
+
 java_library(
     name = "java_source",
     srcs = [
         "JavaSourceCompilerBackend.java",
-        "JavaSourceCompilerBackendCLIOptions.java",
     ],
     deps = [
+      ":java_source_compiler_backend_cli_options",
       "//:autovalue",
       "//:google-options",
       "//:guava",

src/java/com/claro/module_system/module_serialization/proto/BUILD

@@ -13,6 +13,7 @@ java_proto_library(
         "//src/java/com/claro/module_system/clarodocs:__subpackages__",
         "//src/java/com/claro/module_system/module_serialization/json:__pkg__",
         "//src/java/com/claro/stdlib/utils:__pkg__",
+        "//tools/clarodocs/generator:__pkg__",
     ]
 )
 

tools/clarodocs/.npmrc

@@ -0,0 +1,6 @@
+# Disabling pnpm [hoisting](https://pnpm.io/npmrc#hoist) by setting `hoist=false` is recommended on
+# projects using rules_js so that pnpm outside of Bazel lays out a node_modules tree similar to what
+# rules_js lays out under Bazel (without a hidden node_modules/.pnpm/node_modules). See
+# https://github.com/aspect-build/rules_js/blob/7377f2d0387cc2a9251137929b1c53ccdb3fbcf0/docs/npm_import.md#npm_translate_lock
+# documentation for more information.
+hoist=false

tools/clarodocs/BUILD

@@ -0,0 +1,88 @@
+load("@aspect_rules_js//js:defs.bzl", "js_library", "js_run_binary", "js_run_devserver", "js_test")
+load("@aspect_rules_ts//ts:defs.bzl", "ts_config")
+load("@npm//:defs.bzl", "npm_link_all_packages")
+load("@npm//tools/clarodocs:eslint/package_json.bzl", eslint_bin = "bin")
+load("@npm//tools/clarodocs:tsconfig-to-swcconfig/package_json.bzl", tsconfig_to_swcconfig = "bin")
+load("@npm//tools/clarodocs:vite/package_json.bzl", "bin")
+load("//tools/clarodocs:defs.bzl", "RUNTIME_DEPS")
+
+# Enable clarodocs() usages from arbitrary BUILD files to start a dev server.
+js_library(
+    name = "runtime_deps",
+    srcs = ["index.html", "package.json"],
+    visibility = ["//visibility:public"],
+)
+
+npm_link_all_packages()
+
+eslint_bin.eslint_binary(name = "eslint")
+
+ts_config(
+    name = "tsconfig",
+    src = "tsconfig.json",
+    visibility = ["//visibility:public"],
+)
+
+tsconfig_to_swcconfig.t2s(
+    name = "write_swcrc",
+    srcs = ["tsconfig.json"],
+    args = [
+        "--filename",
+        "$(location tsconfig.json)",
+    ],
+    stdout = ".swcrc",
+    visibility = ["//tools/clarodocs:__subpackages__"],
+)
+
+js_library(
+    name = "vite.config",
+    srcs = ["vite.config.js"],
+    data = [
+        "//tools/clarodocs:node_modules/@vitejs/plugin-react",
+        "//tools/clarodocs:node_modules/vite-plugin-svgr",
+        "//tools/clarodocs:node_modules/vitest",
+    ],
+    visibility = ["//tools/clarodocs:__subpackages__"],
+)
+
+bin.vite_binary(
+    name = "vite",
+    chdir = package_name(),
+    data = ["vite.config"],
+    visibility = ["//visibility:public"],
+)
+
+# Fast developer round-trip under ibazel
+js_run_devserver(
+    name = "start",
+    args = ["."],
+    data = RUNTIME_DEPS,
+    tool = ":vite",
+)
+
+# Create production release artifacts
+js_run_binary(
+    name = "build",
+    srcs = RUNTIME_DEPS,
+    args = ["build"],
+    mnemonic = "ViteBuild",
+    out_dirs = ["dist"],
+    tool = ":vite",
+)
+
+# Hosts the production-bundled application in a web server
+bin.vite_binary(
+    name = "preview",
+    args = ["preview"],
+    chdir = package_name(),
+    data = [":build"],
+)
+
+# Just verify that the bundle produced "something reasonable" but doesn't verify it functions in a browser.
+# TODO: use something like Cypress for a true end-to-end test
+js_test(
+    name = "build_smoke_test",
+    timeout = "short",
+    data = [":build"],
+    entry_point = "build_smoke_test.js",
+)

tools/clarodocs/README.md

@@ -0,0 +1,34 @@
+
+## Available Scripts
+
+### `npm start`
+
+Runs the app in the development mode using Vite as the devserver.
+
+Uses SWC for transpiling typescript.
+
+Thanks to `js_run_devserver`, the page will reload as you make source code edits.
+
+### `npm test`
+
+Launches the test runner in the interactive watch mode.
+
+It runs several test targets, including:
+
+- typechecking with `tsc`
+- linting with `eslint`
+- unit tests with `vitest`
+
+As you save source code changes, the affected tests will be automatically re-run.
+
+### `npm run build`
+
+Builds the app for production to the `dist` folder.
+It correctly bundles React in production mode and optimizes the build for the best performance.
+
+The build is minified and the filenames include the hashes.<br />
+Your app is ready to be deployed!
+
+### `npm run serve`
+
+Serves the production build assets to your browser.

tools/clarodocs/build_smoke_test.js

@@ -0,0 +1,7 @@
+const assert = require('assert');
+const fs = require('fs');
+
+// Make sure there's a file like tools/clarodocs/dist/assets/index.12345678.js
+const files = fs.readdirSync('tools/clarodocs/dist/assets');
+console.log(files);
+assert.ok(files.some((f) => /index\.[0-9a-f]{8}\.js/.test(f)));

tools/clarodocs/defs.bzl

@@ -0,0 +1,39 @@
+"Constants for conventions used for React source files"
+
+load("@aspect_rules_swc//swc:defs.bzl", "swc")
+load("@bazel_skylib//lib:partial.bzl", "partial")
+
+ASSET_PATTERNS = [
+    "*.svg",
+    "*.css",
+]
+
+SRC_PATTERNS = [
+    "*.tsx",
+    "*.ts",
+]
+
+RUNTIME_DEPS = [
+    "//tools/clarodocs:runtime_deps",
+    "//tools/clarodocs:node_modules/react-dom",
+    "//tools/clarodocs:node_modules/react",
+    "//tools/clarodocs:node_modules/web-vitals",
+    "//tools/clarodocs/src:assets",
+    "//tools/clarodocs/src",
+    "//tools/clarodocs/public",
+]
+
+# Filename conventions described at
+# https://create-react-app.dev/docs/running-tests#filename-conventions
+TEST_PATTERNS = [
+    "*.test.tsx",
+    "*.test.ts",
+    "*.spec.tsx",
+    "*.spec.ts",
+]
+
+# Partially-apply our (generated) .swcrc config file to the swc starlark function
+TRANSPILER = partial.make(
+    swc,
+    swcrc = "//tools/clarodocs:.swcrc",
+)

tools/clarodocs/generator/BUILD

@@ -0,0 +1,31 @@
+load("//tools/clarodocs/generator:clarodocs_rules.bzl", "clarodocs")
+
+exports_files(["vite-config.tmpl.js"])
+
+load("@aspect_rules_js//js:defs.bzl", "js_library", "js_run_devserver")
+js_library(
+    name = "test_setup_sh",
+    srcs = ["test_setup.sh"],
+    visibility = ["//visibility:public"],
+)
+
+java_binary(
+    name = "clarodocs_generator",
+    main_class = "com.claro.tools.clarodocs.generator.ClaroDocsGenerator",
+    srcs = [
+        "ClaroDocsCLIOptions.java",
+        "ClaroDocsGenerator.java",
+    ],
+    deps = [
+        "//:google-options",
+        "//:guava",
+        "//src/java/com/claro/compiler_backends/java_source:java_source_compiler_backend_cli_options",
+        "//src/java/com/claro/module_system/module_serialization/proto:serialized_claro_module_java_proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+clarodocs(
+    name = "test",
+    root = "//examples/claro_programs/demo_server/buggy_buggies:buggy_buggies_http_server",
+)