Merge pull request #16 from carlopi/docs

Add doc generation scripts

.github/workflows/_extension_distribution.yml

@@ -158,12 +158,6 @@ jobs:
         run: |
           ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl
 
-      - name: Setup Ccache
-        uses: hendrikmuhs/[email protected] # Note: pinned due to GLIBC incompatibility in later releases
-        continue-on-error: true
-        with:
-          key: ${{ github.job }}-${{ matrix.duckdb_arch }}
-
       - name: Setup Ubuntu
         if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }}
         uses: ./duckdb/.github/actions/ubuntu_18_setup
@@ -182,6 +176,12 @@ jobs:
           cd duckdb
           git checkout ${{ inputs.duckdb_version }}
           
+      - name: Setup Ccache
+        uses: hendrikmuhs/[email protected] # Note: pinned due to GLIBC incompatibility in later releases
+        continue-on-error: true
+        with:
+          key: ${{ github.job }}-${{ matrix.duckdb_arch }}
+
       - name: Setup vcpkg
         uses: lukka/[email protected]
         with:
@@ -282,7 +282,6 @@ jobs:
     strategy:
       matrix: ${{fromJson(needs.generate_matrix.outputs.windows_matrix)}}
     env:
-      GEN: Ninja
       VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
       VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }}
       BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }}

.github/workflows/build.yml

@@ -49,8 +49,17 @@ jobs:
       repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }}
       ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }}
 
+  doc_test:
+    needs:
+      - prepare
+      - build
+    uses: ./.github/workflows/generate_docs.yml
+    with:
+      extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }}
+
   deploy:
     needs: 
+      - doc_test
       - prepare
       - build
     uses: ./.github/workflows/_extension_deploy.yml
@@ -64,3 +73,7 @@ jobs:
       repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }}
       ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }}
 
+  docs_all:
+    needs:
+     - deploy
+    uses: ./.github/workflows/generate_docs.yml

.github/workflows/generate_docs.yml

@@ -0,0 +1,51 @@
+name: Community Extension Generate Docs
+on:
+  workflow_dispatch:
+    inputs:
+      extension_name:
+        type: string
+  workflow_call:
+    inputs:
+      extension_name:
+        type: string
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up DuckDB
+      run: |
+        wget https://github.com/duckdb/duckdb/releases/download/v1.0.0/duckdb_cli-linux-amd64.zip
+        unzip duckdb_cli-linux-amd64.zip
+        chmod +x duckdb
+
+    - name: Fetch extensions
+      if: ${{ inputs.extension_name == '' }}
+      run: |
+        ./scripts/fetch_extensions.sh ./duckdb
+
+    - uses: actions/download-artifact@v3
+      if: ${{ inputs.extension_name != '' }}
+      with:
+        name: ${{ inputs.extension_name }}-v1.0.0-extension-linux_amd64_gcc4
+        path: build/downloaded
+
+    - name: Install downloaded extension
+      if: ${{ inputs.extension_name != '' }}
+      env:
+        NAME: ${{ inputs.extension_name }} 
+      run: |
+        ./duckdb -c "SET extension_directory = 'build/extension_dir'; FORCE INSTALL 'build/downloaded/$NAME.duckdb_extension';"
+        ./scripts/fetch_extensions.sh ./duckdb
+
+    - name: Generate docs
+      run: |
+        ./scripts/generate_md.sh ./duckdb
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: generated_markdowns
+        path: |
+          build/docs/*.md

extensions/h3/docs/function_descriptions.csv

@@ -0,0 +1,53 @@
+function,description,comment,example
+h3_latlng_to_cell,Convert latitude/longitude coordinate to cell ID,"",""
+h3_cell_to_lat,Convert cell ID to latitude,"",""
+h3_cell_to_lng,Convert cell ID to longitude,"",""
+h3_cell_to_latlng,Convert cell ID to latitude/longitude,"",""
+h3_cell_to_boundary_wkt,Convert cell ID to cell boundary,"",""
+h3_get_resolution,Get resolution number of cell ID,"",""
+h3_get_base_cell_number,Get base cell number of cell ID,"",""
+h3_string_to_h3,Convert VARCHAR cell ID to UBIGINT,"",""
+h3_h3_to_string,Convert BIGINT or UBIGINT cell ID to VARCHAR,"",""
+h3_is_valid_cell,True if this is a valid cell ID,"",""
+h3_is_res_class_iii,True if the cell's resolution is class III,"",""
+h3_is_pentagon,True if the cell is a pentagon,"",""
+h3_get_icosahedron_faces,List of icosahedron face IDs the cell is on,"",""
+h3_cell_to_parent,Get coarser cell for a cell,"",""
+h3_cell_to_children,Get finer cells for a cell,"",""
+h3_cell_to_center_child,Get the center finer cell for a cell,"",""
+h3_cell_to_child_pos,Get a sub-indexing number for a cell inside a parent,"",""
+h3_child_pos_to_cell,Convert parent and sub-indexing number to a cell ID,"",""
+h3_compact_cells,Convert a set of single-resolution cells to the minimal mixed-resolution set,"",""
+h3_uncompact_cells,Convert a mixed-resolution set to a single-resolution set of cells,"",""
+h3_grid_disk,Find cells within a grid distance,"",""
+h3_grid_disk_distances,"Find cells within a grid distance, sorted by distance","",""
+h3_grid_disk_unsafe,"Find cells within a grid distance, with no pentagon distortion","",""
+h3_grid_disk_distances_unsafe,"Find cells within a grid distance, sorted by distance, with no pentagon distortion","",""
+h3_grid_ring_unsafe,"Find cells exactly a grid distance away, with no pentagon distortion","",""
+h3_grid_path_cells,Find a grid path to connect two cells,"",""
+h3_grid_distance,Find the grid distance between two cells,"",""
+h3_cell_to_local_ij,"Convert a cell ID to a local I,J coordinate space","",""
+h3_local_ij_to_cell,"Convert a local I,J coordinate to a cell ID","",""
+h3_cell_to_vertex,Get the vertex ID for a cell ID and vertex number,"",""
+h3_cell_to_vertexes,Get all vertex IDs for a cell ID,"",""
+h3_vertex_to_lat,Convert a vertex ID to latitude,"",""
+h3_vertex_to_lng,Convert a vertex ID to longitude,"",""
+h3_vertex_to_latlng,Convert a vertex ID to latitude/longitude coordinate,"",""
+h3_is_valid_vertex,True if passed a valid vertex ID,"",""
+h3_is_valid_directed_edge,True if passed a valid directed edge ID,"",""
+h3_origin_to_directed_edges,Get all directed edge IDs for a cell ID,"",""
+h3_directed_edge_to_cells,Convert a directed edge ID to origin/destination cell IDs,"",""
+h3_get_directed_edge_origin,Convert a directed edge ID to origin cell ID,"",""
+h3_get_directed_edge_destination,Convert a directed edge ID to destination cell ID,"",""
+h3_cells_to_directed_edge,Convert an origin/destination pair to directed edge ID,"",""
+h3_are_neighbor_cells,True if the two cell IDs are directly adjacent,"",""
+h3_directed_edge_to_boundary_wkt,Convert directed edge ID to linestring WKT,"",""
+h3_get_hexagon_area_avg,Get average area of a hexagon cell at resolution,"",""
+h3_cell_area,Get the area of a cell ID,"",""
+h3_edge_length,Get the length of a directed edge ID,"",""
+h3_get_num_cells,Get the number of cells at a resolution,"",""
+h3_get_res0_cells,Get all resolution 0 cells,"",""
+h3_get_pentagons,Get all pentagons at a resolution,"",""
+h3_great_circle_distance,Compute the great circle distance between two points (haversine),"",""
+h3_cells_to_multi_polygon_wkt,Convert a set of cells to multipolygon WKT,"",""
+h3_polygon_wkt_to_cells,Convert polygon WKT to a set of cells,"",""

scripts/fetch_extensions.sh

@@ -0,0 +1,9 @@
+set -eo pipefail
+
+rm -rf build
+for extension_folder in extensions/*;
+do
+    extension_name=$(basename -- $extension_folder)
+    echo "Installing $extension_name"
+    $1 -c "SET extension_directory = 'build/extension_dir'; FORCE INSTALL '$extension_name' FROM community;"
+done

scripts/generate_md.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Example of use
+# ./scripts/generated_docs_readme.sh build/release/duckdb
+
+set -eo pipefail
+
+platform=$($1 -csv -c "PRAGMA platform" | tail -n1)
+version_raw=$($1 -csv -c "PRAGMA version" | tail -n1)
+version=$(echo "$version_raw-,$version_raw" | cut -d '-' -f 2 | cut -d ',' -f 2)
+
+DOCS=build/docs
+
+rm -rf $DOCS
+mkdir -p $DOCS
+
+echo "Extension repository docs" > $DOCS/README.md
+
+for extension_file in build/extension_dir/$version/$platform/*.duckdb_extension;
+do
+    extension_full=$(basename -- $extension_file)
+    extension="${extension_full%%.*}"
+    echo "Generating docs for $extension"
+    EXTENSION_README=$DOCS/$extension.md
+    things="extensions functions settings types"
+    rm -f pre.db
+    rm -f post.db
+    for thing in $things; do
+        $1 -unsigned pre.db -c "SET extension_directory = 'build/extensions'; CREATE OR REPLACE TABLE $thing AS FROM duckdb_$thing();"
+        rm -rf temp
+        $1 -unsigned post.db -c "SET extension_directory = 'build/extensions'; FORCE INSTALL $extension FROM 'build/extension_dir'; LOAD $extension; CREATE OR REPLACE TABLE $thing AS FROM duckdb_$thing();"
+        rm -rf temp
+    done
+
+    mkdir -p $DOCS/$extension
+
+    $1 post.db -c "ATTACH 'pre.db'; CREATE OR REPLACE TABLE fun_no_overload AS SELECT function_name, function_type, split_part(description, chr(10), 1) as description, comment, example FROM (FROM (SELECT function_name, function_type, description, comment, example FROM functions ORDER BY function_name) EXCEPT (SELECT function_name, function_type, description, comment, example FROM pre.functions ORDER BY function_name)) GROUP BY ALL ORDER BY function_name;"
+    $1 post.db -c "ATTACH 'pre.db'; CREATE OR REPLACE TABLE fun_with_overload AS SELECT function_name, function_type, split_part(description, chr(10), 1) as description, comment, example FROM (FROM ( SELECT count(*), function_name, function_type, description, comment, example FROM functions GROUP BY ALL ORDER BY function_name) EXCEPT (SELECT count(*), function_name, function_type, description, comment, example FROM pre.functions GROUP BY ALL ORDER BY function_name)) GROUP BY ALL ORDER BY function_name;"
+    $1 $DOCS/$extension.db -c "ATTACH 'post.db'; CREATE OR REPLACE TABLE functions AS FROM post.fun_no_overload GROUP BY ALL ORDER BY function_name;"
+    $1 $DOCS/$extension.db -c "ATTACH 'post.db'; CREATE OR REPLACE TABLE functions_overloads AS FROM post.fun_with_overload EXCEPT FROM post.fun_no_overload GROUP BY ALL ORDER BY function_name;"
+    $1 $DOCS/$extension.db -c "ATTACH 'pre.db'; ATTACH 'post.db'; CREATE OR REPLACE TABLE new_settings AS FROM ( SELECT * EXCLUDE (value) FROM post.settings ORDER BY name) EXCEPT (SELECT * EXCLUDE (value) FROM pre.settings ORDER BY name) ORDER BY name;"
+    $1 $DOCS/$extension.db -c "ATTACH 'pre.db'; ATTACH 'post.db'; CREATE OR REPLACE TABLE description AS FROM ( SELECT * EXCLUDE (install_path, loaded, installed) FROM post.extensions ORDER BY extension_name) EXCEPT (SELECT * EXCLUDE (install_path, loaded, installed) FROM pre.extensions ORDER BY extension_name) ORDER BY extension_name;"
+    $1 $DOCS/$extension.db -c "ATTACH 'pre.db'; ATTACH 'post.db'; CREATE OR REPLACE TABLE types AS FROM ( SELECT type_name, type_size, logical_type, type_category, internal FROM post.types ORDER BY ALL) EXCEPT (SELECT type_name, type_size, logical_type, type_category, internal FROM pre.types ORDER BY ALL) ORDER BY type_name;"
+
+    if [ -s "extensions/$extension/docs/function_descriptions.csv" ]; then
+       cp extensions/$extension/docs/function_descriptions.csv $DOCS/functions.csv
+       $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions; CREATE TABLE functions AS FROM tmp; DROP TABLE tmp;"
+       $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions_overloads LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions_overloads; CREATE TABLE functions_overloads AS FROM tmp; DROP TABLE tmp;"
+    elif [ -s "extension/$extension/docs/function_descriptions.csv" ]; then
+       cp extension/$extension/docs/function_descriptions.csv $DOCS/functions.csv
+       $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions; CREATE TABLE functions AS FROM tmp; DROP TABLE tmp;"
+       $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions_overloads LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions_overloads; CREATE TABLE functions_overloads AS FROM tmp; DROP TABLE tmp;"
+    fi
+
+    $1 $DOCS/$extension.db -markdown -c "FROM new_settings;" > $DOCS/$extension/settings.md
+    $1 $DOCS/$extension.db -markdown -c "FROM functions;" > $DOCS/$extension/functions.md
+    $1 $DOCS/$extension.db -markdown -c "FROM functions_overloads;" > $DOCS/$extension/functions_overloads.md
+    $1 $DOCS/$extension.db -markdown -c "FROM description;" > $DOCS/$extension/extension.md
+    $1 $DOCS/$extension.db -markdown -c "FROM types;" > $DOCS/$extension/types.md
+
+    rm -f pre.db
+    rm -f post.db
+
+    if [ -s "$DOCS/$extension/extension.md" ]; then
+       echo "## $extension" > $EXTENSION_README
+       echo "" >> $EXTENSION_README
+       cat $DOCS/$extension/extension.md >> $EXTENSION_README
+       echo "" >> $EXTENSION_README
+
+       if [ -s "$DOCS/$extension/functions.md" ]; then
+          echo "### Added functions" >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+          cat $DOCS/$extension/functions.md >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+       fi
+       if [ -s "$DOCS/$extension/functions_overloads.md" ]; then
+          echo "### Added function overloads" >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+          cat $DOCS/$extension/functions_overloads.md >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+       fi
+       if [ -s "$DOCS/$extension/types.md" ]; then
+          echo "### Added types" >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+          cat $DOCS/$extension/types.md >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+       fi
+       if [ -s "$DOCS/$extension/settings.md" ]; then
+          echo "### Added settings" >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+          cat $DOCS/$extension/settings.md >> $EXTENSION_README
+          echo "" >> $EXTENSION_README
+       fi
+       echo "" >> $EXTENSION_README
+    fi
+    rm -rf $DOCS/$extension
+done
+
+echo "" >> $DOCS/README.md