diff --git a/.github/workflows/_extension_distribution.yml b/.github/workflows/_extension_distribution.yml index 5d1a513..39cb113 100644 --- a/.github/workflows/_extension_distribution.yml +++ b/.github/workflows/_extension_distribution.yml @@ -158,12 +158,6 @@ jobs: run: | ./duckdb/scripts/setup_manylinux2014.sh general aws-cli ccache ssh python_alias openssl - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases - continue-on-error: true - with: - key: ${{ github.job }}-${{ matrix.duckdb_arch }} - - name: Setup Ubuntu if: ${{ matrix.duckdb_arch == 'linux_amd64' || matrix.duckdb_arch == 'linux_arm64' }} uses: ./duckdb/.github/actions/ubuntu_18_setup @@ -182,6 +176,12 @@ jobs: cd duckdb git checkout ${{ inputs.duckdb_version }} + - name: Setup Ccache + uses: hendrikmuhs/ccache-action@v1.2.11 # Note: pinned due to GLIBC incompatibility in later releases + continue-on-error: true + with: + key: ${{ github.job }}-${{ matrix.duckdb_arch }} + - name: Setup vcpkg uses: lukka/run-vcpkg@v11.1 with: @@ -282,7 +282,6 @@ jobs: strategy: matrix: ${{fromJson(needs.generate_matrix.outputs.windows_matrix)}} env: - GEN: Ninja VCPKG_TOOLCHAIN_PATH: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake VCPKG_TARGET_TRIPLET: ${{ matrix.vcpkg_triplet }} BUILD_SHELL: ${{ inputs.build_duckdb_shell && '1' || '0' }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9737af1..c1f7117 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,8 +49,17 @@ jobs: repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }} ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }} + doc_test: + needs: + - prepare + - build + uses: ./.github/workflows/generate_docs.yml + with: + extension_name: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_NAME }} + deploy: needs: + - doc_test - prepare - build uses: ./.github/workflows/_extension_deploy.yml @@ -64,3 +73,7 @@ jobs: repository: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_GITHUB }} ref: ${{ needs.prepare.outputs.COMMUNITY_EXTENSION_REF }} + docs_all: + needs: + - deploy + uses: ./.github/workflows/generate_docs.yml diff --git a/.github/workflows/generate_docs.yml b/.github/workflows/generate_docs.yml new file mode 100644 index 0000000..33632a8 --- /dev/null +++ b/.github/workflows/generate_docs.yml @@ -0,0 +1,51 @@ +name: Community Extension Generate Docs +on: + workflow_dispatch: + inputs: + extension_name: + type: string + workflow_call: + inputs: + extension_name: + type: string + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up DuckDB + run: | + wget https://github.com/duckdb/duckdb/releases/download/v1.0.0/duckdb_cli-linux-amd64.zip + unzip duckdb_cli-linux-amd64.zip + chmod +x duckdb + + - name: Fetch extensions + if: ${{ inputs.extension_name == '' }} + run: | + ./scripts/fetch_extensions.sh ./duckdb + + - uses: actions/download-artifact@v3 + if: ${{ inputs.extension_name != '' }} + with: + name: ${{ inputs.extension_name }}-v1.0.0-extension-linux_amd64_gcc4 + path: build/downloaded + + - name: Install downloaded extension + if: ${{ inputs.extension_name != '' }} + env: + NAME: ${{ inputs.extension_name }} + run: | + ./duckdb -c "SET extension_directory = 'build/extension_dir'; FORCE INSTALL 'build/downloaded/$NAME.duckdb_extension';" + ./scripts/fetch_extensions.sh ./duckdb + + - name: Generate docs + run: | + ./scripts/generate_md.sh ./duckdb + + - uses: actions/upload-artifact@v3 + with: + name: generated_markdowns + path: | + build/docs/*.md diff --git a/extensions/h3/docs/function_descriptions.csv b/extensions/h3/docs/function_descriptions.csv new file mode 100644 index 0000000..5697b98 --- /dev/null +++ b/extensions/h3/docs/function_descriptions.csv @@ -0,0 +1,53 @@ +function,description,comment,example +h3_latlng_to_cell,Convert latitude/longitude coordinate to cell ID,"","" +h3_cell_to_lat,Convert cell ID to latitude,"","" +h3_cell_to_lng,Convert cell ID to longitude,"","" +h3_cell_to_latlng,Convert cell ID to latitude/longitude,"","" +h3_cell_to_boundary_wkt,Convert cell ID to cell boundary,"","" +h3_get_resolution,Get resolution number of cell ID,"","" +h3_get_base_cell_number,Get base cell number of cell ID,"","" +h3_string_to_h3,Convert VARCHAR cell ID to UBIGINT,"","" +h3_h3_to_string,Convert BIGINT or UBIGINT cell ID to VARCHAR,"","" +h3_is_valid_cell,True if this is a valid cell ID,"","" +h3_is_res_class_iii,True if the cell's resolution is class III,"","" +h3_is_pentagon,True if the cell is a pentagon,"","" +h3_get_icosahedron_faces,List of icosahedron face IDs the cell is on,"","" +h3_cell_to_parent,Get coarser cell for a cell,"","" +h3_cell_to_children,Get finer cells for a cell,"","" +h3_cell_to_center_child,Get the center finer cell for a cell,"","" +h3_cell_to_child_pos,Get a sub-indexing number for a cell inside a parent,"","" +h3_child_pos_to_cell,Convert parent and sub-indexing number to a cell ID,"","" +h3_compact_cells,Convert a set of single-resolution cells to the minimal mixed-resolution set,"","" +h3_uncompact_cells,Convert a mixed-resolution set to a single-resolution set of cells,"","" +h3_grid_disk,Find cells within a grid distance,"","" +h3_grid_disk_distances,"Find cells within a grid distance, sorted by distance","","" +h3_grid_disk_unsafe,"Find cells within a grid distance, with no pentagon distortion","","" +h3_grid_disk_distances_unsafe,"Find cells within a grid distance, sorted by distance, with no pentagon distortion","","" +h3_grid_ring_unsafe,"Find cells exactly a grid distance away, with no pentagon distortion","","" +h3_grid_path_cells,Find a grid path to connect two cells,"","" +h3_grid_distance,Find the grid distance between two cells,"","" +h3_cell_to_local_ij,"Convert a cell ID to a local I,J coordinate space","","" +h3_local_ij_to_cell,"Convert a local I,J coordinate to a cell ID","","" +h3_cell_to_vertex,Get the vertex ID for a cell ID and vertex number,"","" +h3_cell_to_vertexes,Get all vertex IDs for a cell ID,"","" +h3_vertex_to_lat,Convert a vertex ID to latitude,"","" +h3_vertex_to_lng,Convert a vertex ID to longitude,"","" +h3_vertex_to_latlng,Convert a vertex ID to latitude/longitude coordinate,"","" +h3_is_valid_vertex,True if passed a valid vertex ID,"","" +h3_is_valid_directed_edge,True if passed a valid directed edge ID,"","" +h3_origin_to_directed_edges,Get all directed edge IDs for a cell ID,"","" +h3_directed_edge_to_cells,Convert a directed edge ID to origin/destination cell IDs,"","" +h3_get_directed_edge_origin,Convert a directed edge ID to origin cell ID,"","" +h3_get_directed_edge_destination,Convert a directed edge ID to destination cell ID,"","" +h3_cells_to_directed_edge,Convert an origin/destination pair to directed edge ID,"","" +h3_are_neighbor_cells,True if the two cell IDs are directly adjacent,"","" +h3_directed_edge_to_boundary_wkt,Convert directed edge ID to linestring WKT,"","" +h3_get_hexagon_area_avg,Get average area of a hexagon cell at resolution,"","" +h3_cell_area,Get the area of a cell ID,"","" +h3_edge_length,Get the length of a directed edge ID,"","" +h3_get_num_cells,Get the number of cells at a resolution,"","" +h3_get_res0_cells,Get all resolution 0 cells,"","" +h3_get_pentagons,Get all pentagons at a resolution,"","" +h3_great_circle_distance,Compute the great circle distance between two points (haversine),"","" +h3_cells_to_multi_polygon_wkt,Convert a set of cells to multipolygon WKT,"","" +h3_polygon_wkt_to_cells,Convert polygon WKT to a set of cells,"","" diff --git a/scripts/fetch_extensions.sh b/scripts/fetch_extensions.sh new file mode 100755 index 0000000..0fcbbd9 --- /dev/null +++ b/scripts/fetch_extensions.sh @@ -0,0 +1,9 @@ +set -eo pipefail + +rm -rf build +for extension_folder in extensions/*; +do + extension_name=$(basename -- $extension_folder) + echo "Installing $extension_name" + $1 -c "SET extension_directory = 'build/extension_dir'; FORCE INSTALL '$extension_name' FROM community;" +done diff --git a/scripts/generate_md.sh b/scripts/generate_md.sh new file mode 100755 index 0000000..b3b459d --- /dev/null +++ b/scripts/generate_md.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +# Example of use +# ./scripts/generated_docs_readme.sh build/release/duckdb + +set -eo pipefail + +platform=$($1 -csv -c "PRAGMA platform" | tail -n1) +version_raw=$($1 -csv -c "PRAGMA version" | tail -n1) +version=$(echo "$version_raw-,$version_raw" | cut -d '-' -f 2 | cut -d ',' -f 2) + +DOCS=build/docs + +rm -rf $DOCS +mkdir -p $DOCS + +echo "Extension repository docs" > $DOCS/README.md + +for extension_file in build/extension_dir/$version/$platform/*.duckdb_extension; +do + extension_full=$(basename -- $extension_file) + extension="${extension_full%%.*}" + echo "Generating docs for $extension" + EXTENSION_README=$DOCS/$extension.md + things="extensions functions settings types" + rm -f pre.db + rm -f post.db + for thing in $things; do + $1 -unsigned pre.db -c "SET extension_directory = 'build/extensions'; CREATE OR REPLACE TABLE $thing AS FROM duckdb_$thing();" + rm -rf temp + $1 -unsigned post.db -c "SET extension_directory = 'build/extensions'; FORCE INSTALL $extension FROM 'build/extension_dir'; LOAD $extension; CREATE OR REPLACE TABLE $thing AS FROM duckdb_$thing();" + rm -rf temp + done + + mkdir -p $DOCS/$extension + + $1 post.db -c "ATTACH 'pre.db'; CREATE OR REPLACE TABLE fun_no_overload AS SELECT function_name, function_type, split_part(description, chr(10), 1) as description, comment, example FROM (FROM (SELECT function_name, function_type, description, comment, example FROM functions ORDER BY function_name) EXCEPT (SELECT function_name, function_type, description, comment, example FROM pre.functions ORDER BY function_name)) GROUP BY ALL ORDER BY function_name;" + $1 post.db -c "ATTACH 'pre.db'; CREATE OR REPLACE TABLE fun_with_overload AS SELECT function_name, function_type, split_part(description, chr(10), 1) as description, comment, example FROM (FROM ( SELECT count(*), function_name, function_type, description, comment, example FROM functions GROUP BY ALL ORDER BY function_name) EXCEPT (SELECT count(*), function_name, function_type, description, comment, example FROM pre.functions GROUP BY ALL ORDER BY function_name)) GROUP BY ALL ORDER BY function_name;" + $1 $DOCS/$extension.db -c "ATTACH 'post.db'; CREATE OR REPLACE TABLE functions AS FROM post.fun_no_overload GROUP BY ALL ORDER BY function_name;" + $1 $DOCS/$extension.db -c "ATTACH 'post.db'; CREATE OR REPLACE TABLE functions_overloads AS FROM post.fun_with_overload EXCEPT FROM post.fun_no_overload GROUP BY ALL ORDER BY function_name;" + $1 $DOCS/$extension.db -c "ATTACH 'pre.db'; ATTACH 'post.db'; CREATE OR REPLACE TABLE new_settings AS FROM ( SELECT * EXCLUDE (value) FROM post.settings ORDER BY name) EXCEPT (SELECT * EXCLUDE (value) FROM pre.settings ORDER BY name) ORDER BY name;" + $1 $DOCS/$extension.db -c "ATTACH 'pre.db'; ATTACH 'post.db'; CREATE OR REPLACE TABLE description AS FROM ( SELECT * EXCLUDE (install_path, loaded, installed) FROM post.extensions ORDER BY extension_name) EXCEPT (SELECT * EXCLUDE (install_path, loaded, installed) FROM pre.extensions ORDER BY extension_name) ORDER BY extension_name;" + $1 $DOCS/$extension.db -c "ATTACH 'pre.db'; ATTACH 'post.db'; CREATE OR REPLACE TABLE types AS FROM ( SELECT type_name, type_size, logical_type, type_category, internal FROM post.types ORDER BY ALL) EXCEPT (SELECT type_name, type_size, logical_type, type_category, internal FROM pre.types ORDER BY ALL) ORDER BY type_name;" + + if [ -s "extensions/$extension/docs/function_descriptions.csv" ]; then + cp extensions/$extension/docs/function_descriptions.csv $DOCS/functions.csv + $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions; CREATE TABLE functions AS FROM tmp; DROP TABLE tmp;" + $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions_overloads LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions_overloads; CREATE TABLE functions_overloads AS FROM tmp; DROP TABLE tmp;" + elif [ -s "extension/$extension/docs/function_descriptions.csv" ]; then + cp extension/$extension/docs/function_descriptions.csv $DOCS/functions.csv + $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions; CREATE TABLE functions AS FROM tmp; DROP TABLE tmp;" + $1 $DOCS/$extension.db -c "CREATE TABLE tmp AS SELECT function_name, function_type, other.description as description, other.comment as comment, other.example as example FROM functions_overloads LEFT JOIN read_csv('$DOCS/functions.csv') AS other ON function_name == other.function; DROP TABLE functions_overloads; CREATE TABLE functions_overloads AS FROM tmp; DROP TABLE tmp;" + fi + + $1 $DOCS/$extension.db -markdown -c "FROM new_settings;" > $DOCS/$extension/settings.md + $1 $DOCS/$extension.db -markdown -c "FROM functions;" > $DOCS/$extension/functions.md + $1 $DOCS/$extension.db -markdown -c "FROM functions_overloads;" > $DOCS/$extension/functions_overloads.md + $1 $DOCS/$extension.db -markdown -c "FROM description;" > $DOCS/$extension/extension.md + $1 $DOCS/$extension.db -markdown -c "FROM types;" > $DOCS/$extension/types.md + + rm -f pre.db + rm -f post.db + + if [ -s "$DOCS/$extension/extension.md" ]; then + echo "## $extension" > $EXTENSION_README + echo "" >> $EXTENSION_README + cat $DOCS/$extension/extension.md >> $EXTENSION_README + echo "" >> $EXTENSION_README + + if [ -s "$DOCS/$extension/functions.md" ]; then + echo "### Added functions" >> $EXTENSION_README + echo "" >> $EXTENSION_README + cat $DOCS/$extension/functions.md >> $EXTENSION_README + echo "" >> $EXTENSION_README + fi + if [ -s "$DOCS/$extension/functions_overloads.md" ]; then + echo "### Added function overloads" >> $EXTENSION_README + echo "" >> $EXTENSION_README + cat $DOCS/$extension/functions_overloads.md >> $EXTENSION_README + echo "" >> $EXTENSION_README + fi + if [ -s "$DOCS/$extension/types.md" ]; then + echo "### Added types" >> $EXTENSION_README + echo "" >> $EXTENSION_README + cat $DOCS/$extension/types.md >> $EXTENSION_README + echo "" >> $EXTENSION_README + fi + if [ -s "$DOCS/$extension/settings.md" ]; then + echo "### Added settings" >> $EXTENSION_README + echo "" >> $EXTENSION_README + cat $DOCS/$extension/settings.md >> $EXTENSION_README + echo "" >> $EXTENSION_README + fi + echo "" >> $EXTENSION_README + fi + rm -rf $DOCS/$extension +done + +echo "" >> $DOCS/README.md