From c84d8683368e74f283fc62a34df536f75051c5fa Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 10:37:48 +0100 Subject: [PATCH 01/31] Add yaml file for code quality checker --- .github/workflows/_extension_code_quality.yml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/_extension_code_quality.yml diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml new file mode 100644 index 0000000..37ea10f --- /dev/null +++ b/.github/workflows/_extension_code_quality.yml @@ -0,0 +1,61 @@ +name: CodeQuality +on: + workflow_dispatch: + inputs: + explicit_checks: + type: string + repository_dispatch: + push: + branches: + - '**' + - '!main' + - '!feature' + paths-ignore: + - '**.md' + - '.github/workflows/**' + - '!.extension-ci-tools/workflows/**' + + pull_request: + types: [opened, reopened, ready_for_review] + paths-ignore: + - '**.md' + - '.github/workflows/**' + - '!.extension-ci-tools/workflows/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} + cancel-in-progress: true + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + +jobs: + format-check: + name: Format Check + runs-on: ubuntu-20.04 + + env: + CC: gcc-10 + CXX: g++-10 + GEN: ninja + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install + shell: bash + run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black cxxheaderparser pcpp + + - name: List Installed Packages + shell: bash + run: pip3 freeze + + - name: Format Check + shell: bash + run: | + clang-format --version + clang-format --dump-config + black --version + make format \ No newline at end of file From b6b2bf1baf2c40fcf930dea99efb894424205a90 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:15:54 +0100 Subject: [PATCH 02/31] Update workflow file --- .github/workflows/_extension_code_quality.yml | 28 ++----------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 37ea10f..4af22c4 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -1,33 +1,11 @@ name: CodeQuality + on: - workflow_dispatch: + workflow_call: inputs: explicit_checks: + required: false type: string - repository_dispatch: - push: - branches: - - '**' - - '!main' - - '!feature' - paths-ignore: - - '**.md' - - '.github/workflows/**' - - '!.extension-ci-tools/workflows/**' - - pull_request: - types: [opened, reopened, ready_for_review] - paths-ignore: - - '**.md' - - '.github/workflows/**' - - '!.extension-ci-tools/workflows/**' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }} - cancel-in-progress: true - -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} jobs: format-check: From 26d6e40941638763d75d9ed855c50a4f3f5d3db5 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:22:17 +0100 Subject: [PATCH 03/31] Go up one dir --- .github/workflows/_extension_code_quality.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 4af22c4..56a9e74 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -36,4 +36,5 @@ jobs: clang-format --version clang-format --dump-config black --version + cd .. make format \ No newline at end of file From f0ef9af02ddf30340254d77a21e57422a392b8ab Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:25:28 +0100 Subject: [PATCH 04/31] Check cur directory --- .github/workflows/_extension_code_quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 56a9e74..7605efb 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -36,5 +36,5 @@ jobs: clang-format --version clang-format --dump-config black --version - cd .. + pwd make format \ No newline at end of file From 2fdbe0c5c09a2e04f4ee80f3e1c89023100b8460 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:31:50 +0100 Subject: [PATCH 05/31] Add ls --- .github/workflows/_extension_code_quality.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 7605efb..7ce0ad7 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -37,4 +37,5 @@ jobs: clang-format --dump-config black --version pwd + ls make format \ No newline at end of file From 67c549c3c5adb9d34be794fa2f81a0a9f47e144f Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:36:53 +0100 Subject: [PATCH 06/31] Add ls to extension-ci-tools --- .github/workflows/_extension_code_quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 7ce0ad7..7a981f4 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -37,5 +37,5 @@ jobs: clang-format --dump-config black --version pwd - ls + ls extension-ci-tools make format \ No newline at end of file From 3c351b6001fef8602b591831da9f368cbcc0db1f Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:41:06 +0100 Subject: [PATCH 07/31] Fetch extension-ci-tools --- .github/workflows/_extension_code_quality.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 7a981f4..feb55d1 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -18,24 +18,35 @@ jobs: GEN: ninja steps: + # Step 1: Checkout repository with submodules - uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: recursive # Ensure submodules are initialized and fetched + # Step 2: Install necessary packages - name: Install shell: bash run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black cxxheaderparser pcpp + # Step 3: List installed packages - name: List Installed Packages shell: bash run: pip3 freeze + # Step 4: Debugging: List submodule content + - name: Debug Submodule Content + shell: bash + run: | + pwd + ls -la + ls -la extension-ci-tools + + # Step 5: Run Format Check - name: Format Check shell: bash run: | clang-format --version clang-format --dump-config black --version - pwd - ls extension-ci-tools make format \ No newline at end of file From e533e108e115d3c84308420459cd7c4d1d0324fa Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 11:42:30 +0100 Subject: [PATCH 08/31] Remove debug step --- .github/workflows/_extension_code_quality.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index feb55d1..77691ed 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -34,15 +34,7 @@ jobs: shell: bash run: pip3 freeze - # Step 4: Debugging: List submodule content - - name: Debug Submodule Content - shell: bash - run: | - pwd - ls -la - ls -la extension-ci-tools - - # Step 5: Run Format Check + # Step 4: Run Format Check - name: Format Check shell: bash run: | From b026daaa0272f91d4d130a6ede81d74c793b3c19 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 14:34:12 +0100 Subject: [PATCH 09/31] Add a make format-check rule --- .github/workflows/_extension_code_quality.yml | 2 +- makefiles/duckdb_extension.Makefile | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 77691ed..fbe6ea5 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -41,4 +41,4 @@ jobs: clang-format --version clang-format --dump-config black --version - make format \ No newline at end of file + make format-check \ No newline at end of file diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 47f32b4..5e3c5ad 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -132,10 +132,18 @@ wasm_threads: emmake make -j8 -Cbuild/wasm_threads #### Misc +# Rule to fix formatting format: + @echo "Fixing formatting issues..." find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i cmake-format -i CMakeLists.txt +# Rule to check formatting without modifying files +format-check: + @echo "Checking formatting..." + @find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -output-replacements-xml | grep -q " Date: Mon, 9 Dec 2024 14:38:28 +0100 Subject: [PATCH 10/31] Update workflow --- .github/workflows/_extension_code_quality.yml | 3 --- makefiles/duckdb_extension.Makefile | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index fbe6ea5..4889e1d 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -38,7 +38,4 @@ jobs: - name: Format Check shell: bash run: | - clang-format --version - clang-format --dump-config - black --version make format-check \ No newline at end of file diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 5e3c5ad..152e535 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -138,12 +138,12 @@ format: find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i cmake-format -i CMakeLists.txt -# Rule to check formatting without modifying files format-check: @echo "Checking formatting..." @find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -output-replacements-xml | grep -q " Date: Mon, 9 Dec 2024 15:28:34 +0100 Subject: [PATCH 11/31] Fix format-check not exiting properly whenever an incorrectly formatted file was found --- makefiles/duckdb_extension.Makefile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 152e535..2b684a8 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -140,9 +140,19 @@ format: format-check: @echo "Checking formatting..." - @find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -output-replacements-xml | grep -q " Date: Mon, 9 Dec 2024 15:42:40 +0100 Subject: [PATCH 12/31] Simplify format makerule --- makefiles/duckdb_extension.Makefile | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 2b684a8..a64d798 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -134,25 +134,7 @@ wasm_threads: #### Misc # Rule to fix formatting format: - @echo "Fixing formatting issues..." - find src/ -iname *.hpp -o -iname *.cpp | xargs clang-format --sort-includes=0 -style=file -i - cmake-format -i CMakeLists.txt - -format-check: - @echo "Checking formatting..." - @if find src/ -iname "*.hpp" -o -iname "*.cpp" | xargs clang-format --sort-includes=0 -style=file -output-replacements-xml | grep -q " Date: Mon, 9 Dec 2024 15:46:30 +0100 Subject: [PATCH 13/31] Add format-fix rule --- makefiles/duckdb_extension.Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index a64d798..ff6236f 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -136,6 +136,9 @@ wasm_threads: format: python scripts/format.py +format-fix: + python3 scripts/format.py --all --fix --noconfirm + update: git submodule update --remote --merge From 3c09b5eba9ecbf076a309f1fe9a39c79f732c9a1 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 15:48:22 +0100 Subject: [PATCH 14/31] Format fix --- makefiles/duckdb_extension.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index ff6236f..c0014c5 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -134,10 +134,10 @@ wasm_threads: #### Misc # Rule to fix formatting format: - python scripts/format.py + python scripts/format_extension.py format-fix: - python3 scripts/format.py --all --fix --noconfirm + python3 scripts/format_extension.py --all --fix --noconfirm update: git submodule update --remote --merge From 43e24b2b86bfcbd7a0f2952a9c21a2bc9c49c36b Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 15:50:49 +0100 Subject: [PATCH 15/31] Should say format-fix --- .github/workflows/_extension_code_quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 4889e1d..ed74c17 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -38,4 +38,4 @@ jobs: - name: Format Check shell: bash run: | - make format-check \ No newline at end of file + make format-fix \ No newline at end of file From f4e3936b6512f7f118a996961b333799391a13b0 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 15:56:19 +0100 Subject: [PATCH 16/31] Commit with formatting mistake --- .github/workflows/_extension_code_quality.yml | 2 +- makefiles/duckdb_extension.Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index ed74c17..4889e1d 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -38,4 +38,4 @@ jobs: - name: Format Check shell: bash run: | - make format-fix \ No newline at end of file + make format-check \ No newline at end of file diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index c0014c5..6a2b1fe 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -133,7 +133,7 @@ wasm_threads: #### Misc # Rule to fix formatting -format: +format-check: python scripts/format_extension.py format-fix: From e5b9d6999c3d9b25281fab531f66251d56545a41 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 16:34:56 +0100 Subject: [PATCH 17/31] Updated make format with some incorrectly formatted code --- .github/workflows/_extension_code_quality.yml | 15 +++++++++------ makefiles/duckdb_extension.Makefile | 3 +++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 4889e1d..4ff4e16 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -18,24 +18,27 @@ jobs: GEN: ninja steps: - # Step 1: Checkout repository with submodules - uses: actions/checkout@v4 with: fetch-depth: 0 - submodules: recursive # Ensure submodules are initialized and fetched - # Step 2: Install necessary packages - name: Install shell: bash run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build clang-format-11 && sudo pip3 install cmake-format black cxxheaderparser pcpp - # Step 3: List installed packages - name: List Installed Packages shell: bash run: pip3 freeze - # Step 4: Run Format Check - name: Format Check shell: bash run: | - make format-check \ No newline at end of file + clang-format --version + clang-format --dump-config + black --version + make format-check-silent + + - name: Generated Check + shell: bash + run: | + git diff --ignore-submodules --exit-code \ No newline at end of file diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index 6a2b1fe..d11bce1 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -136,6 +136,9 @@ wasm_threads: format-check: python scripts/format_extension.py +format-check-silent: + python3 scripts/format_extension.py --all --check --silent + format-fix: python3 scripts/format_extension.py --all --fix --noconfirm From a254f05a8b83fb36df2ef2c3e0a66c2d5e9f3198 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Mon, 9 Dec 2024 16:39:37 +0100 Subject: [PATCH 18/31] Fetch submodules --- .github/workflows/_extension_code_quality.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 4ff4e16..63238bd 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -21,6 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + submodules: recursive - name: Install shell: bash From 3d021a0c30771a94ff511aa08c54496c1d646f26 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 09:43:30 +0100 Subject: [PATCH 19/31] Move format_extension.py to extension-ci-tools --- scripts/format_extension.py | 394 ++++++++++++++++++++++++++++++++++++ 1 file changed, 394 insertions(+) create mode 100644 scripts/format_extension.py diff --git a/scripts/format_extension.py b/scripts/format_extension.py new file mode 100644 index 0000000..0ed9ada --- /dev/null +++ b/scripts/format_extension.py @@ -0,0 +1,394 @@ +#!/usr/bin/python + +# this script is used to format the source directory + +import os +import time +import sys +import inspect +import subprocess +import difflib +import re +import concurrent.futures +from python_helpers import open_utf8 + +try: + ver = subprocess.check_output(('black', '--version'), text=True) + if int(ver.split(' ')[1].split('.')[0]) < 24: + print('you need to run `pip install "black>=24"`', ver) + exit(-1) +except Exception as e: + print('you need to run `pip install "black>=24"`', e) + exit(-1) + +try: + ver = subprocess.check_output(('clang-format', '--version'), text=True) + if '11.' not in ver: + print('you need to run `pip install clang_format==11.0.1 - `', ver) + exit(-1) +except Exception as e: + print('you need to run `pip install clang_format==11.0.1 - `', e) + exit(-1) + +cpp_format_command = 'clang-format --sort-includes=0 -style=file' +cmake_format_command = 'cmake-format' + +try: + subprocess.check_output(('cmake-format', '--version'), text=True) +except Exception as e: + print('you need to run `pip install cmake-format`', e) + exit(-1) + +extensions = [ + '.cpp', + '.ipp', + '.c', + '.hpp', + '.h', + '.cc', + '.hh', + 'CMakeLists.txt', + '.test', + '.test_slow', + '.test_coverage', + '.benchmark', + '.py', + '.java', +] +formatted_directories = ['src', 'test', 'scripts'] +ignored_files = [] +ignored_directories = ['__pycache__', '.pytest_cache'] +format_all = False +check_only = True +confirm = True +silent = False +force = False + + +def print_usage(): + print("Usage: python scripts/format.py [revision|--all] [--check|--fix] [--force]") + print( + " [revision] is an optional revision number, all files that changed since that revision will be formatted (default=HEAD)" + ) + print(" if [revision] is set to --all, all files will be formatted") + print(" --check only prints differences, --fix also fixes the files (--check is default)") + exit(1) + + +if len(sys.argv) == 1: + revision = "HEAD" +elif len(sys.argv) >= 2: + revision = sys.argv[1] +else: + print_usage() + +if len(sys.argv) > 2: + for arg in sys.argv[2:]: + if arg == '--check': + check_only = True + elif arg == '--fix': + check_only = False + elif arg == '--noconfirm': + confirm = False + elif arg == '--confirm': + confirm = True + elif arg == '--silent': + silent = True + elif arg == '--force': + force = True + else: + print_usage() + +if revision == '--all': + format_all = True + + +def file_is_ignored(full_path): + if os.path.basename(full_path) in ignored_files: + return True + dirnames = os.path.sep.join(full_path.split(os.path.sep)[:-1]) + for ignored_directory in ignored_directories: + if ignored_directory in dirnames: + return True + return False + + +def can_format_file(full_path): + global extensions, formatted_directories, ignored_files + if not os.path.isfile(full_path): + return False + fname = full_path.split(os.path.sep)[-1] + found = False + # check file extension + for ext in extensions: + if full_path.endswith(ext): + found = True + break + if not found: + return False + # check ignored files + if file_is_ignored(full_path): + return False + # now check file directory + for dname in formatted_directories: + if full_path.startswith(dname): + return True + return False + + +action = "Formatting" +if check_only: + action = "Checking" + + +def get_changed_files(revision): + proc = subprocess.Popen(['git', 'diff', '--name-only', revision], stdout=subprocess.PIPE) + files = proc.stdout.read().decode('utf8').split('\n') + changed_files = [] + for f in files: + if not can_format_file(f): + continue + if file_is_ignored(f): + continue + changed_files.append(f) + return changed_files + + +if os.path.isfile(revision): + print(action + " individual file: " + revision) + changed_files = [revision] +elif os.path.isdir(revision): + print(action + " files in directory: " + revision) + changed_files = [os.path.join(revision, x) for x in os.listdir(revision)] + + print("Changeset:") + for fname in changed_files: + print(fname) +elif not format_all: + if revision == 'main': + # fetch new changes when comparing to the master + os.system("git fetch origin main:main") + print(action + " since branch or revision: " + revision) + changed_files = get_changed_files(revision) + if len(changed_files) == 0: + print("No changed files found!") + exit(0) + + print("Changeset:") + for fname in changed_files: + print(fname) +else: + print(action + " all files") + +if confirm and not check_only: + print("The files listed above will be reformatted.") + result = input("Continue with changes (y/n)?\n") + if result != 'y': + print("Aborting.") + exit(0) + +format_commands = { + '.cpp': cpp_format_command, + '.ipp': cpp_format_command, + '.c': cpp_format_command, + '.hpp': cpp_format_command, + '.h': cpp_format_command, + '.hh': cpp_format_command, + '.cc': cpp_format_command, + '.txt': cmake_format_command, + '.py': 'black --quiet - --skip-string-normalization --line-length 120 --stdin-filename', + '.java': cpp_format_command, +} + +difference_files = [] + +header_top = "//===----------------------------------------------------------------------===//\n" +header_top += "// DuckDB\n" + "//\n" +header_bottom = "//\n" + "//\n" +header_bottom += "//===----------------------------------------------------------------------===//\n\n" +base_dir = os.path.join(os.getcwd(), 'src/include') + + +def get_formatted_text(f, full_path, directory, ext): + if not can_format_file(full_path): + if not force: + print( + "File " + + full_path + + " is not normally formatted - but attempted to format anyway. Use --force if formatting is desirable" + ) + exit(1) + if f == 'list.hpp': + # fill in list file + file_list = [ + os.path.join(dp, f) + for dp, dn, filenames in os.walk(directory) + for f in filenames + if os.path.splitext(f)[1] == '.hpp' and not f.endswith("list.hpp") + ] + file_list = [x.replace('src/include/', '') for x in file_list] + file_list.sort() + result = "" + for x in file_list: + result += '#include "%s"\n' % (x) + return result + + if ext == ".hpp" and directory.startswith("src/include"): + with open_utf8(full_path, 'r') as f: + lines = f.readlines() + + # format header in files + header_middle = "// " + os.path.relpath(full_path, base_dir) + "\n" + text = header_top + header_middle + header_bottom + is_old_header = True + for line in lines: + if not (line.startswith("//") or line.startswith("\n")) and is_old_header: + is_old_header = False + if not is_old_header: + text += line + + if ext == '.test' or ext == '.test_slow' or ext == '.test_coverage' or ext == '.benchmark': + f = open_utf8(full_path, 'r') + lines = f.readlines() + f.close() + + found_name = False + found_group = False + group_name = full_path.split('/')[-2] + new_path_line = '# name: ' + full_path + '\n' + new_group_line = '# group: [' + group_name + ']' + '\n' + found_diff = False + # Find description. + found_description = False + for line in lines: + if line.lower().startswith('# description:') or line.lower().startswith('#description:'): + if found_description: + print("Error formatting file " + full_path + ", multiple lines starting with # description found") + exit(1) + found_description = True + new_description_line = '# description: ' + line.split(':', 1)[1].strip() + '\n' + # Filter old meta. + meta = ['#name:', '# name:', '#description:', '# description:', '#group:', '# group:'] + lines = [line for line in lines if not any(line.lower().startswith(m) for m in meta)] + # Clean up empty leading lines. + while lines and not lines[0].strip(): + lines.pop(0) + # Ensure header is prepended. + header = [new_path_line] + if found_description: + header.append(new_description_line) + header.append(new_group_line) + header.append('\n') + return ''.join(header + lines) + proc_command = format_commands[ext].split(' ') + [full_path] + proc = subprocess.Popen( + proc_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=open(full_path) if ext == '.py' else None + ) + new_text = proc.stdout.read().decode('utf8') + stderr = proc.stderr.read().decode('utf8') + if len(stderr) > 0: + print(os.getcwd()) + print("Failed to format file " + full_path) + print(' '.join(proc_command)) + print(stderr) + exit(1) + new_text = new_text.replace('\r', '') + new_text = re.sub(r'\n*$', '', new_text) + return new_text + '\n' + + +def file_is_generated(text): + if '// This file is automatically generated by scripts/' in text: + return True + return False + + +def format_file(f, full_path, directory, ext): + global difference_files + with open_utf8(full_path, 'r') as f: + old_text = f.read() + # do not format auto-generated files + if file_is_generated(old_text) and ext != '.py': + return + old_lines = old_text.split('\n') + + new_text = get_formatted_text(f, full_path, directory, ext) + if ext in ('.cpp', '.hpp'): + new_text = new_text.replace('ARGS &&...args', 'ARGS &&... args') + if check_only: + new_lines = new_text.split('\n') + old_lines = [x for x in old_lines if '...' not in x] + new_lines = [x for x in new_lines if '...' not in x] + diff_result = difflib.unified_diff(old_lines, new_lines) + total_diff = "" + for diff_line in diff_result: + total_diff += diff_line + "\n" + total_diff = total_diff.strip() + + if len(total_diff) > 0: + print("----------------------------------------") + print("----------------------------------------") + print("Found differences in file " + full_path) + print("----------------------------------------") + print("----------------------------------------") + print(total_diff) + difference_files.append(full_path) + else: + tmpfile = full_path + ".tmp" + with open_utf8(tmpfile, 'w+') as f: + f.write(new_text) + os.rename(tmpfile, full_path) + + +def format_directory(directory): + files = os.listdir(directory) + files.sort() + + def process_file(f): + full_path = os.path.join(directory, f) + if os.path.isdir(full_path): + if f in ignored_directories or full_path in ignored_directories: + return + if not silent: + print(full_path) + format_directory(full_path) + elif can_format_file(full_path): + format_file(f, full_path, directory, '.' + f.split('.')[-1]) + + # Create thread for each file + with concurrent.futures.ThreadPoolExecutor() as executor: + threads = [executor.submit(process_file, f) for f in files] + # Wait for all tasks to complete + concurrent.futures.wait(threads) + + +if format_all: + try: + os.system(cmake_format_command.replace("${FILE}", "CMakeLists.txt")) + except: + pass + + for direct in formatted_directories: + format_directory(direct) + +else: + for full_path in changed_files: + splits = full_path.split(os.path.sep) + fname = splits[-1] + dirname = os.path.sep.join(splits[:-1]) + ext = '.' + full_path.split('.')[-1] + format_file(fname, full_path, dirname, ext) + +if check_only: + if len(difference_files) > 0: + print("") + print("") + print("") + print("Failed format-check: differences were found in the following files:") + for fname in difference_files: + print("- " + fname) + print('Run "make format-fix" to fix these differences automatically') + exit(1) + else: + print("Passed format-check") + exit(0) From ea1355721a882965038cd9408b3c0445549d16be Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 10:01:02 +0100 Subject: [PATCH 20/31] Add open_utf8 function to format_extension.py --- scripts/format_extension.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/format_extension.py b/scripts/format_extension.py index 0ed9ada..cce4eea 100644 --- a/scripts/format_extension.py +++ b/scripts/format_extension.py @@ -10,7 +10,14 @@ import difflib import re import concurrent.futures -from python_helpers import open_utf8 + +def open_utf8(fpath, flags): + import sys + + if sys.version_info[0] < 3: + return open(fpath, flags) + else: + return open(fpath, flags, encoding="utf8") try: ver = subprocess.check_output(('black', '--version'), text=True) @@ -311,7 +318,6 @@ def format_file(f, full_path, directory, ext): if file_is_generated(old_text) and ext != '.py': return old_lines = old_text.split('\n') - new_text = get_formatted_text(f, full_path, directory, ext) if ext in ('.cpp', '.hpp'): new_text = new_text.replace('ARGS &&...args', 'ARGS &&... args') From 87c94e1debb76337ec22d453ed8384a6a9899760 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 10:01:06 +0100 Subject: [PATCH 21/31] Update paths --- makefiles/duckdb_extension.Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index d11bce1..d139b1e 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -134,13 +134,13 @@ wasm_threads: #### Misc # Rule to fix formatting format-check: - python scripts/format_extension.py + python extension-ci-tools/scripts/format_extension.py format-check-silent: - python3 scripts/format_extension.py --all --check --silent + python3 extension-ci-tools/scripts/format_extension.py --all --check --silent format-fix: - python3 scripts/format_extension.py --all --fix --noconfirm + python3 extension-ci-tools/scripts/format_extension.py --all --fix --noconfirm update: git submodule update --remote --merge From 34c2efc021a017cc709fe1a9960c680b04bd4438 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 11:14:41 +0100 Subject: [PATCH 22/31] Add code quality check to TestCITools.yml --- .github/workflows/TestCITools.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 5672617..fe92a7f 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -61,3 +61,7 @@ jobs: duckdb_version: v1.1.3 exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools;windows_amd64' extra_toolchains: 'rust' + + code-quality-check: + name: Code quality check + uses: ./.github/workflows/_extension_code_quality.yml \ No newline at end of file From cff0722eb309585e64a526cfb18152d5a18a6bce Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 11:15:31 +0100 Subject: [PATCH 23/31] Use python instead of python3 everywhere --- makefiles/duckdb_extension.Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/makefiles/duckdb_extension.Makefile b/makefiles/duckdb_extension.Makefile index d139b1e..ff793d3 100644 --- a/makefiles/duckdb_extension.Makefile +++ b/makefiles/duckdb_extension.Makefile @@ -137,10 +137,10 @@ format-check: python extension-ci-tools/scripts/format_extension.py format-check-silent: - python3 extension-ci-tools/scripts/format_extension.py --all --check --silent + python extension-ci-tools/scripts/format_extension.py --all --check --silent format-fix: - python3 extension-ci-tools/scripts/format_extension.py --all --fix --noconfirm + python extension-ci-tools/scripts/format_extension.py --all --fix --noconfirm update: git submodule update --remote --merge From eb48b6e4676be917d83163c62b1466cd528ddeb9 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 11:19:32 +0100 Subject: [PATCH 24/31] Add newline --- .github/workflows/_extension_code_quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 63238bd..7e99fc2 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -42,4 +42,4 @@ jobs: - name: Generated Check shell: bash run: | - git diff --ignore-submodules --exit-code \ No newline at end of file + git diff --ignore-submodules --exit-code From 1eb4f175222347fabf551d4152e22f7f8cc19ad5 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:04:25 +0100 Subject: [PATCH 25/31] Add override to checkout repo --- .github/workflows/TestCITools.yml | 7 ++++++- .github/workflows/_extension_code_quality.yml | 11 ++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index fe92a7f..6a243fb 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -64,4 +64,9 @@ jobs: code-quality-check: name: Code quality check - uses: ./.github/workflows/_extension_code_quality.yml \ No newline at end of file + uses: ./.github/workflows/_extension_code_quality.yml + with: + extension_name: quack + override_repository: duckdb/extension-template + override_ci_tools_repository: ${{ github.repository }} + ci_tools_version: ${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 7e99fc2..9d956b1 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -19,9 +19,18 @@ jobs: steps: - uses: actions/checkout@v4 + name: Checkout current repository + if: ${{inputs.override_repository == ''}} with: fetch-depth: 0 - submodules: recursive + submodules: 'true' + + - uses: actions/checkout@v4 + name: Checkout Extension CI tools + with: + path: 'extension-ci-tools' + ref: ${{ inputs.ci_tools_version }} + repository: ${{ inputs.override_ci_tools_repository }} - name: Install shell: bash From 82fda34742122c7ccbaba3c6ad02597c4dc07234 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:06:47 +0100 Subject: [PATCH 26/31] Remove override repository --- .github/workflows/TestCITools.yml | 2 -- .github/workflows/_extension_code_quality.yml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 6a243fb..c5a22e5 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -66,7 +66,5 @@ jobs: name: Code quality check uses: ./.github/workflows/_extension_code_quality.yml with: - extension_name: quack - override_repository: duckdb/extension-template override_ci_tools_repository: ${{ github.repository }} ci_tools_version: ${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 9d956b1..485e700 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -20,7 +20,6 @@ jobs: steps: - uses: actions/checkout@v4 name: Checkout current repository - if: ${{inputs.override_repository == ''}} with: fetch-depth: 0 submodules: 'true' From 35f8ba3edd75e205053238d3ff1833e73d7e1016 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:12:04 +0100 Subject: [PATCH 27/31] Add the corresponding inputs --- .github/workflows/TestCITools.yml | 2 +- .github/workflows/_extension_code_quality.yml | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index c5a22e5..1778a39 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -66,5 +66,5 @@ jobs: name: Code quality check uses: ./.github/workflows/_extension_code_quality.yml with: - override_ci_tools_repository: ${{ github.repository }} + override_ci_tools_repository: "dtenwolde/extension-ci-tools" ci_tools_version: ${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 485e700..2c59728 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -6,6 +6,17 @@ on: explicit_checks: required: false type: string + # The version of the https://github.com/duckdb/extension-ci-tools submodule of the extension. In most cases will be identical to `duckdb_version`. + # Passing this explicitly is required because of https://github.com/actions/toolkit/issues/1264 + ci_tools_version: + required: true + type: string + # Override the repo for the CI tools (for testing CI tools itself) + override_ci_tools_repository: + required: false + type: string + default: "duckdb/extension-ci-tools" + jobs: format-check: From 714173dca9b590fd654f625af373ea0dfc6ec184 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:21:50 +0100 Subject: [PATCH 28/31] Change to use main ci tools version --- .github/workflows/TestCITools.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 1778a39..445df01 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -67,4 +67,4 @@ jobs: uses: ./.github/workflows/_extension_code_quality.yml with: override_ci_tools_repository: "dtenwolde/extension-ci-tools" - ci_tools_version: ${{ github.sha }} \ No newline at end of file + ci_tools_version: main From 762791ffc3cface5160e5fc98576b34a009bebf3 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:27:43 +0100 Subject: [PATCH 29/31] Add some debugging --- .github/workflows/TestCITools.yml | 2 +- .github/workflows/_extension_code_quality.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/TestCITools.yml b/.github/workflows/TestCITools.yml index 445df01..37f9845 100644 --- a/.github/workflows/TestCITools.yml +++ b/.github/workflows/TestCITools.yml @@ -66,5 +66,5 @@ jobs: name: Code quality check uses: ./.github/workflows/_extension_code_quality.yml with: - override_ci_tools_repository: "dtenwolde/extension-ci-tools" + override_ci_tools_repository: "Dtenwolde/extension-ci-tools" ci_tools_version: main diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 2c59728..ce43f57 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -56,6 +56,8 @@ jobs: clang-format --version clang-format --dump-config black --version + ls + pwd make format-check-silent - name: Generated Check From f09ed60a6de2cac9693a7761f78a9729059e9b43 Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:28:50 +0100 Subject: [PATCH 30/31] Try cd .. --- .github/workflows/_extension_code_quality.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index ce43f57..72b3cdd 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -56,8 +56,10 @@ jobs: clang-format --version clang-format --dump-config black --version + cd .. ls pwd + make format-check-silent - name: Generated Check From ac95c032dddb0ff81af94aeb33273771bfd3212b Mon Sep 17 00:00:00 2001 From: dtenwolde Date: Tue, 10 Dec 2024 13:31:14 +0100 Subject: [PATCH 31/31] Output content of makefile --- .github/workflows/_extension_code_quality.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/_extension_code_quality.yml b/.github/workflows/_extension_code_quality.yml index 72b3cdd..c123286 100644 --- a/.github/workflows/_extension_code_quality.yml +++ b/.github/workflows/_extension_code_quality.yml @@ -56,10 +56,7 @@ jobs: clang-format --version clang-format --dump-config black --version - cd .. - ls - pwd - + cat makefiles/duckdb_extension.Makefile make format-check-silent - name: Generated Check