Skip to content

Commit

Permalink
ci: add pre-commit to cleanup ignore lists
Browse files Browse the repository at this point in the history
  • Loading branch information
hfudev committed Aug 9, 2023
1 parent 0cee3ac commit 60db076
Show file tree
Hide file tree
Showing 10 changed files with 73 additions and 378 deletions.
2 changes: 2 additions & 0 deletions .gitlab/ci/pre_check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,15 @@ check_pre_commit_master_release:
- .check_pre_commit_template
- .rules:protected
script:
- fetch_submodules
- git diff-tree --no-commit-id --name-only -r $PIPELINE_COMMIT_SHA | xargs pre-commit run --files

check_pre_commit_MR:
extends:
- .check_pre_commit_template
- .rules:dev
script:
- fetch_submodules
- python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py files ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME} | xargs pre-commit run --files

check_MR_style_dangerjs:
Expand Down
1 change: 0 additions & 1 deletion .gitlab/ci/rules.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
- "examples/**/*"

.patterns-build-example_test: &patterns-build-example_test
- "tools/ci/build_example_dirs.txt"
- "tools/ci/get_supported_examples.sh"

.patterns-build_components: &patterns-build_components
Expand Down
6 changes: 6 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,12 @@ repos:
language: python
always_run: true
pass_filenames: false
- id: cleanup-ignore-lists
name: Remove non-existing patterns from ignore lists
entry: tools/ci/cleanup_ignore_lists.py
language: python
always_run: true
require_serial: true
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
hooks:
Expand Down
5 changes: 0 additions & 5 deletions tools/ci/build_example_dirs.txt

This file was deleted.

243 changes: 0 additions & 243 deletions tools/ci/check_copyright_ignore.txt

Large diffs are not rendered by default.

10 changes: 0 additions & 10 deletions tools/ci/check_public_headers_exceptions.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ components/xtensa/esp32s3/include/xtensa/config/



components/freertos/esp_additions/include/freertos_tasks_c_additions.h
components/freertos/FreeRTOS-Kernel/include/freertos/
components/freertos/FreeRTOS-Kernel/portable/xtensa/include/freertos/
components/freertos/FreeRTOS-Kernel-SMP/include/freertos/
Expand All @@ -19,7 +18,6 @@ components/log/include/esp_log_internal.h

components/esp_rom/include/esp32s2/rom/rsa_pss.h

components/esp_common/include/esp_private/

# LWIP: sockets.h uses #include_next<>, which doesn't work correctly with the checker
# memp_std.h is supposed to be included multiple times with different settings
Expand Down Expand Up @@ -54,14 +52,11 @@ components/wpa_supplicant/esp_supplicant/include/
components/mbedtls/mbedtls/
components/mbedtls/port/include/
components/mbedtls/port/dynamic/esp_mbedtls_dynamic_impl.h
components/mbedtls/test/

components/esp-tls/private_include/

components/protobuf-c/

components/esp_serial_slave_link/include/esp_serial_slave_link/essl.h
components/esp_serial_slave_link/include/esp_serial_slave_link/essl_sdio.h

components/fatfs/vfs/vfs_fat_internal.h
components/fatfs/src/ffconf.h
Expand All @@ -87,7 +82,6 @@ components/openthread/openthread/
#
components/app_trace/include/esp_sysview_trace.h
components/esp_gdbstub/include/esp_gdbstub.h
components/openssl/include/platform/ssl_pm.h

components/esp_hw_support/include/esp_memprot.h
components/esp_hw_support/include/esp_private/esp_memprot_internal.h
Expand Down Expand Up @@ -121,16 +115,12 @@ components/esp_wifi/include/esp_wifi_netif.h
components/esp_wifi/include/smartconfig_ack.h
components/esp_wifi/include/esp_wifi_default.h
components/esp_event/include/esp_event_base.h
components/esp_netif/include/esp_netif_sta_list.h
components/esp_netif/include/esp_netif_defaults.h
components/esp_netif/include/esp_netif_net_stack.h
components/esp_netif/include/esp_netif_ppp.h
components/protocomm/include/transports/protocomm_httpd.h
components/fatfs/src/diskio.h
components/fatfs/diskio/diskio_sdmmc.h
components/openssl/include/openssl/ssl.h
components/ulp/include/ulp_common.h
components/ulp/include/esp32s2/ulp_riscv.h
components/mbedtls/esp_crt_bundle/include/esp_crt_bundle.h
components/wifi_provisioning/include/wifi_provisioning/scheme_softap.h
components/usb/include/esp_private/usb_phy.h
Expand Down
63 changes: 63 additions & 0 deletions tools/ci/cleanup_ignore_lists.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env python

# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0

import glob
import os
import typing as t

from idf_ci_utils import IDF_PATH


def print_list(_list: t.Iterable[str], title: t.Optional[str] = None) -> None:
if not _list:
return

if title:
print(title)

for i in _list:
print('- ', i)


if __name__ == '__main__':
os.chdir(IDF_PATH)
ignore_lists = set()
ignore_lists.update(glob.glob('tools/ci/*.txt', recursive=True))
ignore_lists.remove('tools/ci/ignore_build_warnings.txt')
ignore_lists.remove('tools/ci/check_ldgen_mapping_exceptions.txt')
print_list(ignore_lists, 'Ignore lists:')

updated_files = []
for f in ignore_lists:
print('Checking file:', f)

updated = False
lines = []
with open(f) as fr:
for line in map(str.strip, fr.readlines()):
if line.startswith('#'):
lines.append(line)
continue

if not line:
lines.append(line)
continue

glob_pattern = line
if not list(glob.glob(glob_pattern, recursive=True)):
print(' - No match:', glob_pattern)
updated = True
else:
lines.append(glob_pattern)
lines.append('')

if updated:
updated_files.append(f)
with open(f, 'w') as fw:
fw.write('\n'.join(lines))

if updated_files:
print_list(updated_files, 'Updated files:')
exit(1)
3 changes: 1 addition & 2 deletions tools/ci/exclude_check_tools_files.txt
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,9 @@ tools/ci/fix_empty_prototypes.sh
tools/ci/get-full-sources.sh
tools/ci/idf_ci_utils.py
tools/ci/ignore_build_warnings.txt
tools/ci/ignore_build_warnings_linux.txt
tools/ci/mirror-submodule-update.sh
tools/ci/multirun_with_pyenv.sh
tools/ci/mypy_ignore_list.txt
tools/ci/normalize_clangtidy_path.py
tools/ci/push_to_github.sh
tools/ci/python_packages/wifi_tools.py
tools/ci/utils.sh
Expand All @@ -40,3 +38,4 @@ tools/set-submodules-to-github.sh
tools/templates/sample_component/CMakeLists.txt
tools/templates/sample_component/include/main.h
tools/templates/sample_component/main.c
tools/ci/cleanup_ignore_lists.py
1 change: 1 addition & 0 deletions tools/ci/executable-list.txt
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ tools/ci/check_soc_struct_headers.py
tools/ci/check_tools_files_patterns.py
tools/ci/check_type_comments.py
tools/ci/checkout_project_ref.py
tools/ci/cleanup_ignore_lists.py
tools/ci/deploy_docs.py
tools/ci/envsubst.py
tools/ci/fix_empty_prototypes.sh
Expand Down
117 changes: 0 additions & 117 deletions tools/ci/mypy_ignore_list.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,9 @@ components/app_update/otatool.py
components/efuse/efuse_table_gen.py
components/efuse/test_efuse_host/efuse_tests.py
components/esp_local_ctrl/python/esp_local_ctrl_pb2.py
components/esp_netif/test_apps/component_ut_test.py
components/lwip/weekend_test/net_suite_test.py
components/mbedtls/esp_crt_bundle/gen_crt_bundle.py
components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/test_gen_crt_bundle.py
components/mqtt/weekend_test/mqtt_publish_test.py
components/nvs_flash/nvs_partition_generator/nvs_partition_gen.py
components/partition_table/gen_empty_partition.py
components/partition_table/gen_esp32part.py
Expand All @@ -21,116 +19,12 @@ components/wifi_provisioning/python/wifi_constants_pb2.py
components/wifi_provisioning/python/wifi_ctrl_pb2.py
components/wifi_provisioning/python/wifi_scan_pb2.py
components/xtensa/trax/traceparse.py
docs/build_docs.py
docs/extensions/google_analytics.py
docs/extensions/html_redirects.py
docs/extensions/list_filter.py
docs/extensions/toctree_filter.py
docs/get_github_rev.py
docs/idf_extensions/build_system/__init__.py
docs/idf_extensions/esp_err_definitions.py
docs/idf_extensions/exclude_docs.py
docs/idf_extensions/format_idf_target.py
docs/idf_extensions/gen_defines.py
docs/idf_extensions/gen_idf_tools_links.py
docs/idf_extensions/gen_toolchain_links.py
docs/idf_extensions/gen_version_specific_includes.py
docs/idf_extensions/include_build_file.py
docs/idf_extensions/kconfig_reference.py
docs/idf_extensions/latex_builder.py
docs/idf_extensions/link_roles.py
docs/idf_extensions/run_doxygen.py
docs/idf_extensions/util.py
docs/sanitize_version.py
docs/test/en/conf.py
docs/test/test_docs.py
docs/test/test_sphinx_idf_extensions.py
examples/bluetooth/nimble/blecent/blecent_test.py
examples/bluetooth/nimble/blehr/blehr_test.py
examples/bluetooth/nimble/bleprph/bleprph_test.py
examples/cxx/exceptions/example_test.py
examples/cxx/pthread/example_test.py
examples/cxx/rtti/example_test.py
examples/get-started/blink/example_test.py
examples/get-started/hello_world/example_test.py
examples/peripherals/gpio/generic_gpio/example_test.py
examples/peripherals/i2c/i2c_tools/example_test.py
examples/peripherals/rmt/ir_protocols/example_test.py
examples/peripherals/sdio/sdio_test.py
examples/peripherals/twai/twai_alert_and_recovery/example_test.py
examples/peripherals/twai/twai_network/example_test.py
examples/peripherals/twai/twai_self_test/example_test.py
examples/protocols/esp_http_client/esp_http_client_test.py
examples/protocols/esp_local_ctrl/example_test.py
examples/protocols/esp_local_ctrl/scripts/esp_local_ctrl.py
examples/protocols/esp_local_ctrl/scripts/proto_lc.py
examples/protocols/http_server/advanced_tests/http_server_advanced_test.py
examples/protocols/http_server/advanced_tests/scripts/test.py
examples/protocols/http_server/persistent_sockets/http_server_persistence_test.py
examples/protocols/http_server/simple/http_server_simple_test.py
examples/protocols/http_server/ws_echo_server/ws_server_example_test.py
examples/protocols/https_request/example_test.py
examples/protocols/https_x509_bundle/example_test.py
examples/protocols/icmp_echo/example_test.py
examples/protocols/mqtt/ssl/mqtt_ssl_example_test.py
examples/protocols/mqtt/tcp/mqtt_tcp_example_test.py
examples/protocols/mqtt/ws/mqtt_ws_example_test.py
examples/protocols/mqtt/wss/mqtt_wss_example_test.py
examples/protocols/openssl_client/example_test.py
examples/protocols/openssl_server/example_test.py
examples/protocols/pppos_client/example_test.py
examples/protocols/sntp/example_test.py
examples/protocols/sockets/non_blocking/example_test.py
examples/protocols/sockets/tcp_client/example_test.py
examples/protocols/sockets/tcp_server/example_test.py
examples/protocols/sockets/udp_client/example_test.py
examples/protocols/sockets/udp_server/example_test.py
examples/protocols/websocket/example_test.py
examples/provisioning/legacy/ble_prov/ble_prov_test.py
examples/provisioning/legacy/custom_config/components/custom_provisioning/python/custom_config_pb2.py
examples/provisioning/legacy/softap_prov/softap_prov_test.py
examples/provisioning/wifi_prov_mgr/wifi_prov_mgr_test.py
examples/security/flash_encryption/example_test.py
examples/storage/ext_flash_fatfs/example_test.py
examples/storage/nvs_rw_blob/nvs_rw_blob_example_test.py
examples/storage/nvs_rw_value/nvs_rw_value_example_test.py
examples/storage/nvs_rw_value_cxx/nvs_rw_value_cxx_example_test.py
examples/storage/partition_api/partition_find/partition_find_example_test.py
examples/storage/partition_api/partition_mmap/partition_mmap_example_test.py
examples/storage/partition_api/partition_ops/partition_ops_example_test.py
examples/storage/parttool/example_test.py
examples/storage/parttool/parttool_example.py
examples/storage/sd_card/sd_card_example_test.py
examples/storage/semihost_vfs/semihost_vfs_example_test.py
examples/storage/spiffs/spiffs_example_test.py
examples/storage/spiffsgen/example_test.py
examples/storage/wear_levelling/wear_levelling_example_test.py
examples/system/base_mac_address/example_test.py
examples/system/console/example_test.py
examples/system/deep_sleep/example_test.py
examples/system/esp_event/default_event_loop/example_test.py
examples/system/esp_event/user_event_loops/example_test.py
examples/system/esp_timer/example_test.py
examples/system/freertos/real_time_stats/example_test.py
examples/system/gcov/example_test.py
examples/system/himem/example_test.py
examples/system/light_sleep/example_test.py
examples/system/ota/advanced_https_ota/example_test.py
examples/system/ota/native_ota_example/example_test.py
examples/system/ota/otatool/example_test.py
examples/system/ota/otatool/get_running_partition.py
examples/system/ota/otatool/otatool_example.py
examples/system/ota/simple_ota_example/example_test.py
examples/system/perfmon/example_test.py
examples/system/select/example_test.py
examples/system/startup_time/example_test.py
examples/system/sysview_tracing/example_test.py
examples/system/sysview_tracing_heap_log/example_test.py
examples/system/task_watchdog/example_test.py
examples/system/ulp/ulp_fsm/ulp/example_test.py
examples/system/ulp/ulp_fsm/ulp_adc/example_test.py
examples/system/unit_test/example_test.py
examples/wifi/iperf/iperf_test.py
tools/ble/lib_ble_client.py
tools/ble/lib_gap.py
tools/ble/lib_gatt.py
Expand Down Expand Up @@ -195,17 +89,6 @@ tools/ldgen/test/test_generation.py
tools/ldgen/test/test_output_commands.py
tools/mass_mfg/mfg_gen.py
tools/test_apps/build_system/ldgen_test/check_placements.py
tools/test_apps/protocols/mqtt/publish_connect_test/app_test.py
tools/test_apps/protocols/openssl/app_test.py
tools/test_apps/protocols/pppos/app_test.py
tools/test_apps/system/gdb_loadable_elf/app_test.py
tools/test_apps/system/longjmp_test/app_test.py
tools/test_apps/system/memprot/app_test.py
tools/test_apps/system/monitor_ide_integration/app_test.py
tools/test_apps/system/panic/app_test.py
tools/test_apps/system/panic/panic_tests.py
tools/test_apps/system/panic/test_panic_util/test_panic_util.py
tools/test_apps/system/startup/app_test.py
tools/test_idf_py/extra_path/some_ext.py
tools/test_idf_py/idf_ext.py
tools/test_idf_py/test_idf_extensions/test_ext/test_extension.py
Expand Down

0 comments on commit 60db076

Please sign in to comment.