Merge branch 'sonic-net:master' into show_appdb_acl

azure-pipelines.yml

@@ -43,14 +43,15 @@ stages:
       vmImage: ubuntu-20.04
 
     container:
-      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bullseye:$(BUILD_BRANCH)
+      image: sonicdev-microsoft.azurecr.io:443/sonic-slave-bookworm:$(BUILD_BRANCH)
 
     steps:
     - script: |
         set -ex
         sudo apt-get update
         sudo apt-get install -y python3-pip
         sudo pip3 install requests==2.31.0
+        sudo apt-get install -y python3-protobuf
       displayName: "Install dependencies"
 
     - script: |
@@ -84,16 +85,15 @@ stages:
         sudo dpkg -i libyang_1.0.73_amd64.deb
         sudo dpkg -i libyang-cpp_1.0.73_amd64.deb
         sudo dpkg -i python3-yang_1.0.73_amd64.deb
-        sudo dpkg -i libprotobuf*.deb
-      workingDirectory: $(Pipeline.Workspace)/target/debs/bullseye/
+      workingDirectory: $(Pipeline.Workspace)/target/debs/bookworm/
       displayName: 'Install Debian dependencies'
 
     - task: DownloadPipelineArtifact@2
       inputs:
         source: specific
         project: build
         pipeline: 9
-        artifact: sonic-swss-common
+        artifact: sonic-swss-common-bookworm
         runVersion: 'latestFromBranch'
         runBranch: 'refs/heads/$(sourceBranch)'
       displayName: "Download sonic swss common deb packages"
@@ -134,20 +134,22 @@ stages:
         sudo pip3 install sonic_yang_models-1.0-py3-none-any.whl
         sudo pip3 install sonic_config_engine-1.0-py3-none-any.whl
         sudo pip3 install sonic_platform_common-1.0-py3-none-any.whl
-      workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bullseye/
+      workingDirectory: $(Pipeline.Workspace)/target/python-wheels/bookworm/
       displayName: 'Install Python dependencies'
 
     - script: |
         set -ex
         # Install .NET CORE
         curl -sSL https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
-        sudo apt-add-repository https://packages.microsoft.com/debian/11/prod
+        sudo apt-add-repository https://packages.microsoft.com/debian/12/prod
         sudo apt-get update
         sudo apt-get install -y dotnet-sdk-8.0
       displayName: "Install .NET CORE"
 
     - script: |
-        python3 setup.py test
+        pip3 install ".[testing]"
+        pip3 uninstall --yes sonic-utilities
+        pytest
       displayName: 'Test Python 3'
 
     - task: PublishTestResults@2
@@ -167,7 +169,7 @@ stages:
 
     - script: |
         set -e
-        python3 setup.py bdist_wheel
+        python3 -m build -n
       displayName: 'Build Python 3 wheel'
 
     - publish: '$(System.DefaultWorkingDirectory)/dist/'

config/main.py

@@ -889,8 +889,9 @@ def _get_disabled_services_list(config_db):
 def _stop_services():
     try:
         subprocess.check_call(['sudo', 'monit', 'status'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        click.echo("Disabling container monitoring ...")
+        click.echo("Disabling container and routeCheck monitoring ...")
         clicommon.run_command(['sudo', 'monit', 'unmonitor', 'container_checker'])
+        clicommon.run_command(['sudo', 'monit', 'unmonitor', 'routeCheck'])
     except subprocess.CalledProcessError as err:
         pass
 
@@ -948,14 +949,14 @@ def _restart_services():
     # If load_minigraph exit before eth0 restart, commands after load_minigraph may failed
     wait_service_restart_finish('interfaces-config', last_interface_config_timestamp)
     wait_service_restart_finish('networking', last_networking_timestamp)
-
     try:
         subprocess.check_call(['sudo', 'monit', 'status'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-        click.echo("Enabling container monitoring ...")
+        click.echo("Enabling container and routeCheck monitoring ...")
+        clicommon.run_command(['sudo', 'monit', 'monitor', 'routeCheck'])
         clicommon.run_command(['sudo', 'monit', 'monitor', 'container_checker'])
+        time.sleep(1)
     except subprocess.CalledProcessError as err:
         pass
-
     # Reload Monit configuration to pick up new hostname in case it changed
     click.echo("Reloading Monit configuration ...")
     clicommon.run_command(['sudo', 'monit', 'reload'])
@@ -1321,6 +1322,18 @@ def flush_configdb(namespace=DEFAULT_NAMESPACE):
     return client, config_db
 
 
+def delete_transceiver_tables():
+    tables = ["TRANSCEIVER_INFO", "TRANSCEIVER_STATUS", "TRANSCEIVER_PM",
+              "TRANSCEIVER_FIRMWARE_INFO", "TRANSCEIVER_DOM_SENSOR", "TRANSCEIVER_DOM_THRESHOLD"]
+    state_db_del_pattern = "|*"
+
+    # delete TRANSCEIVER tables from State DB
+    state_db = SonicV2Connector(use_unix_socket_path=True)
+    state_db.connect(state_db.STATE_DB, False)
+    for table in tables:
+        state_db.delete_all_by_pattern(state_db.STATE_DB, table + state_db_del_pattern)
+
+
 def migrate_db_to_lastest(namespace=DEFAULT_NAMESPACE):
     # Migrate DB contents to latest version
     db_migrator = '/usr/local/bin/db_migrator.py'
@@ -1373,17 +1386,21 @@ def multiasic_write_to_db(filename, load_sysinfo):
 
 
 def config_file_yang_validation(filename):
-    config_to_check = read_json_file(filename)
+    config = read_json_file(filename)
     sy = sonic_yang.SonicYang(YANG_DIR)
     sy.loadYangModel()
-    try:
-        sy.loadData(configdbJson=config_to_check)
-        sy.validate_data_tree()
-    except sonic_yang.SonicYangException as e:
-        click.secho("{} fails YANG validation! Error: {}".format(filename, str(e)),
-                    fg='magenta')
-        raise click.Abort()
-
+    asic_list = [HOST_NAMESPACE]
+    if multi_asic.is_multi_asic():
+        asic_list.extend(multi_asic.get_namespace_list())
+    for scope in asic_list:
+        config_to_check = config.get(scope) if multi_asic.is_multi_asic() else config
+        try:
+            sy.loadData(configdbJson=config_to_check)
+            sy.validate_data_tree()
+        except sonic_yang.SonicYangException as e:
+            click.secho("{} fails YANG validation! Error: {}".format(filename, str(e)),
+                        fg='magenta')
+            raise click.Abort()
 
 # This is our main entrypoint - the main 'config' command
 @click.group(cls=clicommon.AbbreviationGroup, context_settings=CONTEXT_SETTINGS)
@@ -1830,7 +1847,7 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
         if multi_asic.is_multi_asic():
             # Multiasic has not 100% fully validated. Thus pass here.
             pass
-        else:
+        elif "golden" in filename.lower():
             config_file_yang_validation(filename)
 
     #Stop services before config push
@@ -1900,6 +1917,7 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart, force, file_form
                 cfg_hwsku = output.strip()
 
             client, config_db = flush_configdb(namespace)
+            delete_transceiver_tables()
 
             if load_sysinfo:
                 if namespace is DEFAULT_NAMESPACE:
@@ -2018,23 +2036,21 @@ def load_minigraph(db, no_service_restart, traffic_shift_away, override_config,
                         fg='magenta')
             raise click.Abort()
 
-        config_to_check = read_json_file(golden_config_path)
-        if multi_asic.is_multi_asic():
-            # Multiasic has not 100% fully validated. Thus pass here.
-            pass
-        else:
-            config_file_yang_validation(golden_config_path)
+        config_file_yang_validation(golden_config_path)
 
+        config_to_check = read_json_file(golden_config_path)
         # Dependency check golden config json
+        asic_list = [HOST_NAMESPACE]
         if multi_asic.is_multi_asic():
-            host_config = config_to_check.get('localhost', {})
-        else:
-            host_config = config_to_check
-        table_hard_dependency_check(host_config)
+            asic_list.extend(multi_asic.get_namespace_list())
+        for scope in asic_list:
+            host_config = config_to_check.get(scope) if multi_asic.is_multi_asic() else config_to_check
+            table_hard_dependency_check(host_config)
 
-    #Stop services before config push
+    # Stop services before config push
     if not no_service_restart:
         log.log_notice("'load_minigraph' stopping services...")
+        delete_transceiver_tables()
         _stop_services()
 
     # For Single Asic platform the namespace list has the empty string
@@ -2206,8 +2222,8 @@ def generate_sysinfo(cur_config, config_input, ns=None):
     if not platform:
         platform = device_info.get_platform()
 
-    device_metadata['localhost']['mac'] = mac
-    device_metadata['localhost']['platform'] = platform
+    device_metadata['localhost']['mac'] = mac.rstrip('\n')
+    device_metadata['localhost']['platform'] = platform.rstrip('\n')
 
     return
 

counterpoll/main.py

@@ -397,15 +397,12 @@ def disable(ctx):
 
 
 # ENI counter commands
-@cli.group()
+@click.group()
 @click.pass_context
 def eni(ctx):
     """ ENI counter commands """
     ctx.obj = ConfigDBConnector()
     ctx.obj.connect()
-    if not is_dpu(ctx.obj):
-        click.echo("ENI counters are not supported on non DPU platforms")
-        exit(1)
 
 
 @eni.command(name='interval')
@@ -534,3 +531,27 @@ def disable(filename):
 def delay(filename):
     """ Delay counters in config_db file """
     _update_config_db_flex_counter_table("delay", filename)
+
+
+"""
+The list of dynamic commands that are added on a specific condition.
+Format:
+    (click group/command, callback function)
+"""
+dynamic_commands = [
+    (eni, is_dpu)
+]
+
+
+def register_dynamic_commands(cmds):
+    """
+    Dynamically register commands based on condition callback.
+    """
+    db = ConfigDBConnector()
+    db.connect()
+    for cmd, cb in cmds:
+        if cb(db):
+            cli.add_command(cmd)
+
+
+register_dynamic_commands(dynamic_commands)

doc/Command-Reference.md

@@ -5134,6 +5134,37 @@ This command is to display the link-training status of the selected interfaces.
     Ethernet8      trained          on      up       up
   ```
 
+**show interfaces errors**
+
+The show interface errors command provides detailed statistics and error counters for MAC-level operations on an interface. It displays the status of various operational parameters, error counts, and timestamps for when these errors occurred.
+
+- Usage:
+  ```
+  show interfaces errors [<interface_name>]
+  ```
+
+- Example:
+  ```
+  admin@sonic:~$ show interfaces errors Ethernet4
+  Port Errors                        Count           Last timestamp(UTC)
+  ---------------------------------- -----           -------------------
+  oper_error_status                  5442            2024-11-02 04:00:05
+  mac_local_fault                    2               2024-11-02 04:00:05
+  fec_sync_loss                      2               2024-11-02 04:00:05
+  fec_alignment_loss                 2               2024-11-02 04:00:05
+  high_ser_error                     2               2024-11-02 04:00:05
+  high ber_error                     2               2024-11-02 04:00:05
+  data_unit_crc_error                2               2024-11-02 04:00:05
+  data_unit_misalignment_error       2               2024-11-02 04:00:05
+  signal_local_error                 2               2024-11-02 04:00:05
+  mac_remote_fault                   2               2024-11-02 04:00:50
+  crc_rate                           2               2024-11-02 04:00:50
+  data_unit_size                     2               2024-11-02 04:00:50
+  code_group_error                   0               Never
+  no_rx_reachability                 0               Never
+  ```
+
+
 **show interfaces mpls**
 
 This command is used to display the configured MPLS state for the list of configured interfaces.

generic_config_updater/field_operation_validators.py

@@ -12,6 +12,7 @@
 GCU_TABLE_MOD_CONF_FILE = f"{SCRIPT_DIR}/gcu_field_operation_validators.conf.json"
 GET_HWSKU_CMD = "sonic-cfggen -d -v DEVICE_METADATA.localhost.hwsku"
 
+
 def get_asic_name():
     asic = "unknown"
 

generic_config_updater/gcu_field_operation_validators.conf.json

@@ -27,7 +27,7 @@
             },
             "broadcom_asics": {
                 "th": [ "Force10-S6100", "Arista-7060CX-32S-C32", "Arista-7060CX-32S-C32-T1", "Arista-7060CX-32S-D48C8", "Celestica-DX010-C32", "Seastone-DX010" ],
-                "th2": [ "Arista-7260CX3-D108C8",  "Arista-7260CX3-C64", "Arista-7260CX3-Q64" ],
+                "th2": [ "Arista-7260CX3-D108C10", "Arista-7260CX3-D108C8",  "Arista-7260CX3-C64", "Arista-7260CX3-Q64" ],
                 "th3": [ "Nokia-IXR7220-H3" ],
                 "th4": [ "Nokia-IXR7220-H4-64D", "Nokia-IXR7220-H4-32D" ],
                 "th5": [ "Nokia-IXR7220-H5-64D" ],

generic_config_updater/gu_common.py

@@ -181,6 +181,8 @@ def validate_field_operation(self, old_config, target_config):
                 if any(op['op'] == operation and field == op['path'] for op in patch):
                     raise IllegalPatchOperationError("Given patch operation is invalid. Operation: {} is illegal on field: {}".format(operation, field))
 
+        self.illegal_dataacl_check(old_config, target_config)
+
         def _invoke_validating_function(cmd, jsonpatch_element):
             # cmd is in the format as <package/module name>.<method name>
             method_name = cmd.split(".")[-1]
@@ -212,6 +214,48 @@ def _invoke_validating_function(cmd, jsonpatch_element):
                 if not _invoke_validating_function(function, element):
                     raise IllegalPatchOperationError("Modification of {} table is illegal- validating function {} returned False".format(table, function))
 
+    def illegal_dataacl_check(self, old_config, upd_config):
+        '''
+        Block data ACL changes when patch includes:
+        1. table "type" being replaced
+        2. rule update on tables with table "type" replaced
+        This will cause race condition when swss consume the change of
+        acl table and acl rule and make the changed acl rule inactive
+        '''
+        old_acl_table = old_config.get("ACL_TABLE", {})
+        upd_acl_table = upd_config.get("ACL_TABLE", {})
+
+        # Pick data acl table with "type" field
+        old_dacl_table = [table for table, fields in old_acl_table.items()
+                          if fields.get("type") and fields["type"] != "CTRLPLANE"]
+        upd_dacl_table = [table for table, fields in upd_acl_table.items()
+                          if fields.get("type") and fields["type"] != "CTRLPLANE"]
+
+        # Pick intersect common tables that "type" being replaced
+        common_dacl_table = set(old_dacl_table).intersection(set(upd_dacl_table))
+        # Identify tables from the intersection where the "type" field differs
+        modified_common_dacl_table = [
+            table for table in common_dacl_table
+            if old_acl_table[table]["type"] != upd_acl_table[table]["type"]
+        ]
+
+        old_acl_rule = old_config.get("ACL_RULE", {})
+        upd_acl_rule = upd_config.get("ACL_RULE", {})
+
+        # Pick rules with its dependent table which has "type" replaced
+        old_dacl_rule = [rule for rule in old_acl_rule
+                         if rule.split("|")[0] in modified_common_dacl_table]
+        upd_dacl_rule = [rule for rule in upd_acl_rule
+                         if rule.split("|")[0] in modified_common_dacl_table]
+
+        # Block changes if acl rule change on tables with "type" replaced
+        for key in set(old_dacl_rule).union(set(upd_dacl_rule)):
+            if (old_acl_rule.get(key, {}) != upd_acl_rule.get(key, {})):
+                raise IllegalPatchOperationError(
+                    "Modification of dataacl rule {} is illegal: \
+                        acl table type changed in {}".format(
+                            key, modified_common_dacl_table
+                    ))
 
     def validate_lanes(self, config_db):
         if "PORT" not in config_db: