diff --git a/images/sonic/Dockerfile b/images/sonic/Dockerfile index 324bdea..f2e389c 100644 --- a/images/sonic/Dockerfile +++ b/images/sonic/Dockerfile @@ -17,4 +17,4 @@ COPY --from=ghcr.io/metal-stack/mini-lab-sonic:base /frr-pythontools.deb /frr-py ENTRYPOINT ["/launch.py"] -COPY config_db.json mirror_tap_to_eth.sh launch.py / +COPY mirror_tap_to_eth.sh mirror_tap_to_front_panel.sh port_config.ini launch.py / diff --git a/images/sonic/config_db.json b/images/sonic/config_db.json deleted file mode 100644 index 56f4742..0000000 --- a/images/sonic/config_db.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "AUTO_TECHSUPPORT": { - "GLOBAL": { - "state": "disabled" - } - }, - "DEVICE_METADATA": { - "localhost": { - "docker_routing_config_mode": "split-unified", - "hostname": "{{ hostname }}", - "hwsku": "Force10-S6000", - "mac": "{{ mac }}", - "platform": "x86_64-kvm_x86_64-r0", - "type": "LeafRouter" - } - }, - "FEATURE": { - "gnmi": { - "state": "disabled" - }, - "mgmt-framework": { - "state": "disabled" - }, - "snmp": { - "state": "disabled" - }, - "teamd": { - "state": "disabled" - } - }, - "MGMT_INTERFACE": null, - "MGMT_PORT": { - "eth0": { - "alias": "eth0", - "admin_status": "up" - } - }, - "PORT": { - "Ethernet0": { - "lanes": "25,26,27,28", - "alias": "fortyGigE0/0", - "index": "0", - "speed": "40000", - "admin_status": "up", - "mtu": "9100" - }, - "Ethernet4": { - "lanes": "29,30,31,32", - "alias": "fortyGigE0/4", - "index": "1", - "speed": "40000", - "admin_status": "up", - "mtu": "9100" - }, - "Ethernet120": { - "lanes": "33,34,35,36", - "alias": "fortyGigE0/8", - "index": "2", - "speed": "40000", - "admin_status": "up", - "mtu": "9100" - } - }, - "VERSIONS": { - "DATABASE": { - "VERSION": "version_202311_03" - } - } -} \ No newline at end of file diff --git a/images/sonic/launch.py b/images/sonic/launch.py index f4921d4..2bdf825 100755 --- a/images/sonic/launch.py +++ b/images/sonic/launch.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 import fcntl +import glob import json import logging import os @@ -15,13 +16,14 @@ BASE_IMG = '/sonic-vs.img' +VS_DEVICES_PATH = '/usr/share/sonic/device/x86_64-kvm_x86_64-r0/' + class Qemu: - def __init__(self, name: str, smp: str, memory: str, interfaces: int): + def __init__(self, name: str, smp: str, memory: str): self._name = name self._smp = smp self._memory = memory - self._interfaces = interfaces self._p = None self._disk = '/overlay.img' @@ -58,13 +60,21 @@ def start(self) -> None: '-serial', 'telnet:127.0.0.1:5000,server,nowait', ] - for i in range(self._interfaces): - with open(f'/sys/class/net/eth{i}/address', 'r') as f: + with open(f'/sys/class/net/eth0/address', 'r') as f: + mac = f.read().strip() + cmd.append('-device') + cmd.append(f'virtio-net-pci,netdev=hn0,mac={mac}') + cmd.append(f'-netdev') + cmd.append(f'tap,id=hn0,ifname=tap0,script=/mirror_tap_to_eth.sh,downscript=no') + + ifaces = get_ethernet_interfaces() + for i, iface in enumerate(ifaces, start=1): + with open(f'/sys/class/net/{iface}/address', 'r') as f: mac = f.read().strip() cmd.append('-device') cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac}') cmd.append(f'-netdev') - cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_eth.sh,downscript=no') + cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_front_panel.sh,downscript=no') self._p = subprocess.Popen(cmd) @@ -72,7 +82,7 @@ def wait(self) -> None: self._p.wait() -def initial_configuration(g: GuestFS) -> None: +def initial_configuration(g: GuestFS, hwsku: str) -> None: image = g.glob_expand('/image-*')[0] g.rm(image + 'platform/firsttime') @@ -105,31 +115,49 @@ def initial_configuration(g: GuestFS) -> None: g.ln_s(linkname=systemd_system + 'system-health.service', target='/dev/null') g.ln_s(linkname=systemd_system + 'watchdog-control.service', target='/dev/null') + sonic_share = image + 'rw/usr/share/sonic/' + hwsku_dir = image + 'rw' + VS_DEVICES_PATH + hwsku + g.mkdir_p(hwsku_dir) + + g.write(path=image + 'rw' + VS_DEVICES_PATH + 'default_sku', content=f'{hwsku} empty'.encode('utf-8')) + g.ln_s(linkname=sonic_share + 'hwsku', target=VS_DEVICES_PATH + hwsku) + g.ln_s(linkname=sonic_share + 'platform', target=VS_DEVICES_PATH) + + ifaces = get_ethernet_interfaces() + # The port_config.ini file contains the assignment of front panels to lanes. + port_config = parse_port_config() + # The lanemap.ini file is used by the virtual switch image to assign front panels to the Linux interfaces ethX. + # This assignment will later also be used by the script mirror_tap_to_front_panel.sh. + lanemap = create_lanemap(port_config, ifaces) + with open('/lanemap.ini', 'w') as f: + f.write('\n'.join(lanemap)) + + g.copy_in(localpath='/lanemap.ini', remotedir=hwsku_dir) + g.copy_in(localpath='/port_config.ini', remotedir=hwsku_dir) + etc_sonic = image + 'rw/etc/sonic/' g.mkdir_p(etc_sonic) sonic_version = image.removeprefix('/image-').removesuffix('/') sonic_environment = f''' SONIC_VERSION=${sonic_version} PLATFORM=x86_64-kvm_x86_64-r0 - HWSKU=Force10-S6000 + HWSKU={hwsku} DEVICE_TYPE=LeafRouter ASIC_TYPE=vs '''.encode('utf-8') g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment) - with open('/config_db.json') as f: - config_db = json.load(f) - - config_db['DEVICE_METADATA']['localhost']['hostname'] = socket.gethostname() - config_db['DEVICE_METADATA']['localhost']['mac'] = get_mac_address('eth0') - cidr = get_ip_address('eth0') + '/16' - config_db['MGMT_INTERFACE'] = { - f'eth0|{cidr}': { - 'gwaddr': get_default_gateway() + config_db = create_config_db(hwsku) + ports = {} + for iface in ifaces: + ports[iface] = { + **port_config[iface], + 'admin_status': 'up', + 'mtu': '9100' } - } - + config_db['PORT'] = ports config_db_json = json.dumps(config_db, indent=4, sort_keys=True) + g.write(path=etc_sonic + 'config_db.json', content=config_db_json.encode('utf-8')) if os.path.exists('/authorized_keys'): @@ -150,21 +178,22 @@ def main(): smp = os.getenv('QEMU_SMP', default='2') memory = os.getenv('QEMU_MEMORY', default='2048') interfaces = int(os.getenv('CLAB_INTFS', 0)) + 1 + hwsku = os.getenv('HWSKU', default='Accton-AS7726-32X') - vm = Qemu(name, smp, memory, interfaces) + vm = Qemu(name, smp, memory) logger.info('Prepare disk') vm.prepare_overlay(BASE_IMG) + logger.info(f'Waiting for {interfaces} interfaces to be connected') + wait_until_all_interfaces_are_connected(interfaces) + logger.info('Deploy initial config') g = vm.guestfs() - initial_configuration(g) + initial_configuration(g, hwsku) g.shutdown() g.close() - logger.info(f'Waiting for {interfaces} interfaces to be connected') - wait_until_all_interfaces_are_connected(interfaces) - logger.info('Start QEMU') vm.start() @@ -180,7 +209,7 @@ def wait_until_all_interfaces_are_connected(interfaces: int) -> None: while True: i = 0 for iface in os.listdir('/sys/class/net/'): - if iface.startswith('eth'): + if iface.startswith('eth') or iface.startswith('Ethernet'): i += 1 if i == interfaces: break @@ -218,5 +247,91 @@ def get_default_gateway() -> str: return socket.inet_ntoa(struct.pack(" list[str]: + ethernet_dirs = glob.glob('/sys/class/net/Ethernet*') + ifaces = [os.path.basename(dir) for dir in ethernet_dirs] + ifaces.sort() + return ifaces + + +def create_lanemap(port_config: dict[str, dict], ifaces: list[str]) -> list[str]: + lanemap = [] + + for i, iface in enumerate(ifaces, start=1): + lanes = port_config[iface]['lanes'] + lanemap.append(f'eth{i}:{lanes}') + + return lanemap + + +def parse_port_config() -> dict[str, dict]: + with open('/port_config.ini', 'r') as file: + lines = file.readlines() + + port_config = {} + + for line in lines[1:]: + columns = line.split() + name = columns[0] + port_config[name] = { + "lanes": columns[1], + "alias": columns[2], + "index": columns[3], + "speed": columns[4], + } + + return port_config + + +def create_config_db(hwsku: str) -> dict: + return { + 'AUTO_TECHSUPPORT': { + 'GLOBAL': { + 'state': 'disabled' + } + }, + 'DEVICE_METADATA': { + 'localhost': { + 'docker_routing_config_mode': 'split-unified', + 'hostname': socket.gethostname(), + 'hwsku': hwsku, + 'mac': get_mac_address('eth0'), + 'platform': 'x86_64-kvm_x86_64-r0', + 'type': 'LeafRouter', + } + }, + 'FEATURE': { + 'gnmi': { + 'state': 'disabled' + }, + 'mgmt-framework': { + 'state': 'disabled' + }, + 'snmp': { + 'state': 'disabled' + }, + 'teamd': { + 'state': 'disabled' + } + }, + 'MGMT_INTERFACE': { + f'eth0|{get_ip_address("eth0")}/16': { + 'gwaddr': get_default_gateway(), + } + }, + 'MGMT_PORT': { + 'eth0': { + 'alias': 'eth0', + 'admin_status': 'up' + } + }, + 'VERSIONS': { + 'DATABASE': { + 'VERSION': 'version_202311_03' + } + } + } + + if __name__ == '__main__': main() diff --git a/images/sonic/mirror_tap_to_front_panel.sh b/images/sonic/mirror_tap_to_front_panel.sh new file mode 100755 index 0000000..f348a22 --- /dev/null +++ b/images/sonic/mirror_tap_to_front_panel.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Script is taken from https://netdevops.me/2021/transparently-redirecting-packetsframes-between-interfaces/ +# Read it for better understanding + +TAP_IF=$1 +# get interface index number up to 3 digits (everything after first three chars) +# tap0 -> 0 +# tap123 -> 123 +INDEX=${TAP_IF:3:3} + +# tap$INDEX corresponds to eth$INDEX in the virtual machine +# The virtual switch assigns lanes to the Linux interface ethX. The assignment is specified in the lanemap.ini file in the following format: ethX:. +LANES=$(grep ^eth$INDEX: /lanemap.ini | cut -d':' -f2) +# Identify the front panel using the lanes. +FRONT_PANEL=$(grep -E "^Ethernet[0-9]+\s+$LANES\s+Eth" /port_config.ini | cut -d' ' -f1) + +ip link set $TAP_IF up +ip link set $TAP_IF mtu 65000 + +# create tc Ethernet<->tap redirect rules +tc qdisc add dev $FRONT_PANEL ingress +tc filter add dev $FRONT_PANEL parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TAP_IF + +tc qdisc add dev $TAP_IF ingress +tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $FRONT_PANEL diff --git a/images/sonic/port_config.ini b/images/sonic/port_config.ini new file mode 100644 index 0000000..acc1f3d --- /dev/null +++ b/images/sonic/port_config.ini @@ -0,0 +1,123 @@ +# name lanes alias index speed +Ethernet0 1 Eth1/1 1 25000 +Ethernet1 2 Eth1/2 1 25000 +Ethernet2 3 Eth1/3 1 25000 +Ethernet3 4 Eth1/4 1 25000 +Ethernet4 5 Eth2/1 2 25000 +Ethernet5 6 Eth2/2 2 25000 +Ethernet6 7 Eth2/3 2 25000 +Ethernet7 8 Eth2/4 2 25000 +Ethernet8 9 Eth3/1 3 25000 +Ethernet9 10 Eth3/2 3 25000 +Ethernet10 11 Eth3/3 3 25000 +Ethernet11 12 Eth3/4 3 25000 +Ethernet12 13 Eth4/1 4 25000 +Ethernet13 14 Eth4/2 4 25000 +Ethernet14 15 Eth4/3 4 25000 +Ethernet15 16 Eth4/4 4 25000 +Ethernet16 17 Eth5/1 5 25000 +Ethernet17 18 Eth5/2 5 25000 +Ethernet18 19 Eth5/3 5 25000 +Ethernet19 20 Eth5/4 5 25000 +Ethernet20 21 Eth6/1 6 25000 +Ethernet21 22 Eth6/2 6 25000 +Ethernet22 23 Eth6/3 6 25000 +Ethernet23 24 Eth6/4 6 25000 +Ethernet24 25 Eth7/1 7 25000 +Ethernet25 26 Eth7/2 7 25000 +Ethernet26 27 Eth7/3 7 25000 +Ethernet27 28 Eth7/4 7 25000 +Ethernet28 29 Eth8/1 8 25000 +Ethernet29 30 Eth8/2 8 25000 +Ethernet30 31 Eth8/3 8 25000 +Ethernet31 32 Eth8/4 8 25000 +Ethernet32 33 Eth9/1 9 25000 +Ethernet33 34 Eth9/2 9 25000 +Ethernet34 35 Eth9/3 9 25000 +Ethernet35 36 Eth9/4 9 25000 +Ethernet36 37 Eth10/1 10 25000 +Ethernet37 38 Eth10/2 10 25000 +Ethernet38 39 Eth10/3 10 25000 +Ethernet39 40 Eth10/4 10 25000 +Ethernet40 41 Eth11/1 11 25000 +Ethernet41 42 Eth11/2 11 25000 +Ethernet42 43 Eth11/3 11 25000 +Ethernet43 44 Eth11/4 11 25000 +Ethernet44 45 Eth12/1 12 25000 +Ethernet45 46 Eth12/2 12 25000 +Ethernet46 47 Eth12/3 12 25000 +Ethernet47 48 Eth12/4 12 25000 +Ethernet48 49 Eth13/1 13 25000 +Ethernet49 50 Eth13/2 13 25000 +Ethernet50 51 Eth13/3 13 25000 +Ethernet51 52 Eth13/4 13 25000 +Ethernet52 53 Eth14/1 14 25000 +Ethernet53 54 Eth14/2 14 25000 +Ethernet54 55 Eth14/3 14 25000 +Ethernet55 56 Eth14/4 14 25000 +Ethernet56 57 Eth15/1 15 25000 +Ethernet57 58 Eth15/2 15 25000 +Ethernet58 59 Eth15/3 15 25000 +Ethernet59 60 Eth15/4 15 25000 +Ethernet60 61 Eth16/1 16 25000 +Ethernet61 62 Eth16/2 16 25000 +Ethernet62 63 Eth16/3 16 25000 +Ethernet63 64 Eth16/4 16 25000 +Ethernet64 65 Eth17/1 17 25000 +Ethernet65 66 Eth17/2 17 25000 +Ethernet66 67 Eth17/3 17 25000 +Ethernet67 68 Eth17/4 17 25000 +Ethernet68 69 Eth18/1 18 25000 +Ethernet69 70 Eth18/2 18 25000 +Ethernet70 71 Eth18/3 18 25000 +Ethernet71 72 Eth18/4 18 25000 +Ethernet72 73 Eth19/1 19 25000 +Ethernet73 74 Eth19/2 19 25000 +Ethernet74 75 Eth19/3 19 25000 +Ethernet75 76 Eth19/4 19 25000 +Ethernet76 77 Eth20/1 20 25000 +Ethernet77 78 Eth20/2 20 25000 +Ethernet78 79 Eth20/3 20 25000 +Ethernet79 80 Eth20/4 20 25000 +Ethernet80 81 Eth21/1 21 25000 +Ethernet81 82 Eth21/2 21 25000 +Ethernet82 83 Eth21/3 21 25000 +Ethernet83 84 Eth21/4 21 25000 +Ethernet84 85 Eth22/1 22 25000 +Ethernet85 86 Eth22/2 22 25000 +Ethernet86 87 Eth22/3 22 25000 +Ethernet87 88 Eth22/4 22 25000 +Ethernet88 89 Eth23/1 23 25000 +Ethernet89 90 Eth23/2 23 25000 +Ethernet90 91 Eth23/3 23 25000 +Ethernet91 92 Eth23/4 23 25000 +Ethernet92 93 Eth24/1 24 25000 +Ethernet93 94 Eth24/2 24 25000 +Ethernet94 95 Eth24/3 24 25000 +Ethernet95 96 Eth24/4 24 25000 +Ethernet96 97 Eth25/1 25 25000 +Ethernet97 98 Eth25/2 25 25000 +Ethernet98 99 Eth25/3 25 25000 +Ethernet99 100 Eth25/4 25 25000 +Ethernet100 101 Eth26/1 26 25000 +Ethernet101 102 Eth26/2 26 25000 +Ethernet102 103 Eth26/3 26 25000 +Ethernet103 104 Eth26/4 26 25000 +Ethernet104 105 Eth27/1 27 25000 +Ethernet105 106 Eth27/2 27 25000 +Ethernet106 107 Eth27/3 27 25000 +Ethernet107 108 Eth27/4 27 25000 +Ethernet108 109 Eth28/1 28 25000 +Ethernet109 110 Eth28/2 28 25000 +Ethernet110 111 Eth28/3 28 25000 +Ethernet111 112 Eth28/4 28 25000 +Ethernet112 113 Eth29/1 29 25000 +Ethernet113 114 Eth29/2 29 25000 +Ethernet114 115 Eth29/3 29 25000 +Ethernet115 116 Eth29/4 29 25000 +Ethernet116 117 Eth30/1 30 25000 +Ethernet117 118 Eth30/2 30 25000 +Ethernet118 119 Eth30/3 30 25000 +Ethernet119 120 Eth30/4 30 25000 +Ethernet120 121,122,123,124 Eth31 31 100000 +Ethernet124 125,126,127,128 Eth32 32 100000 diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml index cc0f149..81e5372 100644 --- a/mini-lab.sonic.yaml +++ b/mini-lab.sonic.yaml @@ -51,9 +51,9 @@ topology: - endpoints: ["inet:ext", "mini_lab_ext:inet"] mtu: 9000 - endpoints: ["www:ext", "mini_lab_ext:www"] - - endpoints: ["leaf01:eth1", "vms:lan0"] - - endpoints: ["leaf02:eth1", "vms:lan1"] - - endpoints: ["leaf01:eth2", "vms:lan2"] - - endpoints: ["leaf02:eth2", "vms:lan3"] - - endpoints: ["leaf01:eth3", "inet:eth1"] - - endpoints: ["leaf02:eth3", "inet:eth2"] + - endpoints: ["leaf01:Ethernet0", "vms:lan0"] + - endpoints: ["leaf02:Ethernet0", "vms:lan1"] + - endpoints: ["leaf01:Ethernet1", "vms:lan2"] + - endpoints: ["leaf02:Ethernet1", "vms:lan3"] + - endpoints: ["leaf01:Ethernet120", "inet:eth1"] + - endpoints: ["leaf02:Ethernet120", "inet:eth2"]