Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use front panel names in the clab topology #206

Merged
merged 3 commits into from
Dec 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion images/sonic/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@ COPY --from=ghcr.io/metal-stack/mini-lab-sonic:base /frr-pythontools.deb /frr-py

ENTRYPOINT ["/launch.py"]

COPY config_db.json mirror_tap_to_eth.sh launch.py /
COPY mirror_tap_to_eth.sh mirror_tap_to_front_panel.sh port_config.ini launch.py /
69 changes: 0 additions & 69 deletions images/sonic/config_db.json

This file was deleted.

163 changes: 139 additions & 24 deletions images/sonic/launch.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#!/usr/bin/python3
import fcntl
import glob
import json
import logging
import os
Expand All @@ -15,13 +16,14 @@

BASE_IMG = '/sonic-vs.img'

VS_DEVICES_PATH = '/usr/share/sonic/device/x86_64-kvm_x86_64-r0/'


class Qemu:
def __init__(self, name: str, smp: str, memory: str, interfaces: int):
def __init__(self, name: str, smp: str, memory: str):
self._name = name
self._smp = smp
self._memory = memory
self._interfaces = interfaces
self._p = None
self._disk = '/overlay.img'

Expand Down Expand Up @@ -58,21 +60,29 @@ def start(self) -> None:
'-serial', 'telnet:127.0.0.1:5000,server,nowait',
]

for i in range(self._interfaces):
with open(f'/sys/class/net/eth{i}/address', 'r') as f:
with open(f'/sys/class/net/eth0/address', 'r') as f:
mac = f.read().strip()
cmd.append('-device')
cmd.append(f'virtio-net-pci,netdev=hn0,mac={mac}')
cmd.append(f'-netdev')
cmd.append(f'tap,id=hn0,ifname=tap0,script=/mirror_tap_to_eth.sh,downscript=no')

ifaces = get_ethernet_interfaces()
for i, iface in enumerate(ifaces, start=1):
with open(f'/sys/class/net/{iface}/address', 'r') as f:
mac = f.read().strip()
cmd.append('-device')
cmd.append(f'virtio-net-pci,netdev=hn{i},mac={mac}')
cmd.append(f'-netdev')
cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_eth.sh,downscript=no')
cmd.append(f'tap,id=hn{i},ifname=tap{i},script=/mirror_tap_to_front_panel.sh,downscript=no')

self._p = subprocess.Popen(cmd)

def wait(self) -> None:
self._p.wait()


def initial_configuration(g: GuestFS) -> None:
def initial_configuration(g: GuestFS, hwsku: str) -> None:
image = g.glob_expand('/image-*')[0]

g.rm(image + 'platform/firsttime')
Expand Down Expand Up @@ -105,31 +115,49 @@ def initial_configuration(g: GuestFS) -> None:
g.ln_s(linkname=systemd_system + 'system-health.service', target='/dev/null')
g.ln_s(linkname=systemd_system + 'watchdog-control.service', target='/dev/null')

sonic_share = image + 'rw/usr/share/sonic/'
hwsku_dir = image + 'rw' + VS_DEVICES_PATH + hwsku
g.mkdir_p(hwsku_dir)

g.write(path=image + 'rw' + VS_DEVICES_PATH + 'default_sku', content=f'{hwsku} empty'.encode('utf-8'))
g.ln_s(linkname=sonic_share + 'hwsku', target=VS_DEVICES_PATH + hwsku)
g.ln_s(linkname=sonic_share + 'platform', target=VS_DEVICES_PATH)

ifaces = get_ethernet_interfaces()
# The port_config.ini file contains the assignment of front panels to lanes.
port_config = parse_port_config()
# The lanemap.ini file is used by the virtual switch image to assign front panels to the Linux interfaces ethX.
# This assignment will later also be used by the script mirror_tap_to_front_panel.sh.
lanemap = create_lanemap(port_config, ifaces)
with open('/lanemap.ini', 'w') as f:
f.write('\n'.join(lanemap))

g.copy_in(localpath='/lanemap.ini', remotedir=hwsku_dir)
g.copy_in(localpath='/port_config.ini', remotedir=hwsku_dir)

etc_sonic = image + 'rw/etc/sonic/'
g.mkdir_p(etc_sonic)
sonic_version = image.removeprefix('/image-').removesuffix('/')
sonic_environment = f'''
SONIC_VERSION=${sonic_version}
PLATFORM=x86_64-kvm_x86_64-r0
HWSKU=Force10-S6000
HWSKU={hwsku}
DEVICE_TYPE=LeafRouter
ASIC_TYPE=vs
'''.encode('utf-8')
g.write(path=etc_sonic + 'sonic-environment', content=sonic_environment)

with open('/config_db.json') as f:
config_db = json.load(f)

config_db['DEVICE_METADATA']['localhost']['hostname'] = socket.gethostname()
config_db['DEVICE_METADATA']['localhost']['mac'] = get_mac_address('eth0')
cidr = get_ip_address('eth0') + '/16'
config_db['MGMT_INTERFACE'] = {
f'eth0|{cidr}': {
'gwaddr': get_default_gateway()
config_db = create_config_db(hwsku)
ports = {}
for iface in ifaces:
ports[iface] = {
**port_config[iface],
'admin_status': 'up',
'mtu': '9100'
}
}

config_db['PORT'] = ports
config_db_json = json.dumps(config_db, indent=4, sort_keys=True)

g.write(path=etc_sonic + 'config_db.json', content=config_db_json.encode('utf-8'))

if os.path.exists('/authorized_keys'):
Expand All @@ -150,21 +178,22 @@ def main():
smp = os.getenv('QEMU_SMP', default='2')
memory = os.getenv('QEMU_MEMORY', default='2048')
interfaces = int(os.getenv('CLAB_INTFS', 0)) + 1
hwsku = os.getenv('HWSKU', default='Accton-AS7726-32X')

vm = Qemu(name, smp, memory, interfaces)
vm = Qemu(name, smp, memory)

logger.info('Prepare disk')
vm.prepare_overlay(BASE_IMG)

logger.info(f'Waiting for {interfaces} interfaces to be connected')
wait_until_all_interfaces_are_connected(interfaces)

logger.info('Deploy initial config')
g = vm.guestfs()
initial_configuration(g)
initial_configuration(g, hwsku)
g.shutdown()
g.close()

logger.info(f'Waiting for {interfaces} interfaces to be connected')
wait_until_all_interfaces_are_connected(interfaces)

logger.info('Start QEMU')
vm.start()

Expand All @@ -180,7 +209,7 @@ def wait_until_all_interfaces_are_connected(interfaces: int) -> None:
while True:
i = 0
for iface in os.listdir('/sys/class/net/'):
if iface.startswith('eth'):
if iface.startswith('eth') or iface.startswith('Ethernet'):
i += 1
if i == interfaces:
break
Expand Down Expand Up @@ -218,5 +247,91 @@ def get_default_gateway() -> str:
return socket.inet_ntoa(struct.pack("<L", int(fields[2], 16)))


def get_ethernet_interfaces() -> list[str]:
ethernet_dirs = glob.glob('/sys/class/net/Ethernet*')
ifaces = [os.path.basename(dir) for dir in ethernet_dirs]
ifaces.sort()
return ifaces


def create_lanemap(port_config: dict[str, dict], ifaces: list[str]) -> list[str]:
lanemap = []

for i, iface in enumerate(ifaces, start=1):
lanes = port_config[iface]['lanes']
lanemap.append(f'eth{i}:{lanes}')

return lanemap


def parse_port_config() -> dict[str, dict]:
with open('/port_config.ini', 'r') as file:
lines = file.readlines()

port_config = {}

for line in lines[1:]:
columns = line.split()
name = columns[0]
port_config[name] = {
"lanes": columns[1],
"alias": columns[2],
"index": columns[3],
"speed": columns[4],
}

return port_config


def create_config_db(hwsku: str) -> dict:
return {
'AUTO_TECHSUPPORT': {
'GLOBAL': {
'state': 'disabled'
}
},
'DEVICE_METADATA': {
'localhost': {
'docker_routing_config_mode': 'split-unified',
'hostname': socket.gethostname(),
'hwsku': hwsku,
'mac': get_mac_address('eth0'),
'platform': 'x86_64-kvm_x86_64-r0',
'type': 'LeafRouter',
}
},
'FEATURE': {
'gnmi': {
'state': 'disabled'
},
'mgmt-framework': {
'state': 'disabled'
},
'snmp': {
'state': 'disabled'
},
'teamd': {
'state': 'disabled'
}
},
'MGMT_INTERFACE': {
f'eth0|{get_ip_address("eth0")}/16': {
'gwaddr': get_default_gateway(),
}
},
'MGMT_PORT': {
'eth0': {
'alias': 'eth0',
'admin_status': 'up'
}
},
'VERSIONS': {
'DATABASE': {
'VERSION': 'version_202311_03'
}
}
}


if __name__ == '__main__':
main()
26 changes: 26 additions & 0 deletions images/sonic/mirror_tap_to_front_panel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/bash

# Script is taken from https://netdevops.me/2021/transparently-redirecting-packetsframes-between-interfaces/
# Read it for better understanding

TAP_IF=$1
# get interface index number up to 3 digits (everything after first three chars)
# tap0 -> 0
# tap123 -> 123
INDEX=${TAP_IF:3:3}

# tap$INDEX corresponds to eth$INDEX in the virtual machine
# The virtual switch assigns lanes to the Linux interface ethX. The assignment is specified in the lanemap.ini file in the following format: ethX:<lanes>.
LANES=$(grep ^eth$INDEX: /lanemap.ini | cut -d':' -f2)
# Identify the front panel using the lanes.
FRONT_PANEL=$(grep -E "^Ethernet[0-9]+\s+$LANES\s+Eth" /port_config.ini | cut -d' ' -f1)

ip link set $TAP_IF up
ip link set $TAP_IF mtu 65000

# create tc Ethernet<->tap redirect rules
tc qdisc add dev $FRONT_PANEL ingress
tc filter add dev $FRONT_PANEL parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $TAP_IF

tc qdisc add dev $TAP_IF ingress
tc filter add dev $TAP_IF parent ffff: protocol all u32 match u8 0 0 action mirred egress redirect dev $FRONT_PANEL
Loading