From 5815c82cc062ac0f59c6be8840ebfaa7085d470f Mon Sep 17 00:00:00 2001
From: Bruno Travouillon <bruno.travouillon@mila.quebec>
Date: Tue, 2 Aug 2022 16:42:02 -0400
Subject: [PATCH] feat: Add new parameter infiniband_ipoib_enhanced

The role was enforcing Connected mode on RHEL-like, without disabling
ipoib_enhanced, which led to errors.

Introduce a new parameter `infiniband_ipoib_enhanced` which only has
effect on RHEl-like OS. If set to False, the IPoIB interfaces will
default to Connected mode (previous behavior). If set to True (default),
the IPoIB interfaces will default to Datagram mode and Enhanced IPoIB.

For ConnectX-4 and later, Mellanox recommends to keep Enhanced IPoIB
enabled.
---
 defaults/main.yml          | 6 ++++++
 handlers/main.yml          | 5 +++++
 tasks/configure-RedHat.yml | 6 ++++++
 templates/ifcfg-ib.j2      | 2 ++
 4 files changed, 19 insertions(+)

diff --git a/defaults/main.yml b/defaults/main.yml
index ec22aae..61324b1 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -69,3 +69,9 @@ infiniband_install_kernel_modules: True
 # given VM could run.
 #
 infiniband_guid_prefix: "4d:69:6c:61:00"
+
+# Configure Enhanced IPoIB
+# For ConnectX-4 and later, Mellanox recommends to keep Enhanced IPoIB enabled.
+# This parameter only has effect on RHEl-like OS. If set to False, the IPoIB
+# interfaces will default to Connected mode.
+infiniband_ipoib_enhanced: True
diff --git a/handlers/main.yml b/handlers/main.yml
index 51f12db..2123711 100644
--- a/handlers/main.yml
+++ b/handlers/main.yml
@@ -11,6 +11,11 @@
     name: openibd.service
     state: started
 
+- name: Restart openibd
+  ansible.builtin.systemd:
+    name: openibd.service
+    state: restarted
+
 - name: Reboot to load new configurations
   throttle: 1
   ansible.builtin.reboot:
diff --git a/tasks/configure-RedHat.yml b/tasks/configure-RedHat.yml
index 808d90b..4a26f14 100644
--- a/tasks/configure-RedHat.yml
+++ b/tasks/configure-RedHat.yml
@@ -9,3 +9,9 @@
     label: "Configure {{ item.iface }}"
   notify: Restart interface
   register: configure_ipoib
+
+- name: Configure ipoib_enhanced
+  ansible.builtin.lineinfile:
+    path: /etc/modprobe.d/ib_ipoib_enhanced.conf
+    line: "options ib_ipoib ipoib_enhanced={{ infiniband_ipoib_enhanced | bool }}"
+  notify: Restart openibd
diff --git a/templates/ifcfg-ib.j2 b/templates/ifcfg-ib.j2
index 08e7087..2c5719e 100644
--- a/templates/ifcfg-ib.j2
+++ b/templates/ifcfg-ib.j2
@@ -18,5 +18,7 @@ BOOTPROTO=static
 ONBOOT=yes
 IPV4_FAILURE_FATAL=yes
 IPV6INIT=no
+{% if not infiniband_ipoib_enhanced %}
 MTU=65520
 CONNECTED_MODE=yes
+{% endif %}