EX-1476 Create step to create and populate ldev config

Create the step that will populate the ldev config.

Signed-off-by: johnsonw <[email protected]>

chroma_core/migrations/0028_configureldevjob.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2020-10-01 17:00
+from __future__ import unicode_literals
+
+import django.contrib.postgres.fields.jsonb
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("chroma_core", "0027_add_scan_mdt_jobs"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ConfigureLDevJob",
+            fields=[
+                (
+                    "job_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="chroma_core.Job",
+                    ),
+                ),
+                ("ldev_entries", django.contrib.postgres.fields.jsonb.JSONField(default={})),
+            ],
+            options={
+                "ordering": ["id"],
+            },
+            bases=("chroma_core.job",),
+        ),
+    ]

chroma_core/models/__init__.py

@@ -31,6 +31,7 @@
 from repo import *
 from copytool import *
 from client_mount import *
+from ldev_configuration import *
 from lnet_configuration import *
 from sparse_model import *
 from stratagem import *

chroma_core/models/ldev_configuration.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 DDN. All rights reserved.
+# Use of this source code is governed by a MIT-style
+# license that can be found in the LICENSE file.
+
+from chroma_core.lib.job import Step
+from chroma_core.models import Job
+from chroma_help.help import help_text
+from django.contrib.postgres import fields
+
+import json
+
+
+class ConfigureLDevStep(Step):
+    def run(self, kwargs):
+        ldev_entries = kwargs["ldev_entries"]
+        ldev_entries = json.loads(ldev_entries)
+
+        for (fqdn, entries) in ldev_entries.items():
+            self.invoke_rust_agent_expect_result(fqdn, "create_ldev_conf", entries)
+
+
+class ConfigureLDevJob(Job):
+    verb = "Configure LDev"
+    ldev_entries = fields.JSONField(default={})
+
+    class Meta:
+        app_label = "chroma_core"
+        ordering = ["id"]
+
+    @classmethod
+    def long_description(cls):
+        return help_text["configure_ldev"]
+
+    def description(self):
+        return self.long_description()
+
+    def get_steps(self):
+        return [(ConfigureLDevStep, {"ldev_entries": self.ldev_entries})]

chroma_help/help.py

@@ -118,6 +118,7 @@
     "local_diff": "This row has changed locally. Click to reset value to %(initial)s",
     "remote_diff": "This row has changed remotely. Click to set value to %(remote)s.",
     "configure_lnet": "Configure LNet for %s",
+    "configure_ldev": "Configure LDev",
     "change_host_profile": "Changing host %s to profile %s",
     "change_host_state": "Changing host %s to state %s",
     "configure_lnet_not_allowed": "LNet can only be configured in managed mode.",

iml-agent/src/action_plugins/ldev.rs

@@ -28,7 +28,11 @@ async fn read_ldev_config() -> Result<String, ImlAgentError> {
 }
 
 fn parse_entries(ldev_config: String) -> BTreeSet<LdevEntry> {
-    ldev_config.lines().map(LdevEntry::from).collect()
+    ldev_config
+        .lines()
+        .filter(|x| !x.trim_start().starts_with("#"))
+        .map(LdevEntry::from)
+        .collect()
 }
 
 fn convert(entries: &[LdevEntry]) -> String {
@@ -44,6 +48,7 @@ pub async fn create(entries: Vec<LdevEntry>) -> Result<(), ImlAgentError> {
         let ldev_config = read_ldev_config().await?;
         let existing_entries = parse_entries(ldev_config);
         let entries_set = entries.iter().cloned().collect::<BTreeSet<LdevEntry>>();
+
         if existing_entries != entries_set {
             let data = convert(&entries);
             write_to_file(data).await?;
@@ -63,7 +68,69 @@ mod tests {
     use iml_wire_types::FsType;
 
     #[test]
-    fn test_create() -> Result<(), ImlAgentError> {
+    fn test_ldiskfs_create() -> Result<(), ImlAgentError> {
+        let entries = vec![
+            LdevEntry {
+                primary: "mds1".into(),
+                failover: Some("mds2".into()),
+                label: "MGS".into(),
+                device: "/mnt/mgt".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+            LdevEntry {
+                primary: "mds1".into(),
+                failover: Some("mds2".into()),
+                label: "fs-MDT0000".into(),
+                device: "/mnt/mdt0".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+            LdevEntry {
+                primary: "mds2".into(),
+                failover: Some("mds1".into()),
+                label: "fs-MDT0001".into(),
+                device: "/mnt/mdt1".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+            LdevEntry {
+                primary: "oss1".into(),
+                failover: Some("oss2".into()),
+                label: "fs-OST0000".into(),
+                device: "/mnt/ost0".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+            LdevEntry {
+                primary: "oss2".into(),
+                failover: Some("oss1".into()),
+                label: "fs-OST0001".into(),
+                device: "/mnt/ost1".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+            LdevEntry {
+                primary: "oss1".into(),
+                failover: Some("oss2".into()),
+                label: "fs-OST0002".into(),
+                device: "/mnt/ost2".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+            LdevEntry {
+                primary: "oss2".into(),
+                failover: Some("oss1".into()),
+                label: "fs-OST0003".into(),
+                device: "/mnt/ost3".into(),
+                fs_type: FsType::Ldiskfs,
+            },
+        ]
+        .into_iter()
+        .collect::<Vec<LdevEntry>>();
+
+        let data = convert(&entries);
+        insta::assert_snapshot!(data);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_zfs_create() -> Result<(), ImlAgentError> {
         let entries = vec![
             LdevEntry {
                 primary: "mds1".into(),
@@ -208,21 +275,21 @@ mod tests {
             LdevEntry {
                 primary: "oss2".into(),
                 failover: Some("oss1".into()),
-                label: "zfsmo-OST00011".into(),
+                label: "zfsmo-OST0011".into(),
                 device: "zfs:ost17/ost17".into(),
                 fs_type: FsType::Zfs,
             },
             LdevEntry {
                 primary: "oss2".into(),
                 failover: Some("oss1".into()),
-                label: "zfsmo-OST00012".into(),
+                label: "zfsmo-OST0012".into(),
                 device: "zfs:ost18/ost18".into(),
                 fs_type: FsType::Zfs,
             },
             LdevEntry {
                 primary: "oss2".into(),
                 failover: Some("oss1".into()),
-                label: "zfsmo-OST00013".into(),
+                label: "zfsmo-OST0013".into(),
                 device: "zfs:ost19/ost19".into(),
                 fs_type: FsType::Zfs,
             },
@@ -262,4 +329,50 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn test_parsing_commented_data() -> Result<(), ImlAgentError> {
+        let content: String = r#"# example /etc/ldev.conf
+#
+#local  foreign/-  label       [md|zfs:]device-path   [journal-path]/- [raidtab]
+#
+#zeno-mds1 - zeno-MDT0000 zfs:lustre-zeno-mds1/mdt1
+#
+#zeno1 zeno5 zeno-OST0000 zfs:lustre-zeno1/ost1
+#zeno2 zeno6 zeno-OST0001 zfs:lustre-zeno2/ost1
+#zeno3 zeno7 zeno-OST0002 zfs:lustre-zeno3/ost1
+#zeno4 zeno8 zeno-OST0003 zfs:lustre-zeno4/ost1
+#zeno5 zeno1 zeno-OST0004 zfs:lustre-zeno5/ost1
+#zeno6 zeno2 zeno-OST0005 zfs:lustre-zeno6/ost1
+#zeno7 zeno3 zeno-OST0006 zfs:lus tre-zeno7/ost1
+#zeno8 zeno4 zeno-OST0007 zfs:lustre-zeno8/ost1"#
+            .into();
+
+        let data: BTreeSet<LdevEntry> = parse_entries(content);
+
+        assert!(data.is_empty());
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_parsing_data() -> Result<(), ImlAgentError> {
+        let content: String = r#"#zeno-mds1 - zeno-MDT0000 zfs:lustre-zeno-mds1/mdt1
+# Random comment
+zeno1 zeno5 zeno-OST0000 zfs:lustre-zeno1/ost1
+zeno2 - zeno-OST0001 zfs:lustre-zeno2/ost1
+zeno3 zeno7 zeno-OST0002 zfs:lustre-zeno3/ost1
+zeno4 zeno8 zeno-OST0003 zfs:lustre-zeno4/ost1
+zeno5 zeno1 zeno-OST0004 zfs:lustre-zeno5/ost1
+zeno6 - zeno-OST0005 zfs:lustre-zeno6/ost1
+zeno7 zeno3 zeno-OST0006 zfs:lustre-zeno7/ost1
+zeno8 zeno4 zeno-OST0007 zfs:lustre-zeno8/ost1"#
+            .into();
+
+        let data: BTreeSet<LdevEntry> = parse_entries(content);
+
+        insta::assert_debug_snapshot!(data);
+
+        Ok(())
+    }
 }

iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__ldiskfs_create.snap

@@ -0,0 +1,11 @@
+---
+source: iml-agent/src/action_plugins/ldev.rs
+expression: data
+---
+mds1 mds2 MGS ldiskfs:/mnt/mgt
+mds1 mds2 fs-MDT0000 ldiskfs:/mnt/mdt0
+mds2 mds1 fs-MDT0001 ldiskfs:/mnt/mdt1
+oss1 oss2 fs-OST0000 ldiskfs:/mnt/ost0
+oss2 oss1 fs-OST0001 ldiskfs:/mnt/ost1
+oss1 oss2 fs-OST0002 ldiskfs:/mnt/ost2
+oss2 oss1 fs-OST0003 ldiskfs:/mnt/ost3

iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__parsing_data.snap

@@ -0,0 +1,74 @@
+---
+source: iml-agent/src/action_plugins/ldev.rs
+expression: data
+---
+{
+    LdevEntry {
+        primary: "zeno1",
+        failover: Some(
+            "zeno5",
+        ),
+        label: "zeno-OST0000",
+        device: "zfs:lustre-zeno1/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno2",
+        failover: None,
+        label: "zeno-OST0001",
+        device: "zfs:lustre-zeno2/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno3",
+        failover: Some(
+            "zeno7",
+        ),
+        label: "zeno-OST0002",
+        device: "zfs:lustre-zeno3/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno4",
+        failover: Some(
+            "zeno8",
+        ),
+        label: "zeno-OST0003",
+        device: "zfs:lustre-zeno4/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno5",
+        failover: Some(
+            "zeno1",
+        ),
+        label: "zeno-OST0004",
+        device: "zfs:lustre-zeno5/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno6",
+        failover: None,
+        label: "zeno-OST0005",
+        device: "zfs:lustre-zeno6/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno7",
+        failover: Some(
+            "zeno3",
+        ),
+        label: "zeno-OST0006",
+        device: "zfs:lustre-zeno7/ost1",
+        fs_type: Zfs,
+    },
+    LdevEntry {
+        primary: "zeno8",
+        failover: Some(
+            "zeno4",
+        ),
+        label: "zeno-OST0007",
+        device: "zfs:lustre-zeno8/ost1",
+        fs_type: Zfs,
+    },
+}

iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__create.snap → iml-agent/src/action_plugins/snapshots/iml_agent__action_plugins__ldev__tests__zfs_create.snap

@@ -22,6 +22,6 @@ oss2 oss1 zfsmo-OST000d zfs:ost13/ost13
 oss2 oss1 zfsmo-OST000e zfs:ost14/ost14
 oss2 oss1 zfsmo-OST000f zfs:ost15/ost15
 oss2 oss1 zfsmo-OST0010 zfs:ost16/ost16
-oss2 oss1 zfsmo-OST00011 zfs:ost17/ost17
-oss2 oss1 zfsmo-OST00012 zfs:ost18/ost18
-oss2 oss1 zfsmo-OST00013 zfs:ost19/ost19
+oss2 oss1 zfsmo-OST0011 zfs:ost17/ost17
+oss2 oss1 zfsmo-OST0012 zfs:ost18/ost18
+oss2 oss1 zfsmo-OST0013 zfs:ost19/ost19

iml-api/src/error.rs

@@ -30,6 +30,8 @@ pub enum ImlApiError {
     FilesystemNotFound,
     #[error("Filesystem Not Found")]
     MgsNotFound,
+    #[error("All targets must be mounted when configuring ldev conf.")]
+    TargetsNotMounted,
 }
 
 impl reject::Reject for ImlApiError {}