resource: ubridge: separate remove events from regular scans

Instead of using the regular scan phases for removes, use a separate
set, with just the same initialization and cleanup as a regular scan.
The main remove is done in two phases. The first calls the new module
function, scan_remove for the appropriate type and block modules.
These are currently all stubs. The second phase removes the core SID
kvstore values. Currently this just removes the device from the
device hierarchy in the kvstore.

src/include/resource/ucmd-module.h

@@ -43,6 +43,8 @@ extern "C" {
 #define SID_UCMD_MOD_FN_NAME_SCAN_POST_CURRENT "sid_ucmd_scan_post_current"
 #define SID_UCMD_MOD_FN_NAME_SCAN_POST_NEXT    "sid_ucmd_scan_post_next"
 
+#define SID_UCMD_MOD_FN_NAME_SCAN_REMOVE       "sid_ucmd_scan_remove"
+
 #define SID_UCMD_MOD_FN_NAME_ERROR             "sid_ucmd_error"
 
 struct sid_ucmd_common_ctx;
@@ -62,6 +64,7 @@ struct sid_ucmd_mod_fns {
 	sid_ucmd_fn_t *scan_post_next;
 	sid_ucmd_fn_t *trigger_action_current;
 	sid_ucmd_fn_t *trigger_action_next;
+	sid_ucmd_fn_t *scan_remove;
 	sid_ucmd_fn_t *error;
 } __packed;
 
@@ -123,6 +126,7 @@ struct sid_ucmd_mod_fns {
 #define SID_UCMD_SCAN_POST_NEXT(fn)         SID_UCMD_FN(scan_post_next, _SID_UCMD_FN_CHECK_TYPE(fn))
 #define SID_UCMD_TRIGGER_ACTION_CURRENT(fn) SID_UCMD_FN(trigger_action_current, _SID_UCMD_FN_CHECK_TYPE(fn))
 #define SID_UCMD_TRIGGER_ACTION_NEXT(fn)    SID_UCMD_FN(trigger_action_next, _SID_UCMD_FN_CHECK_TYPE(fn))
+#define SID_UCMD_SCAN_REMOVE(fn)            SID_UCMD_FN(scan_emove, _SID_UCMD_FN_CHECK_TYPE(fn))
 #define SID_UCMD_ERROR(fn)                  SID_UCMD_FN(error, _SID_UCMD_FN_CHECK_TYPE(fn))
 
 /*

src/modules/ucmd/block/blkid/blkid.c

@@ -297,6 +297,13 @@ static int _blkid_scan_next(struct module *module, struct sid_ucmd_ctx *ucmd_ctx
 }
 SID_UCMD_SCAN_NEXT(_blkid_scan_next)
 
+static int _blkid_scan_remove(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
+{
+	log_debug(MID, "scan-remove");
+	return 0;
+}
+SID_UCMD_SCAN_REMOVE(_blkid_scan_remove)
+
 static int _blkid_error(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
 {
 	log_debug(MID, "error");

src/modules/ucmd/block/dm_mpath/dm_mpath.c

@@ -171,6 +171,13 @@ static int _dm_mpath_scan_next(struct module *module, struct sid_ucmd_ctx *ucmd_
 }
 SID_UCMD_SCAN_NEXT(_dm_mpath_scan_next)
 
+static int _dm_mpath_scan_remove(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
+{
+	log_debug(MID, "scan-remove");
+	return 0;
+}
+SID_UCMD_SCAN_REMOVE(_dm_mpath_scan_remove)
+
 static int _dm_mpath_error(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
 {
 	log_debug(MID, "error");

src/modules/ucmd/block/dummy_block/dummy_block.c

@@ -101,6 +101,13 @@ static int _dummy_block_trigger_action_next(struct module *module, struct sid_uc
 }
 SID_UCMD_TRIGGER_ACTION_NEXT(_dummy_block_trigger_action_next)
 
+static int _dummy_block_scan_remove(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
+{
+	log_debug(MID, "scan-remove");
+	return 0;
+}
+SID_UCMD_SCAN_REMOVE(_dummy_block_scan_remove)
+
 static int _dummy_block_error(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
 {
 	log_debug(MID, "error");

src/modules/ucmd/type/dm/dm.c

@@ -68,6 +68,10 @@ static struct module_symbol_params dm_submod_symbol_params[] = {
 		SID_UCMD_MOD_FN_NAME_SCAN_POST_NEXT,
 		MODULE_SYMBOL_INDIRECT,
 	},
+	{
+		SID_UCMD_MOD_FN_NAME_SCAN_REMOVE,
+		MODULE_SYMBOL_INDIRECT,
+	},
 	NULL_MODULE_SYMBOL_PARAMS,
 };
 
@@ -79,6 +83,7 @@ struct dm_submod_fns {
 	sid_ucmd_fn_t *scan_next;
 	sid_ucmd_fn_t *scan_post_current;
 	sid_ucmd_fn_t *scan_post_next;
+	sid_ucmd_fn_t *scan_remove;
 } __packed;
 
 struct dm_mod_ctx {
@@ -356,6 +361,26 @@ static int _dm_scan_post_next(struct module *module, struct sid_ucmd_ctx *ucmd_c
 }
 SID_UCMD_SCAN_POST_NEXT(_dm_scan_post_next)
 
+static int _dm_scan_remove(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
+{
+	struct dm_mod_ctx    *dm_mod;
+	struct dm_submod_fns *submod_fns;
+
+	log_debug(DM_ID, "scan-remove");
+
+	dm_mod = module_get_data(module);
+
+	if (!dm_mod->submod_res_current)
+		return 0;
+
+	module_registry_get_module_symbols(dm_mod->submod_res_current, (const void ***) &submod_fns);
+	if (submod_fns && submod_fns->scan_remove)
+		(void) submod_fns->scan_remove(sid_resource_get_data(dm_mod->submod_res_current), ucmd_ctx);
+
+	return 0;
+}
+SID_UCMD_SCAN_REMOVE(_dm_scan_remove)
+
 static int _dm_error(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
 {
 	log_debug(DM_ID, "error");

src/modules/ucmd/type/dm/lvm/lvm.c

@@ -103,3 +103,10 @@ static int _lvm_scan_post_next(struct module *module, struct sid_ucmd_ctx *ucmd_
 	return 0;
 }
 SID_UCMD_SCAN_POST_NEXT(_lvm_scan_post_next)
+
+static int _lvm_scan_remove(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
+{
+	log_debug(LVM_ID, "scan-remove");
+	return 0;
+}
+SID_UCMD_SCAN_REMOVE(_lvm_scan_remove)

src/modules/ucmd/type/dummy_type/dummy_type.c

@@ -101,6 +101,13 @@ static int _dummy_type_trigger_action_next(struct module *module, struct sid_ucm
 }
 SID_UCMD_TRIGGER_ACTION_NEXT(_dummy_type_trigger_action_next)
 
+static int _dummy_type_scan_remove(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
+{
+	log_debug(MID, "scan-remove");
+	return 0;
+}
+SID_UCMD_SCAN_REMOVE(_dummy_type_scan_remove)
+
 static int _dummy_type_error(struct module *module, struct sid_ucmd_ctx *ucmd_ctx)
 {
 	log_debug(MID, "error");

src/resource/ubridge.c

@@ -156,13 +156,19 @@ typedef enum {
 	CMD_SCAN_PHASE_A_SCAN_POST_CURRENT, /* module */
 	CMD_SCAN_PHASE_A_SCAN_POST_NEXT,    /* module */
 	CMD_SCAN_PHASE_A_WAITING,           /* core waits for confirmation */
-	CMD_SCAN_PHASE_A_EXIT,              /* core exits phase "A" */
+	CMD_SCAN_PHASE_A_EXIT,              /* core finishes phase "A" */
+	CMD_SCAN_PHASE_A_CLEANUP,           /* core exits phase "A" */
 
 	CMD_SCAN_PHASE_B_TRIGGER_ACTION_CURRENT,
 	__CMD_SCAN_PHASE_B_TRIGGER_ACTION_START = CMD_SCAN_PHASE_B_TRIGGER_ACTION_CURRENT,
 	CMD_SCAN_PHASE_B_TRIGGER_ACTION_NEXT,
 	__CMD_SCAN_PHASE_B_TRIGGER_ACTION_END = CMD_SCAN_PHASE_B_TRIGGER_ACTION_NEXT,
 
+	CMD_SCAN_PHASE_REMOVE_INIT,
+	CMD_SCAN_PHASE_REMOVE_MODS,
+	CMD_SCAN_PHASE_REMOVE_CORE,
+	CMD_SCAN_PHASE_REMOVE_CLEANUP,
+
 	CMD_SCAN_PHASE_ERROR,
 } cmd_scan_phase_t;
 
@@ -4072,6 +4078,10 @@ static int _execute_block_modules(struct cmd_exec_arg *exec_arg, cmd_scan_phase_
 				    block_mod_fns->trigger_action_next(block_mod, ucmd_ctx) < 0)
 					goto out;
 				break;
+			case CMD_SCAN_PHASE_REMOVE_MODS:
+				if (block_mod_fns->scan_remove && block_mod_fns->scan_remove(block_mod, ucmd_ctx) < 0)
+					goto out;
+				break;
 			case CMD_SCAN_PHASE_ERROR:
 				if (block_mod_fns->error && block_mod_fns->error(block_mod, ucmd_ctx) < 0)
 					goto out;
@@ -4090,6 +4100,25 @@ static int _execute_block_modules(struct cmd_exec_arg *exec_arg, cmd_scan_phase_
 	return r;
 }
 
+static int _get_device_uuid(sid_resource_t *cmd_res)
+{
+	struct sid_ucmd_ctx *ucmd_ctx = sid_resource_get_data(cmd_res);
+	char                 buf[UTIL_UUID_STR_SIZE]; /* used for both uuid and diskseq */
+	const char          *uuid_p;
+
+	if (!(uuid_p = _do_sid_ucmd_get_kv(NULL, ucmd_ctx, NULL, KV_NS_UDEV, KV_KEY_UDEV_SID_DEV_ID, NULL, NULL, 0)) &&
+	    !(uuid_p = _devno_to_devid(ucmd_ctx, ucmd_ctx->req_env.dev.num_s, buf, sizeof(buf)))) {
+		/* SID doesn't appera to have a record of this device */
+		log_error(ID(cmd_res),
+		          "Couldn't find UUID for device " CMD_DEV_NAME_NUM_FMT ".",
+		          CMD_DEV_NAME_NUM(ucmd_ctx));
+		return -1;
+	}
+	ucmd_ctx->req_env.dev.uid_s = strdup(uuid_p);
+
+	return 0;
+}
+
 static int _set_device_kv_records(sid_resource_t *cmd_res)
 {
 	struct sid_ucmd_ctx *ucmd_ctx = sid_resource_get_data(cmd_res);
@@ -4184,7 +4213,10 @@ static int _cmd_exec_scan_init(struct cmd_exec_arg *exec_arg)
 		goto fail;
 	}
 
-	if (_set_device_kv_records(exec_arg->cmd_res) < 0)
+	if (ucmd_ctx->scan.phase == CMD_SCAN_PHASE_REMOVE_INIT) {
+		if (_get_device_uuid(exec_arg->cmd_res) < 0)
+			goto fail;
+	} else if (_set_device_kv_records(exec_arg->cmd_res) < 0)
 		goto fail;
 
 	return 0;
@@ -4354,8 +4386,13 @@ static int _cmd_exec_scan_exit(struct cmd_exec_arg *exec_arg)
 	struct sid_ucmd_ctx *ucmd_ctx = sid_resource_get_data(exec_arg->cmd_res);
 
 	if (sid_ucmd_dev_get_ready(NULL, ucmd_ctx, 0) == DEV_RDY_UNPROCESSED)
-		sid_ucmd_dev_set_ready(NULL, ucmd_ctx, DEV_RDY_PUBLIC);
+		return sid_ucmd_dev_set_ready(NULL, ucmd_ctx, DEV_RDY_PUBLIC);
 
+	return 0;
+}
+
+static int _cmd_exec_scan_cleanup(struct cmd_exec_arg *exec_arg)
+{
 	if (exec_arg->block_mod_iter) {
 		sid_resource_iter_destroy(exec_arg->block_mod_iter);
 		exec_arg->block_mod_iter = NULL;
@@ -4374,6 +4411,38 @@ static int _cmd_exec_trigger_action_next(struct cmd_exec_arg *exec_arg)
 	return 0;
 }
 
+static int _cmd_exec_scan_remove_mods(struct cmd_exec_arg *exec_arg)
+{
+	struct sid_ucmd_ctx           *ucmd_ctx = sid_resource_get_data(exec_arg->cmd_res);
+	const char                    *mod_name;
+	const struct sid_ucmd_mod_fns *mod_fns;
+
+	_execute_block_modules(exec_arg, CMD_SCAN_PHASE_REMOVE_MODS);
+
+	if (!(mod_name = _do_sid_ucmd_get_kv(NULL, ucmd_ctx, NULL, KV_NS_DEVICE, KV_KEY_DEV_MOD, NULL, NULL, 0))) {
+		log_error(ID(exec_arg->cmd_res),
+		          "Failed to find device " CMD_DEV_NAME_NUM_FMT " module name",
+		          CMD_DEV_NAME_NUM(ucmd_ctx));
+		return -1;
+	}
+
+	if (!(exec_arg->type_mod_res_current = module_registry_get_module(exec_arg->type_mod_registry_res, mod_name))) {
+		log_debug(ID(exec_arg->cmd_res), "Module %s not loaded.", mod_name);
+		return 0;
+	}
+
+	module_registry_get_module_symbols(exec_arg->type_mod_res_current, (const void ***) &mod_fns);
+	if (mod_fns && mod_fns->scan_remove)
+		return mod_fns->scan_remove(sid_resource_get_data(exec_arg->type_mod_res_current), ucmd_ctx);
+
+	return 0;
+}
+
+static int _cmd_exec_scan_remove_core(struct cmd_exec_arg *exec_arg)
+{
+	return _refresh_device_hierarchy_from_sysfs(exec_arg->cmd_res);
+}
+
 static int _cmd_exec_scan_error(struct cmd_exec_arg *exec_arg)
 {
 	struct sid_ucmd_ctx           *ucmd_ctx = sid_resource_get_data(exec_arg->cmd_res);
@@ -4416,29 +4485,47 @@ static struct cmd_reg _cmd_scan_phase_regs[] = {
 
 	[CMD_SCAN_PHASE_A_EXIT]              = {.name = "exit", .flags = CMD_SCAN_CAP_ALL, .exec = _cmd_exec_scan_exit},
 
+	[CMD_SCAN_PHASE_A_CLEANUP]           = {.name = "cleanup", .flags = 0, .exec = _cmd_exec_scan_cleanup},
+
 	[CMD_SCAN_PHASE_B_TRIGGER_ACTION_CURRENT] = {.name  = "trigger-action-current",
                                                      .flags = 0,
                                                      .exec  = _cmd_exec_trigger_action_current},
 
 	[CMD_SCAN_PHASE_B_TRIGGER_ACTION_NEXT] = {.name = "trigger-action-next", .flags = 0, .exec = _cmd_exec_trigger_action_next},
 
+	[CMD_SCAN_PHASE_REMOVE_INIT]           = {.name = "remove-init", .flags = 0, .exec = _cmd_exec_scan_init},
+
+	[CMD_SCAN_PHASE_REMOVE_MODS]           = {.name = "remove-mods", .flags = 0, .exec = _cmd_exec_scan_remove_mods},
+
+	[CMD_SCAN_PHASE_REMOVE_CORE]           = {.name = "remove-core", .flags = 0, .exec = _cmd_exec_scan_remove_core},
+
+	[CMD_SCAN_PHASE_REMOVE_CLEANUP]        = {.name = "remove-cleanup", .flags = 0, .exec = _cmd_exec_scan_cleanup},
+
 	[CMD_SCAN_PHASE_ERROR]                 = {.name = "error", .flags = 0, .exec = _cmd_exec_scan_error},
 };
 
 static int _cmd_exec_scan(struct cmd_exec_arg *exec_arg)
 {
 	struct sid_ucmd_ctx *ucmd_ctx = sid_resource_get_data(exec_arg->cmd_res);
-	cmd_scan_phase_t     phase;
+	cmd_scan_phase_t     phase, phase_start, phase_end;
+
+	if (ucmd_ctx->req_env.dev.udev.action == UDEV_ACTION_REMOVE) {
+		phase_start = CMD_SCAN_PHASE_REMOVE_INIT;
+		phase_end   = CMD_SCAN_PHASE_REMOVE_CLEANUP;
+	} else {
+		phase_start = CMD_SCAN_PHASE_A_INIT;
+		phase_end   = CMD_SCAN_PHASE_A_CLEANUP;
+	}
 
-	for (phase = CMD_SCAN_PHASE_A_INIT; phase <= CMD_SCAN_PHASE_A_EXIT; phase++) {
+	for (phase = phase_start; phase <= phase_end; phase++) {
 		log_debug(ID(exec_arg->cmd_res), "Executing %s phase.", _cmd_scan_phase_regs[phase].name);
 		ucmd_ctx->scan.phase = phase;
 
 		if (_cmd_scan_phase_regs[phase].exec(exec_arg) < 0) {
 			log_error(ID(exec_arg->cmd_res), "%s phase failed.", _cmd_scan_phase_regs[phase].name);
 
-			/* if init or exit phase fails, there's nothing else we can do */
-			if (phase == CMD_SCAN_PHASE_A_INIT || phase == CMD_SCAN_PHASE_A_EXIT)
+			/* if init or cleanup phase fails, there's nothing else we can do */
+			if (phase == CMD_SCAN_PHASE_A_INIT || phase == CMD_SCAN_PHASE_A_CLEANUP)
 				return -1;
 
 			/* otherwise, call out modules to handle the error case */
@@ -6016,6 +6103,10 @@ static struct module_symbol_params block_symbol_params[]                  = {{
                                                                              {
                                                                     SID_UCMD_MOD_FN_NAME_TRIGGER_ACTION_NEXT,
                                                                     MODULE_SYMBOL_INDIRECT,
+                                                            },
+                                                                             {
+                                                                    SID_UCMD_MOD_FN_NAME_SCAN_REMOVE,
+                                                                    MODULE_SYMBOL_INDIRECT,
                                                             },
                                                                              {
                                                                     SID_UCMD_MOD_FN_NAME_ERROR,
@@ -6054,6 +6145,10 @@ static struct module_symbol_params type_symbol_params[]                   = {{
                                                                              {
                                                                    SID_UCMD_MOD_FN_NAME_TRIGGER_ACTION_NEXT,
                                                                    MODULE_SYMBOL_INDIRECT,
+                                                           },
+                                                                             {
+                                                                   SID_UCMD_MOD_FN_NAME_SCAN_REMOVE,
+                                                                   MODULE_SYMBOL_INDIRECT,
                                                            },
                                                                              {
                                                                    SID_UCMD_MOD_FN_NAME_ERROR,