Support mulitple remotehost endpoints for one-tool

- Engines which run tools need to have a more specific label, one that
  includes th endpoint-label, to avoid duplicate labels like
  "profiler-1"
- Logic for adding RB followers was a bit broken, only looking at the
  first RB message with new followers.
- There is still work to do to avoid duplicate tool collection when
  more than 1 remotehost endpoint use the sme host (one could argue to
  not do this, but I'd like it to just avoid duplicte tools in case
  someone does).

endpoints/remotehost/remotehost

@@ -388,7 +388,7 @@ function launch_osruntime() {
     echo "Going to create a pod for each tool"
     for tool in $tools; do
         # This label will not work with mulitple remotehost endpoints!
-        engine_label=profiler-$tool_count
+        engine_label=profiler-$endpoint_label-$tool_count
         exec_pod $engine_label $tool
         new_remotehost_followers+=" $engine_label"
         let tool_count=$tool_count+1

engine/engine-script-library

@@ -302,11 +302,14 @@ function validate_core_env() {
     if [ -z "$cs_label" ]; then
         exit_error "The client/server label (--cs-label) was not defined"
     fi
-    regex='^\w+-\d+(-\w+){0,1}$'
-    if echo $cs_label | grep -q -P $regex; then
+    regex1='^profiler-\w+-\d+-\w+$'
+    regex2='^\w+-\d+$'
+    if echo $cs_label | grep -q -P $regex1; then
+        echo "engine-label \"$cs_label\" is valid and runs this tool:  $tool_name"
+    elif echo $cs_label | grep -q -P $regex2; then
         echo "engine-label \"$cs_label\" is valid"
     else
-        exit_error 'cs_label "'$cs_label'" does not adhere to regex '$regex
+        exit_error 'cs_label "'$cs_label'" is not valid'
     fi
 
     if [ -z "${max_rb_attempts}" ]; then

rickshaw-run

@@ -2161,12 +2161,12 @@ sub process_roadblocks() {
     if (defined $messages_ref) {
         foreach my $msg (@{ $$messages_ref{'received'} })  {
             if (exists $$msg{'payload'}{'message'}{'user-object'}{'new-followers'}) {
-                @new_followers = @{ $$msg{'payload'}{'message'}{'user-object'}{'new-followers'} };
-                printf "Endpoint created followers: %s\n", join(" ", sort(@new_followers));
-                last;
+                debug_log(sprintf "found these new followers: %s\n", join(" ", @{ $$msg{'payload'}{'message'}{'user-object'}{'new-followers'} }));
+                @new_followers = (@new_followers, @{ $$msg{'payload'}{'message'}{'user-object'}{'new-followers'} });
             }
         }
     }
+    debug_log(sprintf "All new followers: %s\n", join(" ", @new_followers));
 
     $roadblock_rc = do_roadblock("endpoint-deploy-end", $endpoint_deploy_timeout, $messages_ref, @active_followers);
     roadblock_exit_on_error($roadblock_rc);
@@ -2237,8 +2237,13 @@ sub organize_run_data() {
         mkdir($tmp_data_dir);
         my $pushd_dir = pushd($tmp_data_dir);
         foreach my $archive (dir_entries($engine_archives_dir,
-                                        qr/^(\w+)-(\d+)-data\.tgz$/)) {
-            $archive =~ /^(\w+)-(\d+)-data\.tgz$/ or next;
+                                        qr/^(\w+)-(.+)-data\.tgz$/)) {
+            if ($archive =~ /^(\w+)-(.+)-data\.tgz$/) {
+                debug_log("found archive $archive");
+            } else {
+                printf "archive %s is not recognized\n";
+                next;
+            }
             my $archive_full_path = $engine_archives_dir . "/" . $archive;
             my $cs_type = $1;
             my $cs_id = $2;