Skip to content

Commit

Permalink
metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@atheurer
atheurer committed Oct 20, 2023
1 parent 7cf67f3 commit 0d698b2
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 8 deletions.
8 changes: 8 additions & 0 deletions endpoints/remotehost/remotehost
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ endpoint_name="remotehost"
image_cache_size=3
osruntime[default]="chroot"
host_mounts=""
hypervisor_host="none" # Default is no hypervisor

function endpoint_remotehost_test_stop() {
echo "Running endpoint_remotehost_test_stop"
Expand Down Expand Up @@ -209,6 +210,9 @@ function process_remotehost_opts() {
controller_ipaddr=`get_controller_ip $host`
fi
;;
hypervisor-host)
hypervisor_host=$val
;;
controller-ip)
controller_ipaddr=$val
;;
Expand Down Expand Up @@ -362,6 +366,8 @@ function launch_osruntime() {
base_cmd+=" --cpu-partitioning=$cpu_partitioning"
base_cmd+=" --engine-script-start-timeout=$engine_script_start_timeout"
base_cmd+=" --disable-tools=$this_disable_tools"
base_cmd+=" --hosted-by=$host" # The host that runs the osruntime for this crucible engine
base_cmd+=" --hypervisor-host=$hypervisor_host" # The hypervisor host that runs the VM (if this remotehost happens to be a VM)
if [ $numa_node -gt -1 ]; then
base_cmd="numactl -N $numa_node -m $numa_node $base_cmd"
fi
Expand All @@ -388,6 +394,8 @@ function launch_osruntime() {
echo "max_rb_attempts=$max_rb_attempts" >> ${endpoint_run_dir}/${env_file}
echo "ssh_id=${ssh_id}" >> ${endpoint_run_dir}/${env_file}
echo "disable_tools=$this_disable_tools" >> ${endpoint_run_dir}/${env_file}
echo "hosted_by=$host" >> ${endpoint_run_dir}/${env_file}
echo "hypervisor_host=$hypervisor_host" >> ${endpoint_run_dir}/${env_file}

for cs_rb_opt in $cs_rb_opts; do
arg=$(echo $cs_rb_opt | awk -F'=' '{print $1}')
Expand Down
10 changes: 10 additions & 0 deletions engine/bootstrap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,16 @@ while true; do
export rickshaw_host="$1"
shift;
;;
--hosted-by)
shift;
export hosted_by="$1"
shift;
;;
--hypervisor-host)
shift;
export hypervisor_host="$1"
shift;
;;
--base-run-dir)
shift;
export base_run_dir=$1
Expand Down
2 changes: 2 additions & 0 deletions engine/engine-script
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ if ! pushd $cs_dir; then
abort_error "Could not chdir to $cs_dir" engine-init-begin
exit 1
fi
# There are a number of env vars needed for post-processing metrics
env >engine-env.txt

if [ "${cpu_partitioning}" == "1" ]; then
if [ -z "${HK_CPUS}" ]; then
Expand Down
54 changes: 47 additions & 7 deletions rickshaw-index
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,8 +352,9 @@ sub wait_for_metric_descs {
# with create_es_doc(). Metrics can be indexed from either a benchmark sample directory or a tool
# directory.
sub index_metrics {
my $index_or_queue = shift;
my $metr_file = shift; # filename without .json or .csv
my $index_or_queue = shift; # what action to take, index = submit to ES, queue = enqueue to file for bulk index later
my $metr_dir = shift; # directory where metric files exist
my $metr_file = shift; # metric filename without .json or .csv
my $cstype = shift;
my $csid = shift;
my $base_doc_ref = shift; # metric_desc doc gets populated with this, usually a run doc or period doc
Expand All @@ -364,6 +365,19 @@ sub index_metrics {
my $earliest_begin;
my $latest_end;
my $coder = JSON::XS->new->canonical;

{
my $dir = pushd($metr_dir);

my $eng_env_file = "engine-env.txt";
if (! -e $eng_env_file) {
$eng_env_file .= ".xz";
}
if (! -e $eng_env_file) {
printf "pwd: %s\nCould not find %s, exiting\n", getcwd(), $eng_env_file;
exit 1;
}

# Copy data from 'parent' doc so querying directly for metric_desc with
# run data is possible
my $metr_json_file = $metr_file . ".json";
Expand All @@ -372,7 +386,7 @@ sub index_metrics {
$metr_json_file .= ".xz";
}
if (! -e $metr_json_file) {
printf "Could not find %s, exiting\n", $metr_json_file;
printf "pwd: %s\nCould not find %s, exiting\n", getcwd(), $metr_json_file;
exit 1;
}
if (! -e $metr_csv_file) {
Expand Down Expand Up @@ -410,8 +424,30 @@ sub index_metrics {
if ( exists $$this_metr{'values'} ) {
$metr_desc_doc{'metric_desc'}{'values'} = $$this_metr{'values'};
}
# this is where we add engine-related metadata
$metr_desc_doc{'metric_desc'}{'names'}{'cstype'} = $cstype;
$metr_desc_doc{'metric_desc'}{'names'}{'csid'} = $csid;
my $eng_env_fh = new IO::Uncompress::UnXz $eng_env_file, Transparent => 1 || die "[ERROR]could not open file " . $eng_env_file;
# Cull out env vars which we want as metadata
printf "Looking for engine env vars\n";
my @varnames = ('hosted_by', 'hypervisor_host', 'osruntime');
while (<$eng_env_fh>) {

chomp;
printf "env: %s\n", $_;

foreach my $varname (@varnames) {
if (/^$varname=(.*)$/) {
my $val = $1;
$varname =~ s/_/-/g;
$metr_desc_doc{'metric_desc'}{'names'}{$varname} = $val;
printf "found %s = %s\n", $varname, $val;
next;
}
}
}
close $eng_env_fh;

my @names_list = sort(keys(%{ $metr_desc_doc{'metric_desc'}{'names'} }));
$metr_desc_doc{'metric_desc'}{'names-list'} = \@names_list;
my $metr_desc_doc_json = $coder->encode(\%metr_desc_doc);
Expand Down Expand Up @@ -483,6 +519,8 @@ sub index_metrics {
} else {
return $num_metric_docs_submitted;
}

} #pushd
}

sub indexed_doc_count {
Expand Down Expand Up @@ -702,7 +740,8 @@ if (-e $tool_dir and $index_tools == 1) {
$tool_file =~ s/(metric-data-\S+)\.json.*/$1/;
printf "Working on tool_file: %s\n", $tool_file;

my %job_args = ( 'tool-file' => $tool_dir . "/" . $tool_file,
my %job_args = ( 'tool-dir' => $tool_dir,
'tool-file' => $tool_file,
'collector' => $collector,
'num' => $num,
'doc-ref' => $base_metric_doc_ref );
Expand All @@ -724,7 +763,7 @@ foreach my $job_args (@jobs) {
push(@pids, $pid);
$num_jobs++;
} else {
my $num_metric_docs_submitted = index_metrics('index', $$job_args{'tool-file'}, $$job_args{'collector'}, $$job_args{'num'}, $$job_args{'doc-ref'});
my $num_metric_docs_submitted = index_metrics('index', $$job_args{'tool-dir'}, $$job_args{'tool-file'}, $$job_args{'collector'}, $$job_args{'num'}, $$job_args{'doc-ref'});
#$tool_dir . "/" . $tool_file, $collector, $num, $base_metric_doc_ref);
exit 0;
}
Expand Down Expand Up @@ -880,11 +919,12 @@ if (exists $result{'iterations'}) {
for (my $j = 0; $j < scalar(@{ $data{'periods'}[$k]{'metric-files'} }); $j++) {
# Metric data is still in other file(s). For each member in 'metric-files' array,
# there should be a 2 files with the same prefix
my $metric_file_prefix = $run_dir . "/" . $cs_id_dir . "/" . $data{'periods'}[$k]{'metric-files'}[$j];
my $metric_file_prefix = $data{'periods'}[$k]{'metric-files'}[$j];
my $metric_dir = $run_dir . "/" . $cs_id_dir;
my $this_begin;
my $this_end;
# index_metric() to return the easliest-begin and latest-end for metric types matching the primary-metric
(my $num_metric_docs_submitted, $this_begin, $this_end) = index_metrics('queue', $metric_file_prefix, $cs_name, $cs_id, $base_metric_doc_ref, $data{'benchmark'}, $data{'primary-metric'});
(my $num_metric_docs_submitted, $this_begin, $this_end) = index_metrics('queue', $metric_dir, $metric_file_prefix, $cs_name, $cs_id, $base_metric_doc_ref, $data{'benchmark'}, $data{'primary-metric'});
# From processing all metric files, get the very-earliest-begin and very-latest-end
if (defined $this_begin and defined $this_end) {
if (not defined $earliest_begin or $earliest_begin > $this_begin) {
Expand Down
2 changes: 1 addition & 1 deletion rickshaw-run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use toolbox::logging;
use toolbox::run;
use toolbox::jsonsettings;

$toolbox::logging::debug = 0;
$toolbox::logging::debug = 1;

my $ug = Data::UUID->new;
my %defaults = ( "num-samples" => 1, "tool-group" => "default", "test-order" => "s",
Expand Down

0 comments on commit 0d698b2

Please sign in to comment.