diff --git a/tests/.gitignore b/tests/.gitignore index 3c9b0cf25..fbbd0dfe2 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -7,3 +7,5 @@ data_files/mpi_write data_files/hmac_drbg_seed data_files/ctr_drbg_seed data_files/entropy_seed + +/instrument_record_status.h diff --git a/tests/Makefile b/tests/Makefile index 8db7920d7..b9aed9494 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -28,6 +28,10 @@ ifdef DEBUG LOCAL_CFLAGS += -g3 endif +ifdef RECORD_PSA_STATUS_COVERAGE_LOG +LOCAL_CFLAGS += -Werror -DRECORD_PSA_STATUS_COVERAGE_LOG +endif + # if we're running on Windows, build for Windows ifdef WINDOWS WINDOWS_BUILD=1 @@ -149,3 +153,8 @@ $(EMBEDDED_TESTS): embedded_%: suites/$$(firstword $$(subst ., ,$$*)).function s generate-target-tests: $(EMBEDDED_TESTS) +ifdef RECORD_PSA_STATUS_COVERAGE_LOG +$(BINARIES): instrument_record_status.h +instrument_record_status.h: ../include/psa/crypto.h Makefile + sed <../include/psa/crypto.h >$@ -n 's/^psa_status_t \([A-Za-z0-9_]*\)(.*/#define \1(...) RECORD_STATUS("\1", \1(__VA_ARGS__))/p' +endif diff --git a/tests/scripts/psa_collect_statuses.py b/tests/scripts/psa_collect_statuses.py new file mode 100755 index 000000000..2fe80aeb8 --- /dev/null +++ b/tests/scripts/psa_collect_statuses.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Measure the test coverage of PSA functions in terms of return statuses. + +1. Build with -DRECORD_PSA_STATUS_COVERAGE_LOG +2. Run psa_collect_statuses.py +""" + +import argparse +import os +import subprocess +import sys + +_default_status_log_file = 'tests/statuses.log' +_default_psa_constant_names = 'programs/psa/psa_constant_names' + +class Statuses: + def __init__(self): + self.functions = {} + self.codes = set() + self.status_names = {} + + def collect_log(self, log_file_name): + with open(log_file_name) as log: + for line in log: + value, function, tail = line.split(':', 2) + if function not in self.functions: + self.functions[function] = {} + fdata = self.functions[function] + if value not in self.functions[function]: + fdata[value] = [] + fdata[value].append(tail) + self.codes.add(int(value)) + + def get_constant_names(self, psa_constant_names): + values = [str(value) for value in self.codes] + cmd = [psa_constant_names, 'status'] + values + output = subprocess.check_output(cmd).decode('ascii') + for value, name in zip(values, output.rstrip().split('\n')): + self.status_names[value] = name + + def report(self): + for function in sorted(self.functions.keys()): + fdata = self.functions[function] + names = [self.status_names[value] for value in fdata.keys()] + for name in sorted(names): + sys.stdout.write('{} {}\n'.format(function, name)) + +def collect_status_logs(options): + rebuilt = False + if not options.use_existing_log and os.path.exists(options.log_file): + os.remove(options.log_file) + if not os.path.exists(options.log_file): + if options.clean_before: + subprocess.check_call(['make', 'clean'], + cwd='tests', + stdout = sys.stderr) + with open(os.devnull, 'w') as devnull: + make_q_ret = subprocess.call(['make', '-q'], + stdout=devnull, stderr=devnull) + if make_q_ret != 0: + subprocess.check_call(['make', 'RECORD_PSA_STATUS_COVERAGE_LOG=1'], + stdout = sys.stderr) + rebuilt = True + subprocess.check_call(['make', 'test'], + stdout = sys.stderr) + data = Statuses() + data.collect_log(options.log_file) + data.get_constant_names(options.psa_constant_names) + if rebuilt and options.clean_after: + subprocess.check_call(['make', 'clean'], + cwd='tests', + stdout = sys.stderr) + return data + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=globals()['__doc__']) + parser.add_argument('--clean-after', + action='store_true', + help='Run "make clean" after rebuilding') + parser.add_argument('--clean-before', + action='store_true', + help='Run "make clean" before regenerating the log file)') + parser.add_argument('--log-file', metavar='FILE', + default=_default_status_log_file, + help='Log file location (default: {})'.format(_default_status_log_file)) + parser.add_argument('--psa-constant-names', metavar='PROGRAM', + default=_default_psa_constant_names, + help='Path to psa_constant_names (default: {})'.format(_default_psa_constant_names)) + parser.add_argument('--use-existing-log', '-e', + action='store_true', + help='Don\'t regenerate the log file if it exists') + options = parser.parse_args() + data = collect_status_logs(options) + data.report() diff --git a/tests/suites/helpers.function b/tests/suites/helpers.function index 122a17da7..6bfe43056 100644 --- a/tests/suites/helpers.function +++ b/tests/suites/helpers.function @@ -368,6 +368,24 @@ typedef enum } #endif +#if defined(RECORD_PSA_STATUS_COVERAGE_LOG) +#include +#define STATUS_LOG_FILE_NAME "statuses.log" +static psa_status_t record_status( psa_status_t status, + const char *func, + const char *file, int line, + const char *expr ) +{ + static FILE *log; + if( log == NULL ) + log = fopen( STATUS_LOG_FILE_NAME, "a" ); + fprintf( log, "%d:%s:%s:%d:%s\n", (int) status, func, file, line, expr ); + return( status ); +} +#define RECORD_STATUS( function, expr ) \ + record_status( ( expr ), function, __FILE__, __LINE__, #expr ) +#include "instrument_record_status.h" +#endif /* defined(RECORD_PSA_STATUS_COVERAGE_LOG) */ /*----------------------------------------------------------------------------*/ /* Global variables */