diff --git a/test_tube/hpc.py b/test_tube/hpc.py index 63c8b33..2143557 100644 --- a/test_tube/hpc.py +++ b/test_tube/hpc.py @@ -247,11 +247,14 @@ def __run_experiment(self, train_function): stop_in_n_seconds = max(stop_in_n_seconds, 10) # make sure we don't go below the 5 mins # schedule timer to interrupt training - threading.Timer(stop_in_n_seconds, self.call_save).start() + timer_instance = threading.Timer(stop_in_n_seconds, self.call_save) + timer_instance.start() # run training train_function(self.hyperparam_optimizer, self, {}) + timer_instance.cancel() + except Exception as e: print('Caught exception in worker thread', e)