From e92db8dddd79ff8d223ba15bfb379ee57e55279a Mon Sep 17 00:00:00 2001 From: Joanna Moy Date: Sat, 17 Mar 2018 14:53:15 +0000 Subject: [PATCH 01/21] Change group_type field so that it expects a list of group types rather than one single group type. --- mlp_group.py | 14 +++++++------- mlp_network.py | 18 +++++++++--------- rand10x40_ag.py | 4 ++-- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mlp_group.py b/mlp_group.py index 0378bef..d8ceb65 100644 --- a/mlp_group.py +++ b/mlp_group.py @@ -9,7 +9,7 @@ class MLPGroup(): def __init__(self, gid, units = None, - gtype = MLPGroupTypes.HIDDEN, + gtype = [MLPGroupTypes.HIDDEN], input_funcs = None, output_funcs = None, write_blk = None, @@ -46,8 +46,8 @@ def __init__(self, self.t_vertex = None # group function parameters - self.output_grp = (self.type == MLPGroupTypes.OUTPUT) - self.input_grp = (self.type == MLPGroupTypes.INPUT) + self.output_grp = (MLPGroupTypes.OUTPUT in self.type) + self.input_grp = (MLPGroupTypes.INPUT in self.type) # weight-related parameters self.learning_rate = None @@ -120,23 +120,23 @@ def __init__(self, self.init_output = MLPConstants.DEF_INIT_OUT # host communication parameters - self.write_out = (self.type == MLPGroupTypes.OUTPUT) + self.write_out = (MLPGroupTypes.OUTPUT in self.type) # group type modifies default values - if (self.type == MLPGroupTypes.BIAS): + if (MLPGroupTypes.BIAS in self.type): self.out_integr_en = 0 self.num_out_procs = 1 self.out_procs_list [0] = MLPOutputProcs.OUT_BIAS self.out_procs_list [1] = MLPOutputProcs.OUT_NONE self.init_output = MLPConstants.BIAS_INIT_OUT - elif (self.type == MLPGroupTypes.INPUT): + elif (MLPGroupTypes.INPUT in self.type): self.out_integr_en = 0 self.num_out_procs = 1 self.out_procs_list [0] = MLPOutputProcs.OUT_HARD_CLAMP self.out_procs_list [1] = MLPOutputProcs.OUT_NONE - elif (self.type == MLPGroupTypes.OUTPUT): + elif (MLPGroupTypes.OUTPUT in self.type): self.write_out = 1 self.group_criterion = MLPConstants.DEF_GRP_CRIT self.criterion_function = MLPStopCriteria.STOP_STD diff --git a/mlp_network.py b/mlp_network.py index 454ab1e..02b633f 100644 --- a/mlp_network.py +++ b/mlp_network.py @@ -68,7 +68,7 @@ def __init__(self, # create single-unit Bias group by default self._bias_group = self.group (units = 1, - group_type = MLPGroupTypes.BIAS, + group_type = [MLPGroupTypes.BIAS], label = "Bias" ) @@ -150,7 +150,7 @@ def config (self): def group (self, units = None, - group_type = MLPGroupTypes.HIDDEN, + group_type = [MLPGroupTypes.HIDDEN], input_funcs = None, output_funcs = None, label = None @@ -158,7 +158,7 @@ def group (self, """ add a group to the network :param units: number of units that form the group - :param group_type: Lens-style group type + :param group_type: list of Lens-style group types :param input_funcs: functions applied in the input pipeline :param output_funcs: functions appllied in the output pipeline :param label: human-readable group identifier @@ -174,7 +174,7 @@ def group (self, _id = len (self.groups) # set properties for OUTPUT group - if (group_type == MLPGroupTypes.OUTPUT): + if (MLPGroupTypes.OUTPUT in group_type): _write_blk = len (self.output_chain) if len (self.output_chain): _is_first_out = 0 @@ -204,17 +204,17 @@ def group (self, ) # if it's an INPUT group add to list - if (group_type == MLPGroupTypes.INPUT): + if (MLPGroupTypes.INPUT in group_type): self.in_grps.append (_group) # if it's an OUTPUT group add to list and to the tail of the chain - if (group_type == MLPGroupTypes.OUTPUT): + if (MLPGroupTypes.OUTPUT in group_type): self.out_grps.append (_group) self.output_chain.append (_group) # OUTPUT and HIDDEN groups instantiate BIAS links by default - if (group_type == MLPGroupTypes.OUTPUT or\ - group_type == MLPGroupTypes.HIDDEN): + if (MLPGroupTypes.OUTPUT in group_type or\ + MLPGroupTypes.HIDDEN in group_type): self.link (self.bias_group, _group) # a new group forces reloading of initial weights file @@ -396,7 +396,7 @@ def read_Lens_weights_file (self, # check that the file contains the right number of weights if int (_wf.readline ()) != _num_wts: - print "error: incorrect number of weights in file" + print "error: incorrect number of weights in file; expected {}".format (_num_wts) _wf.close () return False diff --git a/rand10x40_ag.py b/rand10x40_ag.py index f22de2a..4d98f8a 100644 --- a/rand10x40_ag.py +++ b/rand10x40_ag.py @@ -17,7 +17,7 @@ # instantiate network groups (layers) Input = rand10x40.group (units = 10, - group_type = MLPGroupTypes.INPUT, + group_type = [MLPGroupTypes.INPUT], label = "Input" ) Hidden = rand10x40.group (units = 50, @@ -25,7 +25,7 @@ label = "Hidden" ) Output = rand10x40.group (units = 10, - group_type = MLPGroupTypes.OUTPUT, + group_type = [MLPGroupTypes.OUTPUT], label = "Output" ) From 64e9bb68da944c9d7f03d779f08b115553f5fd6a Mon Sep 17 00:00:00 2001 From: Joanna Moy Date: Fri, 23 Mar 2018 16:06:00 +0000 Subject: [PATCH 02/21] If max_time is not set in the model file, it should default to the number of intervals (as opposed to 0, as it is currently doing. --- mlp_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mlp_network.py b/mlp_network.py index 02b633f..7a7f2d6 100644 --- a/mlp_network.py +++ b/mlp_network.py @@ -36,6 +36,7 @@ def __init__(self, """ # assign network parameter values from arguments self._net_type = net_type.value + self._intervals = intervals self._ticks_per_interval = ticks_per_interval # default network parameter values @@ -285,7 +286,9 @@ def example_set (self, self.label = label # instantiate a new example set - _set = MLPExampleSet (label = label) + _set = MLPExampleSet (label = label, + max_time = self._intervals + ) # add example set to the network list self._ex_set = _set From bf386ddd63200588d7493be22faa7dc009c8252e Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Mon, 26 Mar 2018 16:24:55 +0100 Subject: [PATCH 03/21] - fixed a bug setting parameters for groups that have more than one type. - added code to handle early aborts caused by weight/example files. --- mlp_group.py | 23 ++++++++++++----------- mlp_network.py | 26 ++++++++++++++++++-------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/mlp_group.py b/mlp_group.py index d8ceb65..ac1c735 100644 --- a/mlp_group.py +++ b/mlp_group.py @@ -130,14 +130,15 @@ def __init__(self, self.out_procs_list [1] = MLPOutputProcs.OUT_NONE self.init_output = MLPConstants.BIAS_INIT_OUT - elif (MLPGroupTypes.INPUT in self.type): - self.out_integr_en = 0 - self.num_out_procs = 1 - self.out_procs_list [0] = MLPOutputProcs.OUT_HARD_CLAMP - self.out_procs_list [1] = MLPOutputProcs.OUT_NONE - - elif (MLPGroupTypes.OUTPUT in self.type): - self.write_out = 1 - self.group_criterion = MLPConstants.DEF_GRP_CRIT - self.criterion_function = MLPStopCriteria.STOP_STD - self.error_function = MLPErrorFuncs.ERR_CROSS_ENTROPY + else: + if (MLPGroupTypes.INPUT in self.type): + self.out_integr_en = 0 + self.num_out_procs = 1 + self.out_procs_list [0] = MLPOutputProcs.OUT_HARD_CLAMP + self.out_procs_list [1] = MLPOutputProcs.OUT_NONE + + if (MLPGroupTypes.OUTPUT in self.type): + self.write_out = 1 + self.group_criterion = MLPConstants.DEF_GRP_CRIT + self.criterion_function = MLPStopCriteria.STOP_STD + self.error_function = MLPErrorFuncs.ERR_CROSS_ENTROPY diff --git a/mlp_network.py b/mlp_network.py index 7a7f2d6..fe6f924 100644 --- a/mlp_network.py +++ b/mlp_network.py @@ -76,6 +76,9 @@ def __init__(self, # keep track of the number of vertices in the graph self._num_vertices = 0 + # keep track if errors have occured + self._aborted = False + @property def net_type (self): @@ -600,6 +603,7 @@ def run (self): # cannot run unless weights file exists if self._weights_file is None: print "run aborted: weights file not given" + self._aborted = True return # may need to reload initial weights file if @@ -607,21 +611,26 @@ def run (self): if not self._weights_loaded: if not self.read_Lens_weights_file (self._weights_file): print "run aborted: error reading weights file" + self._aborted = True + return # cannot run unless example set exists if self._ex_set is None: print "run aborted: no example set" + self._aborted = True return # cannot run unless examples have been loaded if not self._ex_set.examples_loaded: print "run aborted: examples not loaded" + self._aborted = True return # generate summary set, example and event data self._num_examples = self._ex_set.compile (self) if self._num_examples == 0: print "run aborted: error compiling example set" + self._aborted = True return # generate machine graph @@ -634,8 +643,8 @@ def run (self): print "running: waiting for application to finish" _txrx = g.transceiver () _app_id = globals_variables.get_simulator ()._app_id -#lap _running = _txrx.get_core_state_count (_app_id, CPUState.RUNNING) - _finished = _txrx.get_core_state_count (_app_id, CPUState.FINISHED) +#lap _running = _txrx.get_core_state_count (_app_id, CPUState.RUNNING) + _finished = _txrx.get_core_state_count (_app_id, CPUState.FINISHED) while _finished < self._num_vertices: time.sleep (0.5) _error = _txrx.get_core_state_count\ @@ -646,15 +655,16 @@ def run (self): RTE, {} WDOG)".format (_error, _wdog) break #lap _running = _txrx.get_core_state_count (_app_id, CPUState.RUNNING) - _finished = _txrx.get_core_state_count (_app_id, CPUState.FINISHED) + _finished = _txrx.get_core_state_count (_app_id, CPUState.FINISHED) def end (self): """ clean up before exiting """ - # pause to allow debugging - raw_input ('paused: press enter to exit') + if not self._aborted: + # pause to allow debugging + raw_input ('paused: press enter to exit') - print "exit: application finished" - # let the gfe clean up - g.stop() + print "exit: application finished" + # let the gfe clean up + g.stop() From bdf6474e88197411afea2ca45e9f35ae43cad97a Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Tue, 3 Apr 2018 17:48:31 +0100 Subject: [PATCH 04/21] - made changes required to partition weight cores with more that 32 units. This prevents the core from running out of DTCM. --- c_code/comms_i.c | 10 +++---- c_code/comms_s.c | 44 ++++++++++++++----------------- c_code/comms_t.c | 63 ++++++++++++++++++++++++-------------------- c_code/comms_w.c | 52 ++++++++++++++++++++---------------- c_code/init_t.c | 4 +-- c_code/init_w.c | 10 ++++--- c_code/mlp_externs.h | 6 ++++- c_code/mlp_params.h | 7 +++++ c_code/mlp_types.h | 2 ++ c_code/process_i.c | 26 +++++++++--------- c_code/process_s.c | 42 +++++++++++++++++++---------- c_code/process_t.c | 7 +++-- c_code/process_w.c | 9 ++++--- c_code/sum.c | 9 ++++++- c_code/threshold.c | 2 +- c_code/weight.c | 9 +++++-- mlp_group.py | 11 +++++++- mlp_network.py | 30 ++++++++++++++++----- mlp_types.py | 5 +++- sum_vertex.py | 8 +++--- threshold_vertex.py | 7 ++--- weight_vertex.py | 55 +++++++++++++++++++++++++++++--------- 22 files changed, 268 insertions(+), 150 deletions(-) diff --git a/c_code/comms_i.c b/c_code/comms_i.c index 51e678f..f1a650a 100644 --- a/c_code/comms_i.c +++ b/c_code/comms_i.c @@ -19,7 +19,7 @@ void i_receivePacket (uint key, uint payload) // check if stop packet if ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY) { - // sync packet received + // stop packet received #ifdef DEBUG stp_recv++; #endif @@ -49,6 +49,7 @@ void i_receivePacket (uint key, uint payload) return; } + // check if network stop packet if ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY) { // network stop packet received @@ -61,13 +62,8 @@ void i_receivePacket (uint key, uint payload) return; } - #ifdef DEBUG - pkt_recv++; - #endif - - // check if space in packet queue, + // queue packet - if space available uint new_tail = (i_pkt_queue.tail + 1) % SPINN_INPUT_PQ_LEN; - if (new_tail == i_pkt_queue.head) { // if queue full exit and report failure diff --git a/c_code/comms_s.c b/c_code/comms_s.c index 092f376..83886a4 100644 --- a/c_code/comms_s.c +++ b/c_code/comms_s.c @@ -45,9 +45,12 @@ void s_receivePacket (uint key, uint payload) // if not done report processing thread done sf_thrds_done -= 1; } + + return; } + // check if network stop packet - else if ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY) + if ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY) { // network stop packet received #ifdef DEBUG @@ -58,33 +61,26 @@ void s_receivePacket (uint key, uint payload) spin1_exit (SPINN_NO_ERROR); return; } + + // queue packet - if space available + uint new_tail = (s_pkt_queue.tail + 1) % SPINN_SUM_PQ_LEN; + if (new_tail == s_pkt_queue.head) + { + // if queue full exit and report failure + spin1_exit (SPINN_QUEUE_FULL); + } else { - #ifdef DEBUG - pkt_recv++; - #endif - - // check if space in packet queue, - uint new_tail = (s_pkt_queue.tail + 1) % SPINN_SUM_PQ_LEN; + // if not full queue packet, + s_pkt_queue.queue[s_pkt_queue.tail].key = key; + s_pkt_queue.queue[s_pkt_queue.tail].payload = payload; + s_pkt_queue.tail = new_tail; - if (new_tail == s_pkt_queue.head) - { - // if queue full exit and report failure - spin1_exit (SPINN_QUEUE_FULL); - } - else + // and schedule processing thread -- if not active already + if (!s_active) { - // if not full queue packet, - s_pkt_queue.queue[s_pkt_queue.tail].key = key; - s_pkt_queue.queue[s_pkt_queue.tail].payload = payload; - s_pkt_queue.tail = new_tail; - - // and schedule processing thread -- if not active already - if (!s_active) - { - s_active = TRUE; - spin1_schedule_callback (s_process, NULL, NULL, SPINN_S_PROCESS_P); - } + s_active = TRUE; + spin1_schedule_callback (s_process, NULL, NULL, SPINN_S_PROCESS_P); } } } diff --git a/c_code/comms_t.c b/c_code/comms_t.c index dfb821e..d2b2e2d 100644 --- a/c_code/comms_t.c +++ b/c_code/comms_t.c @@ -16,43 +16,47 @@ // ------------------------------------------------------------------------ void t_receivePacket (uint key, uint payload) { - // get packet phase - uint ph = (key & SPINN_PHASE_MASK) >> SPINN_PHASE_SHIFT; - - // packet is stop type + // check if packet is stop type uint stop = ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY); - - // packet is chain type - uint chain = ((key & SPINN_TYPE_MASK) == SPINN_STPC_KEY); - - // packet is network stop type - uint stpn = ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY); - - // packet is sync type - uint sync = ((key & SPINN_TYPE_MASK) == SPINN_SYNC_KEY); - - // check packet type if (stop) { - // stop final decision packet + // process stop final decision packet t_stopPacket (key); + return; } - else if (chain) + + // check if packet is chain type + uint chain = ((key & SPINN_TYPE_MASK) == SPINN_STPC_KEY); + if (chain) { - // stop decision chain packet + // process stop decision chain packet t_chainPacket (key); + return; } - else if (stpn) + + // check if packet is network stop type + uint stpn = ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY); + if (stpn) { - // network stop decision packet + // process network stop decision packet t_networkStopPacket (); + return; } - else if (sync) + + // get packet phase + uint ph = (key & SPINN_PHASE_MASK) >> SPINN_PHASE_SHIFT; + + // check if packet is sync type + uint sync = ((key & SPINN_TYPE_MASK) == SPINN_SYNC_KEY); + if (sync) { - // tick synchronisation packet + // process tick synchronisation packet t_syncPacket (ph); + return; } - else if (ph == SPINN_FORWARD) + + // computation packet + if (ph == SPINN_FORWARD) { // FORWARD phase packet t_forwardPacket (key, payload); @@ -174,9 +178,12 @@ void t_syncPacket (uint ph) t_sync_arrived = 0; // and check if can trigger sending data - if (phase == SPINN_FORWARD) + if (t_sync_rdy) { - // schedule sending of unit outputs + // clear synchronisation flag, + t_sync_rdy = FALSE; + + // schedule sending of unit outputs, spin1_schedule_callback (t_init_outputs, NULL, NULL, SPINN_T_INIT_OUT_P ); @@ -191,8 +198,8 @@ void t_syncPacket (uint ph) } else { - // if not ready flag sync done - t_sync_done = TRUE; + // if not flag sync as ready + t_sync_rdy = TRUE; } } } @@ -218,7 +225,7 @@ void t_syncPacket (uint ph) else { // if not ready flag sync done - t_sync_done = TRUE; + t_sync_rdy = TRUE; } } } diff --git a/c_code/comms_w.c b/c_code/comms_w.c index 75f3f8e..d744b3e 100644 --- a/c_code/comms_w.c +++ b/c_code/comms_w.c @@ -16,43 +16,51 @@ // ------------------------------------------------------------------------ void w_receivePacket (uint key, uint payload) { - // get packet phase - uint ph = (key & SPINN_PHASE_MASK) >> SPINN_PHASE_SHIFT; - // check if packet is stop type uint stop = ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY); - - // packet is network stop type - uint stpn = ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY); - - // check if packet is ldsr type - uint ldsr = ((key & SPINN_TYPE_MASK) == SPINN_LDSR_KEY); - - // check packet type if (stop) { - // stop packet + // process stop packet w_stopPacket (key); + return; } - else if (stpn) + + // check if packet is network stop type + uint stpn = ((key & SPINN_TYPE_MASK) == SPINN_STPN_KEY); + if (stpn) { - // network stop decision packet + // process network stop decision packet w_networkStopPacket (); + return; } - else if (ldsr) + + // check if packet is ldsr type + uint ldsr = ((key & SPINN_TYPE_MASK) == SPINN_LDSR_KEY); + if (ldsr) { - // ldsr packet + // process ldsr packet w_ldsrPacket (payload); + return; } - else if (ph == SPINN_FORWARD) + + // computation packet - get packet phase and block + uint ph = (key & SPINN_PHASE_MASK) >> SPINN_PHASE_SHIFT; + uint blk = (key & SPINN_BLOCK_MASK) >> SPINN_BLOCK_SHIFT; + if (ph == SPINN_FORWARD) { - // FORWARD phase - w_forwardPacket (key, payload); + // FORWARD phase packet in my block + if (blk == wcfg.row_blk) + { + w_forwardPacket (key, payload); + } } else { - // BACKPROP phase - w_backpropPacket (key, payload); + // BACKPROP phase packet in my block + if (blk == wcfg.col_blk) + { + w_backpropPacket (key, payload); + } } } // ------------------------------------------------------------------------ @@ -158,7 +166,7 @@ void w_forwardPacket (uint key, uint payload) #endif // get output index: mask out phase, core and block data, - uint inx = key & SPINN_OUTPUT_MASK; + uint inx = key & SPINN_BLKOUT_MASK; // store received unit output, w_outputs[wf_comms][inx] = (activation_t) payload; diff --git a/c_code/init_t.c b/c_code/init_t.c index eaab793..145bd7a 100644 --- a/c_code/init_t.c +++ b/c_code/init_t.c @@ -191,7 +191,7 @@ uint t_init (void) { // "broadcast" key tf_stop_key = rt[STP] | SPINN_STOP_KEY; - + // "stop final" key tf_stpn_key = rt[STP] | SPINN_STPN_KEY; } @@ -213,7 +213,7 @@ uint t_init (void) t_sync_arrived = 0; // initialise sync packets flag - t_sync_done = FALSE; + t_sync_rdy = FALSE; // initialise net packet queue t_net_pkt_q.head = 0; diff --git a/c_code/init_w.c b/c_code/init_w.c index e2dd7a5..955382c 100644 --- a/c_code/init_w.c +++ b/c_code/init_w.c @@ -112,8 +112,8 @@ uint w_init (void) } // allocate memory for errors - if ((w_errors = ((error_t*) - spin1_malloc (wcfg.num_rows * sizeof(delta_t)))) == NULL + if ((w_errors = ((error_t *) + spin1_malloc (wcfg.num_rows * sizeof(error_t)))) == NULL ) { return (SPINN_MEM_UNAVAIL); @@ -211,8 +211,10 @@ uint w_init (void) // initialize packet keys //NOTE: colour is initialized to 0. - fwdKey = rt[FWD] | SPINN_PHASE_KEY(SPINN_FORWARD); - bkpKey = rt[BKP] | SPINN_PHASE_KEY(SPINN_BACKPROP); + fwdKey = rt[FWD] | SPINN_PHASE_KEY(SPINN_FORWARD) + | SPINN_BLOCK_KEY(wcfg.col_blk); + bkpKey = rt[BKP] | SPINN_PHASE_KEY(SPINN_BACKPROP) + | SPINN_BLOCK_KEY(wcfg.row_blk); wf_sync_key = rt[FDS] | SPINN_SYNC_KEY | SPINN_PHASE_KEY(SPINN_FORWARD); wb_sync_key = rt[FDS] | SPINN_SYNC_KEY | SPINN_PHASE_KEY(SPINN_BACKPROP); diff --git a/c_code/mlp_externs.h b/c_code/mlp_externs.h index d7ff6ac..e9b902e 100644 --- a/c_code/mlp_externs.h +++ b/c_code/mlp_externs.h @@ -145,7 +145,7 @@ extern uint t_tot_ticks; // total ticks on current example extern pkt_queue_t t_net_pkt_q; // queue to hold received nets extern uchar t_active; // processing nets/errors from queue? extern scoreboard_t t_sync_arrived; // keep track of expected sync packets -extern uchar t_sync_done; // have expected sync packets arrived? +extern uchar t_sync_rdy; // have expected sync packets arrived? extern sdp_msg_t t_sdp_msg; // SDP message buffer for host comms. extern scoreboard_t tf_arrived; // keep track of expected nets extern uint tf_thrds_done; // sync. semaphore: proc & stop @@ -193,6 +193,10 @@ extern long_deriv_t * t_output_deriv_history; extern uint stp_recv; // stop packets received extern uint stn_sent; // network_stop packets sent extern uint stn_recv; // network_stop packets received + extern uint lda_sent; // partial link_delta packets sent + extern uint lda_recv; // partial link_delta packets received + extern uint ldt_sent; // total link_delta packets sent + extern uint ldt_recv; // total link_delta packets received extern uint tot_tick; // total number of ticks executed extern uint wght_ups; // number of weight updates done extern uint wrng_phs; // packets received in wrong phase diff --git a/c_code/mlp_params.h b/c_code/mlp_params.h index f07db17..1d57957 100644 --- a/c_code/mlp_params.h +++ b/c_code/mlp_params.h @@ -121,6 +121,13 @@ #define SPINN_COLOUR_SHIFT 10 #define SPINN_COLOUR_MASK (1 << SPINN_COLOUR_SHIFT) +// block management +#define SPINN_BLOCK_SHIFT 5 +#define SPINN_BLOCK_MASK ((0xff << SPINN_BLOCK_SHIFT) & 0xff) +#define SPINN_BLOCK_KEY(p) (p << SPINN_BLOCK_SHIFT) +#define SPINN_BLKOUT_MASK ((1 << SPINN_BLOCK_SHIFT) - 1) +#define SPINN_BLKDLT_MASK ((1 << SPINN_BLOCK_SHIFT) - 1) + // packet data masks #define SPINN_OUTPUT_MASK 0x000000ff #define SPINN_NET_MASK 0x000000ff diff --git a/c_code/mlp_types.h b/c_code/mlp_types.h index d0fc3d1..903680b 100644 --- a/c_code/mlp_types.h +++ b/c_code/mlp_types.h @@ -200,6 +200,8 @@ typedef struct w_conf // weight core configuration { uint num_rows; // rows in this core's block uint num_cols; // columns in this core's block + uint row_blk; // this core's row block number + uint col_blk; // this core's column block number short_fpreal_t learningRate; // network learning rate short_fpreal_t weightDecay; // network weight decay short_fpreal_t momentum; // network momentum diff --git a/c_code/process_i.c b/c_code/process_i.c index b1c6754..a17f024 100644 --- a/c_code/process_i.c +++ b/c_code/process_i.c @@ -43,23 +43,11 @@ void i_process (uint null0, uint null1) if (ph == SPINN_FORWARD) { // process FORWARD phase packet - #ifdef DEBUG - recv_fwd++; - if (phase != SPINN_FORWARD) - wrng_phs++; - #endif - i_forward_packet (key, payload); } else { // process BACKPROP phase packet - #ifdef DEBUG - recv_bkp++; - if (phase != SPINN_BACKPROP) - wrng_phs++; - #endif - i_backprop_packet (key, payload); } @@ -81,6 +69,13 @@ void i_process (uint null0, uint null1) // ------------------------------------------------------------------------ void i_forward_packet (uint key, uint payload) { + #ifdef DEBUG + pkt_recv++; + recv_fwd++; + if (phase != SPINN_FORWARD) + wrng_phs++; + #endif + // get net index: mask out block, phase and colour data, uint inx = key & SPINN_NET_MASK; @@ -160,6 +155,13 @@ void i_forward_packet (uint key, uint payload) // ------------------------------------------------------------------------ void i_backprop_packet (uint key, uint payload) { + #ifdef DEBUG + pkt_recv++; + recv_bkp++; + if (phase != SPINN_BACKPROP) + wrng_phs++; + #endif + // get delta index: mask out block, phase and colour data, uint inx = key & SPINN_DELTA_MASK; diff --git a/c_code/process_s.c b/c_code/process_s.c index 44610ea..6cf213c 100644 --- a/c_code/process_s.c +++ b/c_code/process_s.c @@ -42,7 +42,7 @@ void s_process (uint null0, uint null1) // check for an LDS "accumulation" packet if ((key & SPINN_TYPE_MASK) == SPINN_LDSA_KEY) - { + { // process LDS "accumulation" packet s_ldsa_packet (payload); } @@ -55,23 +55,11 @@ void s_process (uint null0, uint null1) // else check packet phase and process accordingly else if (ph == SPINN_FORWARD) { - #ifdef DEBUG - recv_fwd++; - if (phase != SPINN_FORWARD) - wrng_phs++; - #endif - // process FORWARD phase packet s_forward_packet (key, payload); } else { - #ifdef DEBUG - recv_bkp++; - if (phase != SPINN_BACKPROP) - wrng_phs++; - #endif - // process BACKPROP phase packet s_backprop_packet (key, payload); } @@ -94,6 +82,10 @@ void s_process (uint null0, uint null1) // ------------------------------------------------------------------------ void s_ldsa_packet (uint payload) { + #ifdef DEBUG + lda_recv++; + #endif + // add the received value to the total so far, s_lds_part += (lds_t) payload; @@ -108,6 +100,10 @@ void s_ldsa_packet (uint payload) if (scfg.is_first_group == 0) { while (!spin1_send_mc_packet (ldstKey, s_lds_part, WITH_PAYLOAD)); + + #ifdef DEBUG + ldt_sent++; + #endif } // access synchronisation semaphore with interrupts disabled @@ -144,6 +140,10 @@ void s_ldsa_packet (uint payload) // ------------------------------------------------------------------------ void s_ldst_packet (uint payload) { + #ifdef DEBUG + ldt_recv++; + #endif + // add the received value to the total so far, s_lds_part += (lds_t) payload; @@ -190,6 +190,13 @@ void s_ldst_packet (uint payload) // ------------------------------------------------------------------------ void s_forward_packet (uint key, uint payload) { + #ifdef DEBUG + pkt_recv++; + recv_fwd++; + if (phase != SPINN_FORWARD) + wrng_phs++; + #endif + // get net index: mask out block and phase data, uint inx = key & SPINN_NET_MASK; @@ -282,6 +289,13 @@ void s_forward_packet (uint key, uint payload) // ------------------------------------------------------------------------ void s_backprop_packet (uint key, uint payload) { + #ifdef DEBUG + pkt_recv++; + recv_bkp++; + if (phase != SPINN_BACKPROP) + wrng_phs++; + #endif + // get error index: mask out block, phase and colour data, uint inx = key & SPINN_ERROR_MASK; @@ -356,7 +370,7 @@ void s_backprop_packet (uint key, uint payload) && tick == SPINN_SB_END_TICK + 1) { // if this s core relates to the first group in the network, then we - // also need to wait for the link delta sum totals, so set the threads + // also need to wait for the link delta sum totals, so set the threads // pending to 2 if (scfg.is_first_group == 1) { diff --git a/c_code/process_t.c b/c_code/process_t.c index cc127a0..791b026 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -585,10 +585,10 @@ void t_advance_example (void) // access flags with interrupts disabled uint cpsr = spin1_int_disable (); - if (t_sync_done) + if (t_sync_rdy) { // if ready clear synchronisation flag, - t_sync_done = FALSE; + t_sync_rdy = FALSE; // restore interrupts, spin1_mode_restore (cpsr); @@ -607,6 +607,9 @@ void t_advance_example (void) } else { + // if not flag sync as ready + t_sync_rdy = TRUE; + // restore interrupts spin1_mode_restore (cpsr); } diff --git a/c_code/process_w.c b/c_code/process_w.c index f8bfec4..f7aad62 100644 --- a/c_code/process_w.c +++ b/c_code/process_w.c @@ -122,8 +122,8 @@ void wb_process (uint null0, uint null1) // restore interrupts after queue access, spin1_mode_restore (cpsr); - // get delta index: mask out phase, core and block data, - inx &= SPINN_DELTA_MASK; + // get delta index: mask out phase and block data, + inx &= SPINN_BLKDLT_MASK; // update scoreboard, wb_arrived++; @@ -217,13 +217,16 @@ void wb_process (uint null0, uint null1) while (!spin1_send_mc_packet (ldsaKey, (uint) link_delta_sum_short, WITH_PAYLOAD) ); + #ifdef DEBUG + lda_sent++; + #endif } // if done with all deltas advance tick if (wb_arrived == wcfg.num_cols) { // initialize arrival scoreboard for next tick, - wb_arrived = 0; + wb_arrived = 0; // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); diff --git a/c_code/sum.c b/c_code/sum.c index 2f7837b..ec689de 100644 --- a/c_code/sum.c +++ b/c_code/sum.c @@ -91,6 +91,9 @@ scoreboard_t s_ldst_arrived; // keep track of the number of link delta su uint stp_sent = 0; // stop packets sent uint stp_recv = 0; // stop packets received uint stn_recv = 0; // network_stop packets received + uint lda_recv = 0; // partial link_delta packets received + uint ldt_sent = 0; // total link_delta packets snet + uint ldt_recv = 0; // total link_delta packets received uint wrng_phs = 0; // packets received in wrong phase uint wrng_tck = 0; // FORWARD packets received in wrong tick uint wrng_btk = 0; // BACKPROP packets received in wrong tick @@ -144,7 +147,7 @@ uint init () io_printf (IO_BUF, "fg: %d\n", scfg.is_first_group); io_printf (IO_BUF, "fk: 0x%08x\n", rt[FWD]); io_printf (IO_BUF, "bk: 0x%08x\n", rt[BKP]); - io_prtinf (IO_BUF, "lk: 0x%08x\n", rt[LDS]); + io_printf (IO_BUF, "lk: 0x%08x\n", rt[LDS]); #endif // initialize epoch, example and event counters @@ -227,6 +230,10 @@ void done (uint ec) io_printf (IO_BUF, "sync sent:%d\n", spk_sent); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stop sent:%d\n", stp_sent); + io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); + io_printf (IO_BUF, "ldsa recv:%d\n", lda_recv); + io_printf (IO_BUF, "ldst sent:%d\n", ldt_sent); + io_printf (IO_BUF, "ldst recv:%d\n", ldt_recv); #endif } // ------------------------------------------------------------------------ diff --git a/c_code/threshold.c b/c_code/threshold.c index 99cc889..c91fe3e 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -130,7 +130,7 @@ uint t_tot_ticks; // total ticks on current example pkt_queue_t t_net_pkt_q; // queue to hold received nets uchar t_active; // processing nets/errors from queue? scoreboard_t t_sync_arrived; // keep track of expected sync packets -uchar t_sync_done; // have expected sync packets arrived? +uchar t_sync_rdy; // have expected sync packets arrived? sdp_msg_t t_sdp_msg; // SDP message buffer for host comms. // FORWARD phase specific diff --git a/c_code/weight.c b/c_code/weight.c index a053829..234e7a7 100644 --- a/c_code/weight.c +++ b/c_code/weight.c @@ -117,6 +117,7 @@ activation_t * w_output_history; // history array for outputs uint stp_sent = 0; // stop packets sent uint stp_recv = 0; // stop packets received uint stn_recv = 0; // network_stop packets received + uint lda_sent = 0; // partial link_delta packets sent uint wrng_phs = 0; // packets received in wrong phase uint wrng_tck = 0; // FORWARD packets received in wrong tick uint wrng_btk = 0; // BACKPROP packets received in wrong tick @@ -167,9 +168,11 @@ uint init () #ifdef DEBUG_CFG0 io_printf (IO_BUF, "nr: %d\n", wcfg.num_rows); io_printf (IO_BUF, "nc: %d\n", wcfg.num_cols); + io_printf (IO_BUF, "rb: %d\n", wcfg.row_blk); + io_printf (IO_BUF, "cb: %d\n", wcfg.col_blk); io_printf (IO_BUF, "lr: %k\n", wcfg.learningRate); io_printf (IO_BUF, "wd: %k\n", wcfg.weightDecay); - io_printf (IO_BUF, "mm: %l\n", wcfg.momentum); + io_printf (IO_BUF, "mm: %k\n", wcfg.momentum); io_printf (IO_BUF, "fk: 0x%08x\n", rt[FWD]); io_printf (IO_BUF, "bk: 0x%08x\n", rt[BKP]); io_printf (IO_BUF, "sk: 0x%08x\n", rt[FDS]); @@ -226,7 +229,7 @@ void done (uint ec) break; case SPINN_TIMEOUT_EXIT: - io_printf (IO_BUF, "timeout (h: %u e:%u p:%u t:%u) - abort!\n", + io_printf (IO_BUF, "timeout (h:%u e:%u p:%u t:%u) - abort!\n", epoch, example, phase, tick ); @@ -254,6 +257,8 @@ void done (uint ec) io_printf (IO_BUF, "sync sent:%d\n", spk_sent); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stop sent:%d\n", stp_sent); + io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); + io_printf (IO_BUF, "ldsa sent:%d\n", lda_sent); io_printf (IO_BUF, "weight updates:%d\n", wght_ups); #endif } diff --git a/mlp_group.py b/mlp_group.py index ac1c735..40277d4 100644 --- a/mlp_group.py +++ b/mlp_group.py @@ -25,7 +25,16 @@ def __init__(self, self.is_first_out = is_first_out self.label = label - print "creating group {}".format (self.label) + # number of partitions required for this group + self.partitions = (self.units + MLPConstants.MAX_BLK_UNITS - 1)\ + // MLPConstants.MAX_BLK_UNITS + + if self.partitions == 1: + print "creating group {} with 1 partition".\ + format (self.label) + else: + print "creating group {} with {} partitions".\ + format (self.label, self.partitions) # keep track of associated incoming links self.links_from = [] diff --git a/mlp_network.py b/mlp_network.py index fe6f924..8de5980 100644 --- a/mlp_network.py +++ b/mlp_network.py @@ -76,6 +76,9 @@ def __init__(self, # keep track of the number of vertices in the graph self._num_vertices = 0 + # keep track of the number of partitions + self.partitions = 0 + # keep track if errors have occured self._aborted = False @@ -448,16 +451,23 @@ def generate_machine_graph (self): # set the number of write blocks before generating vertices self._num_write_blks = len (self.output_chain) + # compute number of partitions + for grp in self.groups: + self.partitions = self.partitions + grp.partitions + # create associated weight, sum, input and threshold # machine vertices for every network group for grp in self.groups: - # create one weight core per (from_group, group) pair + # create one weight core per partition + # of every (from_group, group) pair # NOTE: all-zero cores can be optimised out for from_grp in self.groups: - wv = WeightVertex (self, grp, from_grp) - grp.w_vertices.append (wv) - g.add_machine_vertex_instance (wv) - self._num_vertices += 1 + for _tp in range (grp.partitions): + for _fp in range (from_grp.partitions): + wv = WeightVertex (self, grp, from_grp, _tp, _fp) + grp.w_vertices.append (wv) + g.add_machine_vertex_instance (wv) + self._num_vertices += 1 # create one sum core per group sv = SumVertex (self, grp) @@ -471,7 +481,7 @@ def generate_machine_graph (self): g.add_machine_vertex_instance (iv) self._num_vertices += 1 - # create one sum core per group + # create one threshold core per group tv = ThresholdVertex (self, grp) grp.t_vertex = tv g.add_machine_vertex_instance (tv) @@ -633,6 +643,14 @@ def run (self): self._aborted = True return + # check that no group is too big + for grp in self.groups: + if grp.units > MLPConstants.MAX_GRP_UNITS: + print "run aborted: group {} has more than {} units.".\ + format (grp.id, MLPConstants.MAX_GRP_UNITS) + self._aborted = True + return + # generate machine graph self.generate_machine_graph () diff --git a/mlp_types.py b/mlp_types.py index 4cedf3b..6ec4718 100644 --- a/mlp_types.py +++ b/mlp_types.py @@ -17,6 +17,9 @@ class MLPConstants (): MAX_IN_PROCS = 2 DEF_IN_PROCS = 0 + MAX_GRP_UNITS = 128 + MAX_BLK_UNITS = 32 + MAX_OUT_PROCS = 5 DEF_OUT_PROCS = 2 @@ -117,7 +120,7 @@ class MLPErrorFuncs (Enum): class MLPUpdateFuncs (Enum): """ MLP weight update functions - """ + """ UPD_STEEPEST = 0 UPD_MOMENTUM = 1 UPD_DOUGSMOMENTUM = 2 diff --git a/sum_vertex.py b/sum_vertex.py index da681ed..94a9863 100644 --- a/sum_vertex.py +++ b/sum_vertex.py @@ -57,10 +57,10 @@ def __init__(self, self._lds_link = "lds_s{}".format (self.group.id) # sum core-specific parameters - # NOTE: if all-zero w cores are optimised out this need reviewing - self._fwd_expect = len (network.groups) - self._bkp_expect = len (network.groups) - self._ldsa_expect = len (network.groups) * self.group.units + # NOTE: if all-zero w cores are optimised out these need reviewing + self._fwd_expect = network.partitions + self._bkp_expect = network.partitions + self._ldsa_expect = network.partitions * self.group.units self._ldst_expect = len (network.groups) - 1 # weight update function diff --git a/threshold_vertex.py b/threshold_vertex.py index b2a64b4..3e887dc 100644 --- a/threshold_vertex.py +++ b/threshold_vertex.py @@ -79,9 +79,10 @@ def __init__(self, self._stp_link = "stp_s{}".format (self.group.id) # threshold core-specific parameters - # NOTE: if all-zero w cores are optimised out this need reviewing - self._fwd_sync_expect = len (network.groups) - self._bkp_sync_expect = len (network.groups) + # NOTE: if all-zero w cores are optimised out these need reviewing + self._fwd_sync_expect = len (self._group.w_vertices) + # NOTE: not used any more, may need reviewing if re-introduced + self._bkp_sync_expect = 0 # reserve key space for every link self._n_keys = MLPConstants.KEY_SPACE_SIZE diff --git a/weight_vertex.py b/weight_vertex.py index bc673e6..8557978 100644 --- a/weight_vertex.py +++ b/weight_vertex.py @@ -34,21 +34,45 @@ class WeightVertex( def __init__(self, network, group, - from_group + from_group, + col_blk, + row_blk ): """ """ MachineVertex.__init__(self, label =\ - "w{}_{} core".format ( - group.id, from_group.id) + "w{}_{}_{}_{} core".format ( + group.id, from_group.id, + row_blk, col_blk) ) # application-level data self._network = network self._group = group self._from_group = from_group + self._col_blk = col_blk + self._row_blk = row_blk self._ex_cfg = network._ex_set.example_config + # compute number of rows and columns + if self._row_blk != (self.from_group.partitions - 1): + self._num_rows = MLPConstants.MAX_BLK_UNITS + else: + _r = self.from_group.units % MLPConstants.MAX_BLK_UNITS + if _r == 0: + self._num_rows = MLPConstants.MAX_BLK_UNITS + else: + self._num_rows = _r + + if self._col_blk != (self.group.partitions - 1): + self._num_cols = MLPConstants.MAX_BLK_UNITS + else: + _r = self.group.units % MLPConstants.MAX_BLK_UNITS + if _r == 0: + self._num_cols = MLPConstants.MAX_BLK_UNITS + else: + self._num_cols = _r + # forward, backprop, synchronisation, and link delta summation link partition names self._fwd_link = "fwd_w{}_{}".format (self.group.id, self.from_group.id) @@ -183,6 +207,8 @@ def config (self): { uint num_rows; uint num_cols; + uint row_blk; + uint col_blk; short_fpreal_t learningRate; short_fpreal_t weightDecay; short_fpreal_t momentum; @@ -204,9 +230,11 @@ def config (self): momentum = int (self.momentum *\ (1 << MLPConstants.SHORT_FPREAL_SHIFT)) - return struct.pack ("<2I3hBx", - self.from_group.units, - self.group.units, + return struct.pack ("<4I3hBx", + self._num_rows, + self._num_cols, + self._row_blk, + self._col_blk, learning_rate & 0xffff, weight_decay & 0xffff, momentum & 0xffff, @@ -282,15 +310,18 @@ def generate_data_specification ( # weight matrix is kept in column-major order # and has to be written out in row-major order _wts = self.group.weights[self.from_group] - _nrows = self.from_group.units - _ncols = self.group.units + _nr = self._num_rows + _nc = self._num_cols + _rb = self._row_blk * MLPConstants.MAX_BLK_UNITS + _cb = self._col_blk * MLPConstants.MAX_BLK_UNITS if len (_wts): - for row in range (_nrows): - for col in range (_ncols): - _wt = self.cast_float_to_weight (_wts[col * _nrows + row]) + for _r in range (_nr): + for _c in range (_nc): + _wt = self.cast_float_to_weight ( + _wts[(_cb + _c) * _nr + (_rb + _r)]) spec.write_value (_wt, data_type = DataType.INT32) else: - for _ in range (_nrows * _ncols): + for _ in range (_nr * _nc): spec.write_value (0, data_type = DataType.INT32) # Reserve and write the routing region From 5791c2b65e221a8417bf8509d126467a74eda226 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Tue, 10 Apr 2018 17:47:02 +0100 Subject: [PATCH 05/21] - fixed a bug in the writing to SDRAM of weights in partitioned groups. --- weight_vertex.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weight_vertex.py b/weight_vertex.py index 8557978..4f6f91d 100644 --- a/weight_vertex.py +++ b/weight_vertex.py @@ -310,6 +310,7 @@ def generate_data_specification ( # weight matrix is kept in column-major order # and has to be written out in row-major order _wts = self.group.weights[self.from_group] + _nrows = self.from_group.units _nr = self._num_rows _nc = self._num_cols _rb = self._row_blk * MLPConstants.MAX_BLK_UNITS @@ -318,7 +319,7 @@ def generate_data_specification ( for _r in range (_nr): for _c in range (_nc): _wt = self.cast_float_to_weight ( - _wts[(_cb + _c) * _nr + (_rb + _r)]) + _wts[(_cb + _c) * _nrows + (_rb + _r)]) spec.write_value (_wt, data_type = DataType.INT32) else: for _ in range (_nr * _nc): From 52367fe5560d6bda9789d98a77a4418a85910242 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Thu, 26 Apr 2018 15:14:58 +0100 Subject: [PATCH 06/21] - fixed tick reporting in DEBUG mode. --- c_code/process_i.c | 126 ++++++++-------- c_code/process_s.c | 134 ++++++++--------- c_code/process_t.c | 356 ++++++++++++++++++++++----------------------- c_code/process_w.c | 301 +++++++++++++++++++------------------- 4 files changed, 448 insertions(+), 469 deletions(-) diff --git a/c_code/process_i.c b/c_code/process_i.c index a17f024..264e52f 100644 --- a/c_code/process_i.c +++ b/c_code/process_i.c @@ -19,9 +19,9 @@ // ------------------------------------------------------------------------ void i_process (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "i_process\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "i_process\n"); +#endif // process packet queue // access queue with interrupts disabled @@ -69,12 +69,12 @@ void i_process (uint null0, uint null1) // ------------------------------------------------------------------------ void i_forward_packet (uint key, uint payload) { - #ifdef DEBUG - pkt_recv++; - recv_fwd++; - if (phase != SPINN_FORWARD) - wrng_phs++; - #endif +#ifdef DEBUG + pkt_recv++; + recv_fwd++; + if (phase != SPINN_FORWARD) + wrng_phs++; +#endif // get net index: mask out block, phase and colour data, uint inx = key & SPINN_NET_MASK; @@ -103,17 +103,17 @@ void i_forward_packet (uint key, uint payload) net_tmp = (net_t) i_nets[inx]; } +#ifdef DEBUG_CFG3 + io_printf (IO_BUF, "in[%u]: 0x%08x\n", inx, net_tmp); +#endif + // incorporate net index to the packet key and send, while (!spin1_send_mc_packet ((fwdKey | inx), net_tmp, WITH_PAYLOAD)); - #ifdef DEBUG_CFG3 - io_printf (IO_BUF, "in[%u]: 0x%08x\n", inx, net_tmp); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_fwd++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_fwd++; +#endif // mark net as done, if_done++; @@ -155,12 +155,12 @@ void i_forward_packet (uint key, uint payload) // ------------------------------------------------------------------------ void i_backprop_packet (uint key, uint payload) { - #ifdef DEBUG - pkt_recv++; - recv_bkp++; - if (phase != SPINN_BACKPROP) - wrng_phs++; - #endif +#ifdef DEBUG + pkt_recv++; + recv_bkp++; + if (phase != SPINN_BACKPROP) + wrng_phs++; +#endif // get delta index: mask out block, phase and colour data, uint inx = key & SPINN_DELTA_MASK; @@ -193,17 +193,17 @@ void i_backprop_packet (uint key, uint payload) delta = (delta_t) delta_tmp; } +#ifdef DEBUG_CFG4 + io_printf (IO_BUF, "id[%u]: 0x%08x\n", inx, delta); +#endif + // incorporate delta index to the packet key and send, while (!spin1_send_mc_packet ((bkpKey | inx), delta, WITH_PAYLOAD)); - #ifdef DEBUG_CFG4 - io_printf (IO_BUF, "id[%u]: 0x%08x\n", inx, delta); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_bkp++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_bkp++; +#endif // mark delta as done, ib_done++; @@ -225,17 +225,18 @@ void i_backprop_packet (uint key, uint payload) // ------------------------------------------------------------------------ void if_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "if_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "if_advance_tick\n"); +#endif + +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "if_tick: %d/%d\n", tick, tot_tick); +#endif // prepare for next tick, if_done = 0; - #ifdef DEBUG - tot_tick++; - #endif - // and check if end of example's FORWARD phase if (tick_stop) { @@ -245,10 +246,6 @@ void if_advance_tick (uint null0, uint null1) { // if not done increment tick tick++; - - #ifdef DEBUG - io_printf (IO_BUF, "if_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -260,20 +257,21 @@ void if_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void ib_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "ib_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "ib_advance_tick\n"); +#endif - // prepare for next tick, - ib_done = 0; +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "ib_tick: %d/%d\n", tick, tot_tick); +#endif - #ifdef DEBUG - tot_tick++; - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, "ib_advance_tick - tick: %d, num_ticks: %d\n", tick, num_ticks); +#endif - #ifdef DEBUG_VRB - io_printf (IO_BUF, "ib_advance_tick - tick: %d, num_ticks: %d\n", tick, num_ticks); - #endif + // prepare for next tick, + ib_done = 0; // and check if end of BACKPROP phase if (tick == SPINN_IB_END_TICK) @@ -281,9 +279,9 @@ void ib_advance_tick (uint null0, uint null1) // initialize the tick count tick = SPINN_I_INIT_TICK; - #ifdef TRACE - io_printf (IO_BUF, "w_switch_to_fw\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "w_switch_to_fw\n"); +#endif // switch to FORWARD phase, phase = SPINN_FORWARD; @@ -295,10 +293,6 @@ void ib_advance_tick (uint null0, uint null1) { // if not done decrement tick tick--; - - #ifdef DEBUG - io_printf (IO_BUF, "ib_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -309,9 +303,9 @@ void ib_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void if_advance_event (void) { - #ifdef TRACE - io_printf (IO_BUF, "if_advance_event\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "if_advance_event\n"); +#endif // check if done with ticks if (tick == ncfg.global_max_ticks - 1) @@ -362,9 +356,9 @@ void if_advance_event (void) // ------------------------------------------------------------------------ void i_advance_example (void) { - #ifdef TRACE - io_printf (IO_BUF, "i_advance_example\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "i_advance_example\n"); +#endif // check if done with examples if (++example >= ncfg.num_examples) diff --git a/c_code/process_s.c b/c_code/process_s.c index 6cf213c..bcf8a13 100644 --- a/c_code/process_s.c +++ b/c_code/process_s.c @@ -19,9 +19,9 @@ // ------------------------------------------------------------------------ void s_process (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "s_process\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "s_process\n"); +#endif // process packet queue // access queue with interrupts disabled @@ -82,9 +82,9 @@ void s_process (uint null0, uint null1) // ------------------------------------------------------------------------ void s_ldsa_packet (uint payload) { - #ifdef DEBUG - lda_recv++; - #endif +#ifdef DEBUG + lda_recv++; +#endif // add the received value to the total so far, s_lds_part += (lds_t) payload; @@ -101,9 +101,9 @@ void s_ldsa_packet (uint payload) { while (!spin1_send_mc_packet (ldstKey, s_lds_part, WITH_PAYLOAD)); - #ifdef DEBUG - ldt_sent++; - #endif +#ifdef DEBUG + ldt_sent++; +#endif } // access synchronisation semaphore with interrupts disabled @@ -140,9 +140,9 @@ void s_ldsa_packet (uint payload) // ------------------------------------------------------------------------ void s_ldst_packet (uint payload) { - #ifdef DEBUG - ldt_recv++; - #endif +#ifdef DEBUG + ldt_recv++; +#endif // add the received value to the total so far, s_lds_part += (lds_t) payload; @@ -190,12 +190,12 @@ void s_ldst_packet (uint payload) // ------------------------------------------------------------------------ void s_forward_packet (uint key, uint payload) { - #ifdef DEBUG - pkt_recv++; - recv_fwd++; - if (phase != SPINN_FORWARD) - wrng_phs++; - #endif +#ifdef DEBUG + pkt_recv++; + recv_fwd++; + if (phase != SPINN_FORWARD) + wrng_phs++; +#endif // get net index: mask out block and phase data, uint inx = key & SPINN_NET_MASK; @@ -229,17 +229,17 @@ void s_forward_packet (uint key, uint payload) net_tmp = (net_t) s_nets[clr][inx]; } +#ifdef DEBUG_CFG3 + io_printf (IO_BUF, "sn[%u]: 0x%08x\n", inx, net_tmp); +#endif + // incorporate net index to the packet key and send, while (!spin1_send_mc_packet ((fwdKey | inx), net_tmp, WITH_PAYLOAD)); - #ifdef DEBUG_CFG3 - io_printf (IO_BUF, "sn[%u]: 0x%08x\n", inx, net_tmp); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_fwd++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_fwd++; +#endif // prepare for next tick, s_nets[clr][inx] = 0; @@ -289,12 +289,12 @@ void s_forward_packet (uint key, uint payload) // ------------------------------------------------------------------------ void s_backprop_packet (uint key, uint payload) { - #ifdef DEBUG - pkt_recv++; - recv_bkp++; - if (phase != SPINN_BACKPROP) - wrng_phs++; - #endif +#ifdef DEBUG + pkt_recv++; + recv_bkp++; + if (phase != SPINN_BACKPROP) + wrng_phs++; +#endif // get error index: mask out block, phase and colour data, uint inx = key & SPINN_ERROR_MASK; @@ -332,17 +332,17 @@ void s_backprop_packet (uint key, uint payload) } */ +#ifdef DEBUG_CFG4 + io_printf (IO_BUF, "se[%u]: 0x%08x\n", inx, error); +#endif + // incorporate error index to the packet key and send, while (!spin1_send_mc_packet ((bkpKey | inx), error, WITH_PAYLOAD)); - #ifdef DEBUG_CFG4 - io_printf (IO_BUF, "se[%u]: 0x%08x\n", inx, error); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_bkp++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_bkp++; +#endif // prepare for next tick, s_errors[clr][inx] = 0; @@ -413,13 +413,14 @@ void s_backprop_packet (uint key, uint payload) // ------------------------------------------------------------------------ void sf_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "sf_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "sf_advance_tick\n"); +#endif - #ifdef DEBUG - tot_tick++; - #endif +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "sf_tick: %d/%d\n", tick, tot_tick); +#endif // and check if end of example's FORWARD phase if (tick_stop) @@ -430,10 +431,6 @@ void sf_advance_tick (uint null0, uint null1) { // if not done increment tick tick++; - - #ifdef DEBUG - io_printf (IO_BUF, "sf_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -445,26 +442,27 @@ void sf_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void sb_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "sb_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "sb_advance_tick\n"); +#endif + +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "sb_tick: %d/%d\n", tick, tot_tick); +#endif // prepare for next tick, sb_done = 0; - #ifdef DEBUG - tot_tick++; - #endif - // and check if end of BACKPROP phase if (tick == SPINN_SB_END_TICK) { // initialize the tick count tick = SPINN_S_INIT_TICK; - #ifdef TRACE - io_printf (IO_BUF, "s_switch_to_fw\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "s_switch_to_fw\n"); +#endif // switch to FORWARD phase, phase = SPINN_FORWARD; @@ -476,10 +474,6 @@ void sb_advance_tick (uint null0, uint null1) { // if not done decrement tick tick--; - - #ifdef DEBUG - io_printf (IO_BUF, "sb_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -490,9 +484,9 @@ void sb_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void sf_advance_event (void) { - #ifdef TRACE - io_printf (IO_BUF, "sf_advance_event\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "sf_advance_event\n"); +#endif // check if done with ticks if (tick == ncfg.global_max_ticks - 1) @@ -535,9 +529,9 @@ void sf_advance_event (void) // ------------------------------------------------------------------------ void s_advance_example (void) { - #ifdef TRACE - io_printf (IO_BUF, "s_advance_example\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "s_advance_example\n"); +#endif // check if done with examples if (++example >= ncfg.num_examples) diff --git a/c_code/process_t.c b/c_code/process_t.c index 791b026..a06c46d 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -19,6 +19,10 @@ // ------------------------------------------------------------------------ void tf_process (uint null0, uint null1) { +#ifdef TRACE + io_printf (IO_BUF, "tb_process\n"); +#endif + // process packet queue // access queue with interrupts disabled uint cpsr = spin1_int_disable (); @@ -55,6 +59,13 @@ void tf_process (uint null0, uint null1) store_outputs (inx); } +#ifdef DEBUG_VRB + io_printf (IO_BUF, "o[%2d]:%11.7f (0x%08x)\n", inx, + SPINN_CONV_TO_PRINT(activation, SPINN_SHORT_ACTIV_SHIFT), + activation + ); +#endif + // incorporate output index into packet key and send to next stage, while (!spin1_send_mc_packet ((fwdKey | inx), (uint) activation, @@ -62,17 +73,10 @@ void tf_process (uint null0, uint null1) ) ); - #ifdef DEBUG_VRB - io_printf (IO_BUF, "o[%2d]:%11.7f (0x%08x)\n", inx, - SPINN_CONV_TO_PRINT(activation, SPINN_SHORT_ACTIV_SHIFT), - activation - ); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_fwd++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_fwd++; +#endif // evaluate stop criterion, if (tcfg.output_grp) @@ -198,9 +202,9 @@ void tf_process (uint null0, uint null1) // ------------------------------------------------------------------------ void tb_process (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "tb_process\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "tb_process\n"); +#endif // compute deltas based on pre-computed errors, //TODO: this needs checking! @@ -233,24 +237,24 @@ void tb_process (uint null0, uint null1) restore_outputs (inx, tick - 1); +#ifdef DEBUG_CFG4 + io_printf (IO_BUF, "td[%u]: 0x%08x\n", inx, delta); +#endif + +#ifdef DEBUG_VRB + io_printf(IO_BUF, "d[%2d] = %10.7f (%08x)\n", inx, + SPINN_CONV_TO_PRINT (delta, SPINN_DELTA_SHIFT), + delta + ); +#endif + // send delta to input core for further processing while (!spin1_send_mc_packet ((bkpKey | inx), (uint) delta, WITH_PAYLOAD)); - #ifdef DEBUG_CFG4 - io_printf (IO_BUF, "td[%u]: 0x%08x\n", inx, delta); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_bkp++; - #endif - - #ifdef DEBUG_VRB - io_printf(IO_BUF, "d[%2d] = %10.7f (%08x)\n", inx, - SPINN_CONV_TO_PRINT (delta, SPINN_DELTA_SHIFT), - delta - ); - #endif +#ifdef DEBUG + pkt_sent++; + sent_bkp++; +#endif } // access synchronisation semaphore with interrupts disabled @@ -267,9 +271,9 @@ void tb_process (uint null0, uint null1) // and advance tick //TODO: check if need to schedule or can simply call - #ifdef TRACE_VRB - io_printf (IO_BUF, "tbp calling tb_advance_tick\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "tbp calling tb_advance_tick\n"); +#endif tb_advance_tick (NULL, NULL); } @@ -291,17 +295,22 @@ void tb_process (uint null0, uint null1) // ------------------------------------------------------------------------ void tf_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "tf_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "tf_advance_tick\n"); +#endif + +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "tf_tick: %d/%d\n", tick, tot_tick); +#endif + +#if SPINN_OUTPUT_HISTORY == TRUE + //TODO: dump outputs to SDRAM for record keeping, +#endif // initialize scoreboard for next tick, tf_arrived = 0; - #if SPINN_OUTPUT_HISTORY == TRUE - //TODO: dump outputs to SDRAM for record keeping, - #endif - // if requested report outputs to host, if (tcfg.write_out) { @@ -322,10 +331,6 @@ void tf_advance_tick (uint null0, uint null1) } } - #ifdef DEBUG - tot_tick++; - #endif - // and check if done with FORWARD phase if (tick_stop) { @@ -341,10 +346,6 @@ void tf_advance_tick (uint null0, uint null1) // if not done increment ticks tick++; ev_tick++; - - #ifdef DEBUG - io_printf (IO_BUF, "tf_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -356,13 +357,14 @@ void tf_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void tb_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "tb_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "tb_advance_tick\n"); +#endif - #ifdef DEBUG - tot_tick++; - #endif +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "tb_tick: %d/%d\n", tick, tot_tick); +#endif // update pointer to processing unit outputs, tb_procs = 1 - tb_procs; @@ -396,10 +398,6 @@ void tb_advance_tick (uint null0, uint null1) // and trigger computation spin1_schedule_callback (tb_process, NULL, NULL, SPINN_TB_PROCESS_P); - - #ifdef DEBUG - io_printf (IO_BUF, "tb_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -410,9 +408,9 @@ void tb_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void tf_advance_event (void) { - #ifdef TRACE - io_printf (IO_BUF, "tf_advance_event\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "tf_advance_event\n"); +#endif // check if done with ticks if (tick == ncfg.global_max_ticks - 1) @@ -491,9 +489,9 @@ void tf_advance_event (void) // ------------------------------------------------------------------------ void t_advance_example (void) { - #ifdef TRACE - io_printf (IO_BUF, "t_advance_example\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "t_advance_example\n"); +#endif // check if done with examples //TODO: alternative algorithms for chosing example order! @@ -519,9 +517,9 @@ void t_advance_example (void) ) ); - #ifdef DEBUG - stn_sent++; - #endif +#ifdef DEBUG + stn_sent++; +#endif //done spin1_exit (SPINN_NO_ERROR); @@ -623,9 +621,9 @@ void t_advance_example (void) // ------------------------------------------------------------------------ void t_switch_to_fw (void) { - #ifdef TRACE - io_printf (IO_BUF, "t_switch_to_fw\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "t_switch_to_fw\n"); +#endif // access queues with interrupts disabled uint cpsr = spin1_int_disable (); @@ -658,9 +656,9 @@ void t_switch_to_fw (void) // ------------------------------------------------------------------------ void t_switch_to_bp (void) { - #ifdef TRACE - io_printf (IO_BUF, "t_switch_to_bp\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "t_switch_to_bp\n"); +#endif // access flags and queues with interrupts disabled uint cpsr = spin1_int_disable (); @@ -692,9 +690,9 @@ void t_switch_to_bp (void) // ------------------------------------------------------------------------ void tf_send_stop (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "tf_send_stop\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "tf_send_stop\n"); +#endif // "aggregate" criteria, tf_stop_crit = tf_stop_crit && tf_chain_prev; @@ -709,9 +707,9 @@ void tf_send_stop (uint null0, uint null1) tick_stop = tf_stop_crit; } - #ifdef DEBUG_VRB - io_printf (IO_BUF, "M:%d t:%d sc:%x\n", max_ticks, ev_tick, tf_stop_crit); - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, "M:%d t:%d sc:%x\n", max_ticks, ev_tick, tf_stop_crit); +#endif // FORWARD aggregated criterion, while (!spin1_send_mc_packet ((tf_stop_key | tf_stop_crit), @@ -720,9 +718,9 @@ void tf_send_stop (uint null0, uint null1) ) ); - #ifdef DEBUG - stp_sent++; - #endif +#ifdef DEBUG + stp_sent++; +#endif // and initialise criterion for next tick tf_stop_crit = TRUE; @@ -738,9 +736,9 @@ void tf_send_stop (uint null0, uint null1) // ------------------------------------------------------------------------ void t_init_outputs (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "t_init_outputs\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "t_init_outputs\n"); +#endif // initialize every unit output and send for processing for (uint i = 0; i < tcfg.num_units; i++) @@ -765,6 +763,10 @@ void t_init_outputs (uint null0, uint null1) t_last_integr_output_deriv[i] = 0; } +#ifdef DEBUG_CFG3 + io_printf (IO_BUF, "to[%u]: 0x%08x\n", i, t_outputs[i]); +#endif + // and send unit output to weight cores while (!spin1_send_mc_packet ((fwdKey | i), (uint) t_outputs[i], @@ -772,14 +774,10 @@ void t_init_outputs (uint null0, uint null1) ) ); - #ifdef DEBUG_CFG3 - io_printf (IO_BUF, "to[%u]: 0x%08x\n", i, t_outputs[i]); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_fwd++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_fwd++; +#endif } } // ------------------------------------------------------------------------ @@ -800,15 +798,15 @@ void t_init_outputs (uint null0, uint null1) // ------------------------------------------------------------------------ void compute_out (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "compute_out\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "compute_out\n"); +#endif - #ifdef DEBUG_VRB - char* group; - group = (tcfg.input_grp) ? "Input" : ((tcfg.output_grp) ? "Output" : ((tcfg.num_units == 1) ? "Bias" : "Hidden")); - io_printf (IO_BUF, "compute_out - Group: %s - Example: %d - Tick: %d, Unit: %d\n", group, example, tick, inx); - #endif +#ifdef DEBUG_VRB + char* group; + group = (tcfg.input_grp) ? "Input" : ((tcfg.output_grp) ? "Output" : ((tcfg.num_units == 1) ? "Bias" : "Hidden")); + io_printf (IO_BUF, "compute_out - Group: %s - Example: %d - Tick: %d, Unit: %d\n", group, example, tick, inx); +#endif // initialize the array element where to store the output value for the t_outputs[inx] = 0; @@ -827,9 +825,9 @@ void compute_out (uint inx) // using the appropriate function if (ncfg.training && tcfg.output_grp) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "compute output deriv\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "compute output deriv\n"); +#endif // if the error function to be called is not null, // compute the output derivative @@ -857,9 +855,9 @@ void compute_out (uint inx) // ------------------------------------------------------------------------ void store_targets (uint inx) { - #ifdef TRACE - io_printf (IO_BUF, "store_targets\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "store_targets\n"); +#endif t_target_history[(tick * tcfg.num_units) + inx] = tt[t_it_idx + inx]; } @@ -871,9 +869,9 @@ void store_targets (uint inx) // ------------------------------------------------------------------------ void store_output_deriv (uint inx) { - #ifdef TRACE - io_printf (IO_BUF, "store_output_deriv\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "store_output_deriv\n"); +#endif t_output_deriv_history[(tick * tcfg.num_units) + inx] = t_output_deriv[inx]; } @@ -885,9 +883,9 @@ void store_output_deriv (uint inx) // ------------------------------------------------------------------------ void restore_output_deriv (uint inx, uint tick) { - #ifdef TRACE - io_printf (IO_BUF, "restore_output_deriv\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "restore_output_deriv\n"); +#endif t_output_deriv[inx] = t_output_deriv_history[(tick * tcfg.num_units) + inx]; @@ -900,9 +898,9 @@ void restore_output_deriv (uint inx, uint tick) // ------------------------------------------------------------------------ void store_nets (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "store_nets\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "store_nets\n"); +#endif t_net_history[(tick * tcfg.num_units) + inx] = t_nets[inx]; } @@ -914,9 +912,9 @@ void store_nets (uint inx) // ------------------------------------------------------------------------ void restore_nets (uint inx, uint tick) { - #ifdef TRACE +#ifdef TRACE io_printf (IO_BUF, "restore_nets\n"); - #endif +#endif t_nets[inx] = t_net_history[((tick * tcfg.num_units) + inx)]; } @@ -927,9 +925,9 @@ void restore_nets (uint inx, uint tick) // ------------------------------------------------------------------------ void store_outputs (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "store_outputs\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "store_outputs\n"); +#endif t_output_history[(tick * tcfg.num_units) + inx] = t_outputs[inx]; } @@ -940,9 +938,9 @@ void store_outputs (uint inx) // ------------------------------------------------------------------------ void restore_outputs (uint inx, uint tick) { - #ifdef TRACE - io_printf (IO_BUF, "restore_outputs\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "restore_outputs\n"); +#endif t_outputs[inx] = t_output_history[((tick * tcfg.num_units) + inx)]; } @@ -954,9 +952,9 @@ void restore_outputs (uint inx, uint tick) // ------------------------------------------------------------------------ void out_logistic (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_logistic\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_logistic\n"); +#endif // compute the sigmoid using a lookup table and an interpolation function t_outputs[inx] = sigmoid (t_nets[inx]); @@ -969,9 +967,9 @@ void out_logistic (uint inx) // ------------------------------------------------------------------------ void out_integr (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_integr\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_integr\n"); +#endif // s4.27 activation_t last_output = t_last_integr_output[inx]; @@ -1019,9 +1017,9 @@ void out_integr (uint inx) // ------------------------------------------------------------------------ void out_hard_clamp (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_hard_clamp\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_hard_clamp\n"); +#endif // compute only if input is not NaN if (it[t_it_idx + inx] != SPINN_ACTIV_NaN) @@ -1049,9 +1047,9 @@ void out_hard_clamp (uint inx) // ------------------------------------------------------------------------ void out_bias (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_bias\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_bias\n"); +#endif // set output value to 1 t_outputs[inx] = SPINN_ACTIV_ONE; @@ -1113,15 +1111,15 @@ void out_weak_clamp (uint inx) // ------------------------------------------------------------------------ void compute_out_back (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "compute_out_back\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "compute_out_back\n"); +#endif - #ifdef DEBUG_VRB - char* group; - group = (tcfg.input_grp) ? "Input" : ((tcfg.output_grp) ? "Output" : ((tcfg.num_units == 1) ? "Bias" : "Hidden")); - io_printf (IO_BUF, "compute_out_back - Group: %s - Example: %d - Tick: %d - Unit: %d\n", group, example, tick, inx); - #endif +#ifdef DEBUG_VRB + char* group; + group = (tcfg.input_grp) ? "Input" : ((tcfg.output_grp) ? "Output" : ((tcfg.num_units == 1) ? "Bias" : "Hidden")); + io_printf (IO_BUF, "compute_out_back - Group: %s - Example: %d - Tick: %d - Unit: %d\n", group, example, tick, inx); +#endif int i; @@ -1146,9 +1144,9 @@ void compute_out_back (uint inx) // ------------------------------------------------------------------------ void out_logistic_back (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_logistic_back\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_logistic_back\n"); +#endif // compute o * (1 - o), // s36.27 = s4.27 * s4.27 @@ -1179,9 +1177,9 @@ void out_logistic_back (uint inx) // ------------------------------------------------------------------------ void out_integr_back (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_integr_back\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_integr_back\n"); +#endif long_deriv_t last_output_deriv = t_last_integr_output_deriv[inx]; @@ -1206,9 +1204,9 @@ void out_integr_back (uint inx) // ------------------------------------------------------------------------ void out_hard_clamp_back (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_hard_clamp_back\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_hard_clamp_back\n"); +#endif /* short_activ_t * tmp = t_out_hard_clamp_data + tick * tcfg.num_units; @@ -1225,9 +1223,9 @@ void out_hard_clamp_back (uint inx) // ------------------------------------------------------------------------ void out_weak_clamp_back (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_weak_clamp_back\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_weak_clamp_back\n"); +#endif } // ------------------------------------------------------------------------ @@ -1237,9 +1235,9 @@ void out_weak_clamp_back (uint inx) // ------------------------------------------------------------------------ void out_bias_back (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "out_bias_back\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "out_bias_back\n"); +#endif t_output_deriv[inx] = 0; } @@ -1254,9 +1252,9 @@ void out_bias_back (uint inx) // ------------------------------------------------------------------------ int init_out_integr () { - #ifdef TRACE_VRB - io_printf (IO_BUF, "init_out_integr\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "init_out_integr\n"); +#endif // allocate memory for integrator state //NOTE: these variables are initialised in function init_outputs () @@ -1292,9 +1290,9 @@ int init_out_integr () // ------------------------------------------------------------------------ int init_out_hard_clamp () { - #ifdef TRACE_VRB - io_printf (IO_BUF, "init_out_hard_clamp\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "init_out_hard_clamp\n"); +#endif /* if (ncfg.training) @@ -1325,9 +1323,9 @@ int init_out_hard_clamp () // ------------------------------------------------------------------------ int init_out_weak_clamp () { - #ifdef TRACE_VRB - io_printf (IO_BUF, "init_out_weak_clamp\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "init_out_weak_clamp\n"); +#endif /* if (ncfg.training) @@ -1358,9 +1356,9 @@ int init_out_weak_clamp () // ------------------------------------------------------------------------ void std_stop_crit (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "std_stop_crit\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "std_stop_crit\n"); +#endif // evaluate only if target is not NaN if (tt[t_it_idx + inx] != SPINN_ACTIV_NaN) @@ -1386,9 +1384,9 @@ void std_stop_crit (uint inx) // ------------------------------------------------------------------------ void max_stop_crit (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "max_stop_crit\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "max_stop_crit\n"); +#endif // evaluate only if target is not NaN if (tt[t_it_idx + inx] != SPINN_ACTIV_NaN) @@ -1428,9 +1426,9 @@ void max_stop_crit (uint inx) // ------------------------------------------------------------------------ void error_squared (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "error_squared\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "error_squared\n"); +#endif // evaluate only if target is not NaN if (tt[t_it_idx + inx] != SPINN_ACTIV_NaN) @@ -1451,9 +1449,9 @@ void error_squared (uint inx) // ------------------------------------------------------------------------ void error_cross_entropy (uint inx) { - #ifdef TRACE_VRB - io_printf (IO_BUF, "error_cross_entropy\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "error_cross_entropy\n"); +#endif // if the target is defined, compute the output derivative, otherwise set it to 0 if (tt[t_it_idx + inx] != SPINN_ACTIV_NaN) diff --git a/c_code/process_w.c b/c_code/process_w.c index f7aad62..dca5d81 100644 --- a/c_code/process_w.c +++ b/c_code/process_w.c @@ -19,9 +19,9 @@ // ------------------------------------------------------------------------ void wf_process (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "wf_process\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "wf_process\n"); +#endif // compute all net block dot-products and send them for accumulation, for (uint j = 0; j < wcfg.num_cols; j++) @@ -49,17 +49,17 @@ void wf_process (uint null0, uint null1) // representation in 40.23 within the range (-255; 255) can be reduced to 8.23 net_part = (net_t) net_part_tmp; +#ifdef DEBUG_CFG3 + io_printf (IO_BUF, "wn[%u]: 0x%08x\n", j, net_part); +#endif + // incorporate net index to the packet key and send while (!spin1_send_mc_packet ((fwdKey | j), (uint) net_part, WITH_PAYLOAD)); - #ifdef DEBUG_CFG3 - io_printf (IO_BUF, "wn[%u]: 0x%08x\n", j, net_part); - #endif - - #ifdef DEBUG - pkt_sent++; - sent_fwd++; - #endif +#ifdef DEBUG + pkt_sent++; + sent_fwd++; +#endif } // access synchronisation semaphore with interrupts disabled @@ -76,9 +76,9 @@ void wf_process (uint null0, uint null1) // and advance tick //TODO: check if need to schedule or can simply call - #ifdef TRACE_VRB - io_printf (IO_BUF, "wfp calling wf_advance_tick\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "wfp calling wf_advance_tick\n"); +#endif wf_advance_tick (NULL, NULL); } @@ -99,13 +99,13 @@ void wf_process (uint null0, uint null1) // ------------------------------------------------------------------------ void wb_process (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "wb_process\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "wb_process\n"); +#endif - #ifdef PROFILE - io_printf (IO_STD, "tin: %u\n", tc[T2_COUNT]); - #endif +#ifdef PROFILE + io_printf (IO_STD, "tin: %u\n", tc[T2_COUNT]); +#endif // process delta packet queue // access queue with interrupts disabled @@ -190,14 +190,14 @@ void wb_process (uint null0, uint null1) (uint) w_errors[i], WITH_PAYLOAD) ); - #ifdef DEBUG_CFG4 - io_printf (IO_BUF, "we[%u]: 0x%08x\n", i, w_errors[i]); - #endif +#ifdef DEBUG + pkt_sent++; + sent_bkp++; +#endif - #ifdef DEBUG - pkt_sent++; - sent_bkp++; - #endif +#ifdef DEBUG_CFG4 + io_printf (IO_BUF, "we[%u]: 0x%08x\n", i, w_errors[i]); +#endif // and initialize error for next tick w_errors[i] = 0; @@ -217,9 +217,9 @@ void wb_process (uint null0, uint null1) while (!spin1_send_mc_packet (ldsaKey, (uint) link_delta_sum_short, WITH_PAYLOAD) ); - #ifdef DEBUG - lda_sent++; - #endif +#ifdef DEBUG + lda_sent++; +#endif } // if done with all deltas advance tick @@ -253,9 +253,9 @@ void wb_process (uint null0, uint null1) // restore interrupts after flag access, spin1_mode_restore (cpsr); - #ifdef TRACE_VRB - io_printf (IO_BUF, "wbp calling wb_advance_tick\n"); - #endif +#ifdef TRACE_VRB + io_printf (IO_BUF, "wbp calling wb_advance_tick\n"); +#endif //TODO: check if need to schedule or can simply call wb_advance_tick (NULL, NULL); @@ -280,9 +280,9 @@ void wb_process (uint null0, uint null1) // restore interrupts and leave spin1_mode_restore (cpsr); - #ifdef PROFILE - io_printf (IO_STD, "tout: %u\n", tc[T2_COUNT]); - #endif +#ifdef PROFILE + io_printf (IO_STD, "tout: %u\n", tc[T2_COUNT]); +#endif } // ------------------------------------------------------------------------ @@ -295,23 +295,22 @@ void wb_process (uint null0, uint null1) // ------------------------------------------------------------------------ void steepest_update_weights (void) { - #ifdef DEBUG - wght_ups++; - #endif - - #ifdef TRACE - io_printf (IO_BUF, "steepest_update_weights\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "steepest_update_weights\n"); +#endif +#ifdef DEBUG + wght_ups++; +#endif // update weights for (uint j = 0; j < wcfg.num_cols; j++) { for (uint i = 0; i < wcfg.num_rows; i++) { - #ifdef DEBUG_VRB - weight_t old_weight = w_weights[i][j]; - #endif +#ifdef DEBUG_VRB + weight_t old_weight = w_weights[i][j]; +#endif // do not update weights that are 0 -- indicates no connection! if (w_weights[i][j] != 0) @@ -389,22 +388,22 @@ void steepest_update_weights (void) } } - #ifdef DEBUG_VRB - io_printf (IO_BUF, - "[%2d][%2d] wo = %10.7f (0x%08x) wn = %10.7f (0x%08x)\n", - i, j, - SPINN_CONV_TO_PRINT(old_weight, SPINN_WEIGHT_SHIFT), - old_weight, - SPINN_CONV_TO_PRINT(w_weights[i][j], SPINN_WEIGHT_SHIFT), - w_weights[i][j] - ); - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, + "[%2d][%2d] wo = %10.7f (0x%08x) wn = %10.7f (0x%08x)\n", + i, j, + SPINN_CONV_TO_PRINT(old_weight, SPINN_WEIGHT_SHIFT), + old_weight, + SPINN_CONV_TO_PRINT(w_weights[i][j], SPINN_WEIGHT_SHIFT), + w_weights[i][j] + ); +#endif } } - #if SPINN_WEIGHT_HISTORY == TRUE - //TODO: dump weights to SDRAM for record keeping - #endif +#if SPINN_WEIGHT_HISTORY == TRUE + //TODO: dump weights to SDRAM for record keeping +#endif } // ------------------------------------------------------------------------ @@ -416,22 +415,22 @@ void steepest_update_weights (void) // ------------------------------------------------------------------------ void momentum_update_weights (void) { - #ifdef DEBUG - wght_ups++; - #endif +#ifdef TRACE + io_printf (IO_BUF, "momentum_update_weights\n"); +#endif - #ifdef TRACE - io_printf (IO_BUF, "momentum_update_weights\n"); - #endif +#ifdef DEBUG + wght_ups++; +#endif // update weights for (uint j = 0; j < wcfg.num_cols; j++) { for (uint i = 0; i < wcfg.num_rows; i++) { - #ifdef DEBUG_VRB - weight_t old_weight = w_weights[i][j]; - #endif +#ifdef DEBUG_VRB + weight_t old_weight = w_weights[i][j]; +#endif // do not update weights that are 0 -- indicates no connection! if (w_weights[i][j] != 0) @@ -521,22 +520,22 @@ void momentum_update_weights (void) } } - #ifdef DEBUG_VRB - io_printf (IO_BUF, - "[%2d][%2d] wo = %10.7f (0x%08x) wn = %10.7f (0x%08x)\n", - i, j, - SPINN_CONV_TO_PRINT(old_weight, SPINN_WEIGHT_SHIFT), - old_weight, - SPINN_CONV_TO_PRINT(w_weights[i][j], SPINN_WEIGHT_SHIFT), - w_weights[i][j] - ); - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, + "[%2d][%2d] wo = %10.7f (0x%08x) wn = %10.7f (0x%08x)\n", + i, j, + SPINN_CONV_TO_PRINT(old_weight, SPINN_WEIGHT_SHIFT), + old_weight, + SPINN_CONV_TO_PRINT(w_weights[i][j], SPINN_WEIGHT_SHIFT), + w_weights[i][j] + ); +#endif } } - #if SPINN_WEIGHT_HISTORY == TRUE - //TODO: dump weights to SDRAM for record keeping - #endif +#if SPINN_WEIGHT_HISTORY == TRUE + //TODO: dump weights to SDRAM for record keeping +#endif } // ------------------------------------------------------------------------ @@ -549,13 +548,13 @@ void momentum_update_weights (void) // ------------------------------------------------------------------------ void dougsmomentum_update_weights (void) { - #ifdef DEBUG - wght_ups++; - #endif +#ifdef TRACE + io_printf (IO_BUF, "dougsmomentum_update_weights\n"); +#endif - #ifdef TRACE - io_printf (IO_BUF, "dougsmomentum_update_weights\n"); - #endif +#ifdef DEBUG + wght_ups++; +#endif wchange_t scale; @@ -580,9 +579,9 @@ void dougsmomentum_update_weights (void) { for (uint i = 0; i < wcfg.num_rows; i++) { - #ifdef DEBUG_VRB - weight_t old_weight = w_weights[i][j]; - #endif +#ifdef DEBUG_VRB + weight_t old_weight = w_weights[i][j]; +#endif // do not update weights that are 0 -- indicates no connection! if (w_weights[i][j] != 0) @@ -672,22 +671,22 @@ void dougsmomentum_update_weights (void) } } - #ifdef DEBUG_VRB - io_printf (IO_BUF, - "[%2d][%2d] wo = %10.7f (0x%08x) wn = %10.7f (0x%08x)\n", - i, j, - SPINN_CONV_TO_PRINT(old_weight, SPINN_WEIGHT_SHIFT), - old_weight, - SPINN_CONV_TO_PRINT(w_weights[i][j], SPINN_WEIGHT_SHIFT), - w_weights[i][j] - ); - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, + "[%2d][%2d] wo = %10.7f (0x%08x) wn = %10.7f (0x%08x)\n", + i, j, + SPINN_CONV_TO_PRINT(old_weight, SPINN_WEIGHT_SHIFT), + old_weight, + SPINN_CONV_TO_PRINT(w_weights[i][j], SPINN_WEIGHT_SHIFT), + w_weights[i][j] + ); +#endif } } - #if SPINN_WEIGHT_HISTORY == TRUE - //TODO: dump weights to SDRAM for record keeping - #endif +#if SPINN_WEIGHT_HISTORY == TRUE + //TODO: dump weights to SDRAM for record keeping +#endif } // ------------------------------------------------------------------------ @@ -698,9 +697,18 @@ void dougsmomentum_update_weights (void) // ------------------------------------------------------------------------ void wf_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "wf_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "wf_advance_tick\n"); +#endif + +#ifdef DEBUG + //NOTE: tick 0 is not a computation tick + if (tick) + { + tot_tick++; + io_printf (IO_BUF, "wf_tick: %d/%d\n", tick, tot_tick); + } +#endif // change packet key colour, fwdKey ^= SPINN_COLOUR_KEY; @@ -718,16 +726,8 @@ void wf_advance_tick (uint null0, uint null1) // if not increment tick, tick++; - #ifdef DEBUG - tot_tick++; - #endif - // and trigger computation spin1_schedule_callback (wf_process, NULL, NULL, SPINN_WF_PROCESS_P); - - #ifdef DEBUG - io_printf (IO_BUF, "wf_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -739,17 +739,18 @@ void wf_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void wb_advance_tick (uint null0, uint null1) { - #ifdef TRACE - io_printf (IO_BUF, "wb_advance_tick\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "wb_advance_tick\n"); +#endif - #ifdef DEBUG - tot_tick++; - #endif +#ifdef DEBUG + tot_tick++; + io_printf (IO_BUF, "wb_tick: %d/%d\n", tick, tot_tick); +#endif - #ifdef DEBUG_VRB - io_printf (IO_BUF, "wb: num_ticks: %d, tick: %d\n", num_ticks, tick); - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, "wb: num_ticks: %d, tick: %d\n", num_ticks, tick); +#endif // change packet key colour, bkpKey ^= SPINN_COLOUR_KEY; @@ -774,10 +775,6 @@ void wb_advance_tick (uint null0, uint null1) // and restore previous tick outputs restore_outputs (tick - 1); - - #ifdef DEBUG - io_printf (IO_BUF, "wb_tick: %d/%d\n", tick, tot_tick); - #endif } } // ------------------------------------------------------------------------ @@ -788,9 +785,9 @@ void wb_advance_tick (uint null0, uint null1) // ------------------------------------------------------------------------ void wf_advance_event (void) { - #ifdef TRACE - io_printf (IO_BUF, "wf_advance_event\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "wf_advance_event\n"); +#endif // check if done with ticks if (tick == ncfg.global_max_ticks - 1) @@ -837,10 +834,6 @@ void wf_advance_event (void) // if not increment tick, tick++; - #ifdef DEBUG - tot_tick++; - #endif - // and trigger computation spin1_schedule_callback (wf_process, NULL, NULL, SPINN_WF_PROCESS_P); } @@ -853,9 +846,9 @@ void wf_advance_event (void) // ------------------------------------------------------------------------ void w_advance_example (void) { - #ifdef TRACE - io_printf (IO_BUF, "w_advance_example\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "w_advance_example\n"); +#endif // check if done with examples if (++example >= ncfg.num_examples) @@ -866,11 +859,11 @@ void w_advance_example (void) //TODO: should be called or scheduled? wb_update_func (); - #if WEIGHT_HISTORY == TRUE - // send weight history to host - //TODO: write this function! - //send_weights_to_host (); - #endif +#if WEIGHT_HISTORY == TRUE + // send weight history to host + //TODO: write this function! + //send_weights_to_host (); +#endif } // check if done with epochs @@ -907,9 +900,9 @@ void w_advance_example (void) // and send sync packet to allow unit outputs to be sent while (!spin1_send_mc_packet (wf_sync_key, 0, NO_PAYLOAD)); - #ifdef DEBUG - spk_sent++; - #endif +#ifdef DEBUG + spk_sent++; +#endif } // ------------------------------------------------------------------------ @@ -919,9 +912,9 @@ void w_advance_example (void) // ------------------------------------------------------------------------ void w_switch_to_fw (void) { - #ifdef TRACE - io_printf (IO_BUF, "w_switch_to_fw\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "w_switch_to_fw\n"); +#endif // move to new FORWARD phase phase = SPINN_FORWARD; @@ -934,9 +927,9 @@ void w_switch_to_fw (void) // ------------------------------------------------------------------------ void w_switch_to_bp (void) { - #ifdef TRACE - io_printf (IO_BUF, "w_switch_to_bp\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "w_switch_to_bp\n"); +#endif // move to new BACKPROP phase, phase = SPINN_BACKPROP; @@ -959,9 +952,9 @@ void w_switch_to_bp (void) // ------------------------------------------------------------------------ void restore_outputs (uint tick) { - #ifdef TRACE - io_printf (IO_BUF, "restore_outputs\n"); - #endif +#ifdef TRACE + io_printf (IO_BUF, "restore_outputs\n"); +#endif for (uint inx = 0; inx < wcfg.num_rows; inx++) { From 2b0a4e3ebf695317e8e773cd7b229662b98b0af4 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Thu, 26 Apr 2018 17:09:25 +0100 Subject: [PATCH 07/21] - changed timeout mechanism to be based on lack of progress. --- c_code/input.c | 18 +++++++++++++++--- c_code/sum.c | 18 +++++++++++++++--- c_code/threshold.c | 18 +++++++++++++++--- c_code/weight.c | 18 +++++++++++++++--- 4 files changed, 60 insertions(+), 12 deletions(-) diff --git a/c_code/input.c b/c_code/input.c index 497ea17..783e4a6 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -69,6 +69,10 @@ uint min_ticks; // minimum number of ticks in current event uint tick; // current tick in phase uchar tick_stop; // current tick stop decision +uint to_epoch = 0; +uint to_example = 0; +uint to_tick = 0; + // ------------------------------------------------------------------------ // data structures in regions of SDRAM // ------------------------------------------------------------------------ @@ -278,16 +282,24 @@ void done (uint ec) // ------------------------------------------------------------------------ -// timer callback: if the execution takes too long it probably deadlocked. -// Therefore the execution is terminated with SPINN_TIMEOUT_EXIT exit code. +// timer callback: check that there has been progress in execution. +// If no progress has been made terminate with SPINN_TIMEOUT_EXIT exit code. // ------------------------------------------------------------------------ void timeout (uint ticks, uint null) { - if (ticks == ncfg.timeout) + // check if progress has been made + if ((to_epoch == epoch) && (to_example == example) && (to_tick == tick)) { // exit and report timeout spin1_exit (SPINN_TIMEOUT_EXIT); } + else + { + // update checked variables + to_epoch = epoch; + to_example = example; + to_tick = tick; + } } // ------------------------------------------------------------------------ diff --git a/c_code/sum.c b/c_code/sum.c index ec689de..74a5c1f 100644 --- a/c_code/sum.c +++ b/c_code/sum.c @@ -37,6 +37,10 @@ uint min_ticks; // minimum number of ticks in current event uint tick; // current tick in phase uchar tick_stop; // current tick stop decision +uint to_epoch = 0; +uint to_example = 0; +uint to_tick = 0; + // ------------------------------------------------------------------------ // data structures in regions of SDRAM // ------------------------------------------------------------------------ @@ -240,16 +244,24 @@ void done (uint ec) // ------------------------------------------------------------------------ -// timer callback: if the execution takes too long it probably deadlocked. -// Therefore the execution is terminated with SPINN_TIMEOUT_EXIT exit code. +// timer callback: check that there has been progress in execution. +// If no progress has been made terminate with SPINN_TIMEOUT_EXIT exit code. // ------------------------------------------------------------------------ void timeout (uint ticks, uint null) { - if (ticks == ncfg.timeout) + // check if progress has been made + if ((to_epoch == epoch) && (to_example == example) && (to_tick == tick)) { // exit and report timeout spin1_exit (SPINN_TIMEOUT_EXIT); } + else + { + // update checked variables + to_epoch = epoch; + to_example = example; + to_tick = tick; + } } // ------------------------------------------------------------------------ diff --git a/c_code/threshold.c b/c_code/threshold.c index c91fe3e..eed2966 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -94,6 +94,10 @@ uint ev_tick; // current tick in event uchar tick_stop; // current tick stop decision uchar network_stop; // network_stop decision +uint to_epoch = 0; +uint to_example = 0; +uint to_tick = 0; + // ------------------------------------------------------------------------ // data structures in regions of SDRAM // ------------------------------------------------------------------------ @@ -403,16 +407,24 @@ void done (uint ec) // ------------------------------------------------------------------------ -// timer callback: if the execution takes too long it probably deadlocked. -// Therefore the execution is terminated with SPINN_TIMEOUT_EXIT exit code. +// timer callback: check that there has been progress in execution. +// If no progress has been made terminate with SPINN_TIMEOUT_EXIT exit code. // ------------------------------------------------------------------------ void timeout (uint ticks, uint null) { - if (ticks == ncfg.timeout) + // check if progress has been made + if ((to_epoch == epoch) && (to_example == example) && (to_tick == tick)) { // exit and report timeout spin1_exit (SPINN_TIMEOUT_EXIT); } + else + { + // update checked variables + to_epoch = epoch; + to_example = example; + to_tick = tick; + } } // ------------------------------------------------------------------------ diff --git a/c_code/weight.c b/c_code/weight.c index 234e7a7..03e348e 100644 --- a/c_code/weight.c +++ b/c_code/weight.c @@ -50,6 +50,10 @@ uint min_ticks; // minimum number of ticks in current event uint tick; // current tick in phase uchar tick_stop; // current tick stop decision +uint to_epoch = 0; +uint to_example = 0; +uint to_tick = 0; + // ------------------------------------------------------------------------ // data structures in regions of SDRAM // ------------------------------------------------------------------------ @@ -266,16 +270,24 @@ void done (uint ec) // ------------------------------------------------------------------------ -// timer callback: if the execution takes too long it probably deadlocked. -// Therefore the execution is terminated with SPINN_TIMEOUT_EXIT exit code. +// timer callback: check that there has been progress in execution. +// If no progress has been made terminate with SPINN_TIMEOUT_EXIT exit code. // ------------------------------------------------------------------------ void timeout (uint ticks, uint null) { - if (ticks == ncfg.timeout) + // check if progress has been made + if ((to_epoch == epoch) && (to_example == example) && (to_tick == tick)) { // exit and report timeout spin1_exit (SPINN_TIMEOUT_EXIT); } + else + { + // update checked variables + to_epoch = epoch; + to_example = example; + to_tick = tick; + } } // ------------------------------------------------------------------------ From 41b5517678f6c05ab6c29678f507b667e5446957 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Thu, 26 Apr 2018 18:05:50 +0100 Subject: [PATCH 08/21] code cleanup to help with debugging. --- c_code/comms_i.c | 6 +++--- c_code/comms_s.c | 6 +++--- c_code/comms_t.c | 14 +++++++------- c_code/comms_w.c | 18 ++++++++--------- c_code/init_i.c | 2 +- c_code/init_s.c | 4 ++-- c_code/init_t.c | 4 ++-- c_code/init_w.c | 4 ++-- c_code/input.c | 2 +- c_code/mlp_externs.h | 14 +++++++------- c_code/process_i.c | 19 +++++++----------- c_code/process_s.c | 46 +++++++++++++++++++------------------------- c_code/process_t.c | 33 +++++++++++++------------------ c_code/process_w.c | 30 ++++++++++++----------------- c_code/sum.c | 4 ++-- c_code/threshold.c | 4 ++-- c_code/weight.c | 6 +++--- 17 files changed, 96 insertions(+), 120 deletions(-) diff --git a/c_code/comms_i.c b/c_code/comms_i.c index f1a650a..762ec7b 100644 --- a/c_code/comms_i.c +++ b/c_code/comms_i.c @@ -32,10 +32,10 @@ void i_receivePacket (uint key, uint payload) #endif // check if all threads done - if (if_thrds_done == 0) + if (if_thrds_pend == 0) { // if done initialize semaphore, - if_thrds_done = 1; + if_thrds_pend = 1; // and advance tick spin1_schedule_callback (if_advance_tick, NULL, NULL, SPINN_I_TICK_P); @@ -43,7 +43,7 @@ void i_receivePacket (uint key, uint payload) else { // if not done report processing thread done - if_thrds_done -= 1; + if_thrds_pend -= 1; } return; diff --git a/c_code/comms_s.c b/c_code/comms_s.c index 83886a4..9cf03cb 100644 --- a/c_code/comms_s.c +++ b/c_code/comms_s.c @@ -32,10 +32,10 @@ void s_receivePacket (uint key, uint payload) #endif // check if all threads done - if (sf_thrds_done == 0) + if (sf_thrds_pend == 0) { // if done initialise semaphore - sf_thrds_done = 1; + sf_thrds_pend = 1; // and advance tick spin1_schedule_callback (sf_advance_tick, NULL, NULL, SPINN_S_TICK_P); @@ -43,7 +43,7 @@ void s_receivePacket (uint key, uint payload) else { // if not done report processing thread done - sf_thrds_done -= 1; + sf_thrds_pend -= 1; } return; diff --git a/c_code/comms_t.c b/c_code/comms_t.c index d2b2e2d..71499a4 100644 --- a/c_code/comms_t.c +++ b/c_code/comms_t.c @@ -87,10 +87,10 @@ void t_stopPacket (uint key) #endif // check if all threads done - if (tf_thrds_done == 0) + if (tf_thrds_pend == 0) { // initialise semaphore - tf_thrds_done = 1; + tf_thrds_pend = 1; // and advance tick spin1_schedule_callback (tf_advance_tick, NULL, NULL, SPINN_TF_TICK_P); @@ -98,7 +98,7 @@ void t_stopPacket (uint key) else { // if not done report stop thread done - tf_thrds_done -= 1; + tf_thrds_pend -= 1; } } // ------------------------------------------------------------------------ @@ -304,11 +304,11 @@ void t_backpropPacket (uint key, uint payload) // update pointer to received errors, tb_comms = 1 - tb_comms; - // and check if other threads are done, - if (tb_thrds_done == 0) + // and check if all other threads are done, + if (tb_thrds_pend == 0) { // if done initialise synchronisation semaphore, - tb_thrds_done = 1; + tb_thrds_pend = 1; // and advance tick #ifdef TRACE_VRB @@ -320,7 +320,7 @@ void t_backpropPacket (uint key, uint payload) else { // if not done report comms thread done - tb_thrds_done -= 1; + tb_thrds_pend -= 1; } } } diff --git a/c_code/comms_w.c b/c_code/comms_w.c index d744b3e..27953f1 100644 --- a/c_code/comms_w.c +++ b/c_code/comms_w.c @@ -85,10 +85,10 @@ void w_stopPacket (uint key) #endif // check if all threads done - if (wf_thrds_done == 0) + if (wf_thrds_pend == 0) { // if done initialize synchronization semaphore, - wf_thrds_done = 2; + wf_thrds_pend = 2; // and advance tick #ifdef TRACE_VRB @@ -100,7 +100,7 @@ void w_stopPacket (uint key) else { // if not done report stop thread done - wf_thrds_done -= 1; + wf_thrds_pend -= 1; } } // ------------------------------------------------------------------------ @@ -131,10 +131,10 @@ void w_ldsrPacket (uint payload) w_lds_final = (lds_t) payload; // check if all threads done - if (wb_thrds_done == 0) + if (wb_thrds_pend == 0) { //NOTE: no need to initialize semaphore - //wb_thrds_done = 0; + //wb_thrds_pend = 0; #ifdef TRACE_VRB io_printf (IO_BUF, "ldsr calling wb_advance_tick\n"); @@ -147,7 +147,7 @@ void w_ldsrPacket (uint payload) else { // if not done report processing thread done, - wb_thrds_done -= 1; + wb_thrds_pend -= 1; } } // ------------------------------------------------------------------------ @@ -190,10 +190,10 @@ void w_forwardPacket (uint key, uint payload) wf_comms = 1 - wf_comms; // and check if other threads are done, - if (wf_thrds_done == 0) + if (wf_thrds_pend == 0) { // if done initialize synchronization semaphore, - wf_thrds_done = 2; + wf_thrds_pend = 2; // and advance tick #ifdef TRACE_VRB @@ -205,7 +205,7 @@ void w_forwardPacket (uint key, uint payload) else { // if not done report comms thread done - wf_thrds_done -= 1; + wf_thrds_pend -= 1; } } } diff --git a/c_code/init_i.c b/c_code/init_i.c index 8e031ef..a901f9a 100644 --- a/c_code/init_i.c +++ b/c_code/init_i.c @@ -59,7 +59,7 @@ uint i_init (void) ib_done = 0; // initialize synchronization semaphores - if_thrds_done = 1; + if_thrds_pend = 1; // initialize processing thread flag i_active = FALSE; diff --git a/c_code/init_s.c b/c_code/init_s.c index 56f8cc6..c73447a 100644 --- a/c_code/init_s.c +++ b/c_code/init_s.c @@ -107,8 +107,8 @@ uint s_init (void) s_ldst_arrived = 0; // initialise synchronisation semaphores - sf_thrds_done = 1; - sb_thrds_done = 0; + sf_thrds_pend = 1; + sb_thrds_pend = 0; // initialize processing thread flag s_active = FALSE; diff --git a/c_code/init_t.c b/c_code/init_t.c index 145bd7a..c16f40b 100644 --- a/c_code/init_t.c +++ b/c_code/init_t.c @@ -157,8 +157,8 @@ uint t_init (void) tb_arrived = 0; // initialise synchronisation semaphores - tf_thrds_done = 1; - tb_thrds_done = 1; + tf_thrds_pend = 1; + tb_thrds_pend = 1; // initialise stop function and related flags if (tcfg.output_grp) diff --git a/c_code/init_w.c b/c_code/init_w.c index 955382c..5c24722 100644 --- a/c_code/init_w.c +++ b/c_code/init_w.c @@ -195,8 +195,8 @@ uint w_init (void) wf_comms = 1; // initialize synchronization semaphores - wf_thrds_done = 0; // just wait for initial unit outputs - wb_thrds_done = 0; // just wait for initial deltas + wf_thrds_pend = 0; // just wait for initial unit outputs + wb_thrds_pend = 0; // just wait for initial deltas // initialize processing thread flag wb_active = FALSE; diff --git a/c_code/input.c b/c_code/input.c index 783e4a6..167f784 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -107,7 +107,7 @@ uint i_it_idx; // index into current inputs/targets // FORWARD phase specific // (net processing) scoreboard_t if_done; // current tick net computation done -uint if_thrds_done; // sync. semaphore: proc & stop +uint if_thrds_pend; // sync. semaphore: proc & stop // BACKPROP phase specific // (delta processing) diff --git a/c_code/mlp_externs.h b/c_code/mlp_externs.h index e9b902e..a24bc69 100644 --- a/c_code/mlp_externs.h +++ b/c_code/mlp_externs.h @@ -63,11 +63,11 @@ extern lds_t w_lds_final; // final link delta sum extern uint wf_procs; // pointer to processing unit outputs extern uint wf_comms; // pointer to receiving unit outputs extern scoreboard_t wf_arrived; // keeps track of received unit outputs -extern uint wf_thrds_done; // sync. semaphore: comms, proc & stop +extern uint wf_thrds_pend; // sync. semaphore: comms, proc & stop extern uint wf_sync_key; // FORWARD processing can start extern uchar wb_active; // processing deltas from queue? extern scoreboard_t wb_arrived; // keeps track of received deltas -extern uint wb_thrds_done; // sync. semaphore: comms, proc & stop +extern uint wb_thrds_pend; // sync. semaphore: comms, proc & stop extern uint wb_sync_key; // BACKPROP processing can start extern weight_update_t wb_update_func; // weight update function @@ -85,10 +85,10 @@ extern uchar s_active; // processing b-d-ps from queue? extern lds_t s_lds_part; // partial link delta sum extern scoreboard_t * sf_arrived[2]; // keep track of expected net b-d-p extern scoreboard_t sf_done; // current tick net computation done -extern uint sf_thrds_done; // sync. semaphore: proc & stop +extern uint sf_thrds_pend; // sync. semaphore: proc & stop extern scoreboard_t * sb_arrived[2]; // keep track of expected error b-d-p extern scoreboard_t sb_done; // current tick error computation done -extern uint sb_thrds_done; // sync. semaphore: proc & stop +extern uint sb_thrds_pend; // sync. semaphore: proc & stop extern scoreboard_t s_ldsa_arrived; // keep track of the number of partial link delta sums extern scoreboard_t s_ldst_arrived; // keep track of the number of link delta sum totals // ------------------------------------------------------------------------ @@ -110,7 +110,7 @@ extern pkt_queue_t i_pkt_queue; // queue to hold received nets/deltas extern uchar i_active; // processing b-d-ps from queue? extern uint i_it_idx; // index into current inputs/targets extern scoreboard_t if_done; // current tick net computation done -extern uint if_thrds_done; // sync. semaphore: proc & stop +extern uint if_thrds_pend; // sync. semaphore: proc & stop extern long_delta_t * ib_init_delta; // initial delta value for every tick extern scoreboard_t ib_done; // current tick delta computation done extern long_net_t * i_last_integr_net; //last integrator output value @@ -148,7 +148,7 @@ extern scoreboard_t t_sync_arrived; // keep track of expected sync packets extern uchar t_sync_rdy; // have expected sync packets arrived? extern sdp_msg_t t_sdp_msg; // SDP message buffer for host comms. extern scoreboard_t tf_arrived; // keep track of expected nets -extern uint tf_thrds_done; // sync. semaphore: proc & stop +extern uint tf_thrds_pend; // sync. semaphore: proc & stop extern uchar tf_chain_prev; // previous daisy chain (DC) value extern uchar tf_chain_init; // previous DC received init extern uchar tf_chain_rdy; // local DC value can be forwarded @@ -162,7 +162,7 @@ extern uint tf_stpn_key; // stop network packet key extern uint tb_procs; // pointer to processing errors extern uint tb_comms; // pointer to receiving errors extern scoreboard_t tb_arrived; // keep track of expected errors -extern uint tb_thrds_done; // sync. semaphore: proc & stop +extern uint tb_thrds_pend; // sync. semaphore: proc & stop extern int t_max_output_unit; // unit with highest output extern int t_max_target_unit; // unit with highest target extern activation_t t_max_output; // highest output value diff --git a/c_code/process_i.c b/c_code/process_i.c index 264e52f..e3f93bd 100644 --- a/c_code/process_i.c +++ b/c_code/process_i.c @@ -124,11 +124,11 @@ void i_forward_packet (uint key, uint payload) // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // check if all threads done - if (if_thrds_done == 0) + // check if all other threads done + if (if_thrds_pend == 0) { // if done initialize semaphore, - if_thrds_done = 1; + if_thrds_pend = 1; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -140,7 +140,7 @@ void i_forward_packet (uint key, uint payload) else { // if not done report processing thread done, - if_thrds_done -= 1; + if_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -307,14 +307,8 @@ void if_advance_event (void) io_printf (IO_BUF, "if_advance_event\n"); #endif - // check if done with ticks - if (tick == ncfg.global_max_ticks - 1) - { - evt = num_events - 1; - } - - // check if done with events - if (++evt >= num_events) + // check if done with example's FORWARD phase + if ((++evt >= num_events) || (tick == ncfg.global_max_ticks - 1)) { // and check if in training mode if (ncfg.training) @@ -329,6 +323,7 @@ void if_advance_event (void) { // if not training, initialize ticks for the next example tick = SPINN_I_INIT_TICK; + // then move to next example i_advance_example (); } diff --git a/c_code/process_s.c b/c_code/process_s.c index bcf8a13..e7da0bd 100644 --- a/c_code/process_s.c +++ b/c_code/process_s.c @@ -109,11 +109,11 @@ void s_ldsa_packet (uint payload) // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // check if all threads done - if (sb_thrds_done == 0) + // check if all other threads done + if (sb_thrds_pend == 0) { // if done initialise semaphore - sb_thrds_done = 0; + sb_thrds_pend = 0; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -125,7 +125,7 @@ void s_ldsa_packet (uint payload) else { // if not done report processing thread done, - sb_thrds_done -= 1; + sb_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -159,11 +159,11 @@ void s_ldst_packet (uint payload) // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // check if all threads done - if (sb_thrds_done == 0) + // check if all other threads done + if (sb_thrds_pend == 0) { // if done initialise semaphore - sb_thrds_done = 0; + sb_thrds_pend = 0; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -175,7 +175,7 @@ void s_ldst_packet (uint payload) else { // if not done report processing thread done, - sb_thrds_done -= 1; + sb_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -257,11 +257,11 @@ void s_forward_packet (uint key, uint payload) // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // check if all threads done - if (sf_thrds_done == 0) + // check if all other threads done + if (sf_thrds_pend == 0) { // if done initialize semaphore - sf_thrds_done = 1; + sf_thrds_pend = 1; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -273,7 +273,7 @@ void s_forward_packet (uint key, uint payload) else { // if not done report processing thread done, - sf_thrds_done -= 1; + sf_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -357,8 +357,8 @@ void s_backprop_packet (uint key, uint payload) // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // check if all threads done - if (sb_thrds_done == 0) + // check if all other threads done + if (sb_thrds_pend == 0) { // if done initialize semaphore: // if we are using Doug's Momentum, and we have reached the end of the @@ -374,17 +374,17 @@ void s_backprop_packet (uint key, uint payload) // pending to 2 if (scfg.is_first_group == 1) { - sb_thrds_done = 2; + sb_thrds_pend = 2; } // for all other groups, set threads pending to 1 else { - sb_thrds_done = 1; + sb_thrds_pend = 1; } } else { - sb_thrds_done = 0; + sb_thrds_pend = 0; } // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -396,7 +396,7 @@ void s_backprop_packet (uint key, uint payload) else { // if not done report processing thread done, - sb_thrds_done -= 1; + sb_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -488,14 +488,8 @@ void sf_advance_event (void) io_printf (IO_BUF, "sf_advance_event\n"); #endif - // check if done with ticks - if (tick == ncfg.global_max_ticks - 1) - { - evt = num_events - 1; - } - - // check if done with events - if (++evt >= num_events) + // check if done with example's FORWARD phase + if ((++evt >= num_events) || (tick == ncfg.global_max_ticks - 1)) { // and check if in training mode if (ncfg.training) diff --git a/c_code/process_t.c b/c_code/process_t.c index a06c46d..dd21f97 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -130,7 +130,7 @@ void tf_process (uint null0, uint null1) uint cpsr = spin1_int_disable (); // report processing thread done, - tf_thrds_done -= 1; + tf_thrds_pend -= 1; // check if chain value can be forwarded if (tf_chain_rdy) @@ -160,11 +160,11 @@ void tf_process (uint null0, uint null1) // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // and check if all threads done - if (tf_thrds_done == 0) + // and check if all other threads done + if (tf_thrds_pend == 0) { // initialize semaphore, - tf_thrds_done = 1; + tf_thrds_pend = 1; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -176,7 +176,7 @@ void tf_process (uint null0, uint null1) else { // if not done report processing thread done, - tf_thrds_done -= 1; + tf_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -260,11 +260,11 @@ void tb_process (uint null0, uint null1) // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // and check if all threads done - if (tb_thrds_done == 0) + // and check if all other threads done + if (tb_thrds_pend == 0) { // if done initialise synchronisation semaphore, - tb_thrds_done = 1; + tb_thrds_pend = 1; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -280,7 +280,7 @@ void tb_process (uint null0, uint null1) else { // if not done report processing thread done, - tb_thrds_done -= 1; + tb_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -412,31 +412,24 @@ void tf_advance_event (void) io_printf (IO_BUF, "tf_advance_event\n"); #endif - // check if done with ticks - if (tick == ncfg.global_max_ticks - 1) - { - evt = num_events - 1; - } - - // check if done with events - if (++evt >= num_events) + // check if done with example's FORWARD phase + if ((++evt >= num_events) || (tick == ncfg.global_max_ticks - 1)) { // check if in training mode if (ncfg.training) { // if training, save the number of ticks num_ticks = tick; + // then do BACKPROP phase t_switch_to_bp (); - - // and stop processing queue in this phase - return; } else { // if not training, initialize ticks for the next example tick = SPINN_T_INIT_TICK; ev_tick = SPINN_T_INIT_TICK; + // then advance to next example t_advance_example (); } diff --git a/c_code/process_w.c b/c_code/process_w.c index dca5d81..00fba6d 100644 --- a/c_code/process_w.c +++ b/c_code/process_w.c @@ -65,11 +65,11 @@ void wf_process (uint null0, uint null1) // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // and check if all threads done - if (wf_thrds_done == 0) + // and check if all other threads done + if (wf_thrds_pend == 0) { // if done initialize synchronization semaphore, - wf_thrds_done = 2; + wf_thrds_pend = 2; // restore interrupts after flag access, spin1_mode_restore (cpsr); @@ -85,7 +85,7 @@ void wf_process (uint null0, uint null1) else { // if not done report processing thread done, - wf_thrds_done -= 1; + wf_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -231,8 +231,8 @@ void wb_process (uint null0, uint null1) // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); - // and check if all threads done - if (wb_thrds_done == 0) + // and check if all other threads done + if (wb_thrds_pend == 0) { // if done initialize synchronization semaphore, // if we are using Doug's Momentum, and we have reached the end of the @@ -243,11 +243,11 @@ void wb_process (uint null0, uint null1) && example == (ncfg.num_examples - 1) && tick == SPINN_WB_END_TICK + 1) { - wb_thrds_done = 1; + wb_thrds_pend = 1; } else { - wb_thrds_done = 0; + wb_thrds_pend = 0; } // restore interrupts after flag access, @@ -263,7 +263,7 @@ void wb_process (uint null0, uint null1) else { // if not done report processing thread done, - wb_thrds_done -= 1; + wb_thrds_pend -= 1; // and restore interrupts after flag access spin1_mode_restore (cpsr); @@ -789,20 +789,14 @@ void wf_advance_event (void) io_printf (IO_BUF, "wf_advance_event\n"); #endif - // check if done with ticks - if (tick == ncfg.global_max_ticks - 1) - { - evt = num_events - 1; - } - - // check if done with events -- end of example's FORWARD phase - if (++evt >= num_events) + // check if done with example's FORWARD phase + if ((++evt >= num_events) || (tick == ncfg.global_max_ticks - 1)) { // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); // initialise synchronisation semaphore, - wf_thrds_done = 0; // no processing and no stop in tick 0 + wf_thrds_pend = 0; // no processing and no stop in tick 0 // restore interrupts after flag access, spin1_mode_restore (cpsr); diff --git a/c_code/sum.c b/c_code/sum.c index 74a5c1f..4088161 100644 --- a/c_code/sum.c +++ b/c_code/sum.c @@ -69,13 +69,13 @@ lds_t s_lds_part; // partial link delta sum // (net computation) scoreboard_t * sf_arrived[2]; // keep track of expected net b-d-p scoreboard_t sf_done; // current tick net computation done -uint sf_thrds_done; // sync. semaphore: proc & stop +uint sf_thrds_pend; // sync. semaphore: proc & stop // BACKPROP phase specific // (error computation) scoreboard_t * sb_arrived[2]; // keep track of expected error b-d-p scoreboard_t sb_done; // current tick error computation done -uint sb_thrds_done; // sync. semaphore: proc & stop +uint sb_thrds_pend; // sync. semaphore: proc & stop scoreboard_t s_ldsa_arrived; // keep track of the number of partial link delta sums scoreboard_t s_ldst_arrived; // keep track of the number of link delta sum totals // ------------------------------------------------------------------------ diff --git a/c_code/threshold.c b/c_code/threshold.c index eed2966..892d1ec 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -140,7 +140,7 @@ sdp_msg_t t_sdp_msg; // SDP message buffer for host comms. // FORWARD phase specific // (output computation) scoreboard_t tf_arrived; // keep track of expected nets -uint tf_thrds_done; // sync. semaphore: proc & stop +uint tf_thrds_pend; // sync. semaphore: proc & stop uchar tf_chain_prev; // previous daisy chain (DC) value uchar tf_chain_init; // previous DC received init uchar tf_chain_rdy; // local DC value can be forwarded @@ -157,7 +157,7 @@ uint tf_stpn_key; // stop network packet key uint tb_procs; // pointer to processing errors uint tb_comms; // pointer to receiving errors scoreboard_t tb_arrived; // keep track of expected errors -uint tb_thrds_done; // sync. semaphore: proc & stop +uint tb_thrds_pend; // sync. semaphore: proc & stop int t_max_output_unit; // unit with highest output int t_max_target_unit; // unit with highest target diff --git a/c_code/weight.c b/c_code/weight.c index 03e348e..1501790 100644 --- a/c_code/weight.c +++ b/c_code/weight.c @@ -91,14 +91,14 @@ lds_t w_lds_final; // final link delta sum uint wf_procs; // pointer to processing unit outputs uint wf_comms; // pointer to receiving unit outputs scoreboard_t wf_arrived; // keeps track of received unit outputs -uint wf_thrds_done; // sync. semaphore: comms, proc & stop +uint wf_thrds_pend; // sync. semaphore: comms, proc & stop uint wf_sync_key; // FORWARD processing can start // BACKPROP phase specific variables // (error b-d-p computation) uchar wb_active; // processing deltas from queue? scoreboard_t wb_arrived; // keeps track of received deltas -uint wb_thrds_done; // sync. semaphore: comms, proc & stop +uint wb_thrds_pend; // sync. semaphore: comms, proc & stop uint wb_sync_key; // BACKPROP processing can start weight_update_t wb_update_func; // weight update function @@ -239,7 +239,7 @@ void done (uint ec) #ifdef DEBUG_VRB io_printf (IO_BUF, "(fp:%u fc:%u)\n", wf_procs, wf_comms); - io_printf (IO_BUF, "(fptd:%u)\n", wf_thrds_done); + io_printf (IO_BUF, "(fptd:%u)\n", wf_thrds_pend); io_printf (IO_BUF, "(fa:0x%08x ba:0x%08x)\n", wf_arrived, wb_arrived From 9aadf191ae518c8611314da4da06c5887fc5c046 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Fri, 27 Apr 2018 17:15:40 +0100 Subject: [PATCH 09/21] more code cleanup. --- c_code/process_i.c | 18 +++++++++--------- c_code/process_s.c | 10 +++++----- c_code/process_t.c | 16 +++++++++------- c_code/process_w.c | 2 +- 4 files changed, 24 insertions(+), 22 deletions(-) diff --git a/c_code/process_i.c b/c_code/process_i.c index e3f93bd..e816be4 100644 --- a/c_code/process_i.c +++ b/c_code/process_i.c @@ -121,6 +121,9 @@ void i_forward_packet (uint key, uint payload) // and check if all nets done if (if_done == icfg.num_units) { + // prepare for next tick, + if_done = 0; + // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); @@ -211,7 +214,10 @@ void i_backprop_packet (uint key, uint payload) // and check if all deltas done if (ib_done == icfg.num_units) { - // advance tick + // prepare for next tick, + ib_done = 0; + + // and advance tick //TODO: check if need to schedule or can simply call ib_advance_tick (NULL, NULL); } @@ -234,10 +240,7 @@ void if_advance_tick (uint null0, uint null1) io_printf (IO_BUF, "if_tick: %d/%d\n", tick, tot_tick); #endif - // prepare for next tick, - if_done = 0; - - // and check if end of example's FORWARD phase + // check if end of example's FORWARD phase if (tick_stop) { if_advance_event (); @@ -270,10 +273,7 @@ void ib_advance_tick (uint null0, uint null1) io_printf (IO_BUF, "ib_advance_tick - tick: %d, num_ticks: %d\n", tick, num_ticks); #endif - // prepare for next tick, - ib_done = 0; - - // and check if end of BACKPROP phase + // check if end of BACKPROP phase if (tick == SPINN_IB_END_TICK) { // initialize the tick count diff --git a/c_code/process_s.c b/c_code/process_s.c index e7da0bd..8d74761 100644 --- a/c_code/process_s.c +++ b/c_code/process_s.c @@ -354,6 +354,9 @@ void s_backprop_packet (uint key, uint payload) // and check if all errors done if (sb_done == scfg.num_units) { + // prepare for next tick, + sb_done = 0; + // access synchronization semaphore with interrupts disabled uint cpsr = spin1_int_disable (); @@ -422,7 +425,7 @@ void sf_advance_tick (uint null0, uint null1) io_printf (IO_BUF, "sf_tick: %d/%d\n", tick, tot_tick); #endif - // and check if end of example's FORWARD phase + // check if end of example's FORWARD phase if (tick_stop) { sf_advance_event (); @@ -451,10 +454,7 @@ void sb_advance_tick (uint null0, uint null1) io_printf (IO_BUF, "sb_tick: %d/%d\n", tick, tot_tick); #endif - // prepare for next tick, - sb_done = 0; - - // and check if end of BACKPROP phase + // check if end of BACKPROP phase if (tick == SPINN_SB_END_TICK) { // initialize the tick count diff --git a/c_code/process_t.c b/c_code/process_t.c index dd21f97..32b95d7 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -88,6 +88,9 @@ void tf_process (uint null0, uint null1) // and check if all nets arrived (i.e., all outputs done) if (tf_arrived == tcfg.num_units) { + // initialize scoreboard for next tick, + tf_arrived = 0; + // if possible, FORWARD stop criterion if (tcfg.output_grp) { @@ -308,9 +311,6 @@ void tf_advance_tick (uint null0, uint null1) //TODO: dump outputs to SDRAM for record keeping, #endif - // initialize scoreboard for next tick, - tf_arrived = 0; - // if requested report outputs to host, if (tcfg.write_out) { @@ -334,11 +334,14 @@ void tf_advance_tick (uint null0, uint null1) // and check if done with FORWARD phase if (tick_stop) { + // update event criterion if (tcfg.is_last_output_group) { tf_event_crit = tf_event_crit && tf_group_crit && (ev_tick >= min_ticks); max_evt = evt; } + + // and move to next event tf_advance_event (); } else @@ -381,15 +384,14 @@ void tb_advance_tick (uint null0, uint null1) // switch to FORWARD phase, t_switch_to_fw (); - // advance to next example, + // update example criterion, if (tcfg.is_last_output_group) { tf_example_crit = tf_example_crit && tf_event_crit && (max_evt >= num_events - 1); } - t_advance_example (); - // and stop processing queue in this phase - return; + // and advance to next example, + t_advance_example (); } else { diff --git a/c_code/process_w.c b/c_code/process_w.c index 00fba6d..0fcde4a 100644 --- a/c_code/process_w.c +++ b/c_code/process_w.c @@ -716,7 +716,7 @@ void wf_advance_tick (uint null0, uint null1) // update pointer to processing unit outputs, wf_procs = 1 - wf_procs; - // check if end of example's FORWARD phase + // and check if end of example's FORWARD phase if (tick_stop) { wf_advance_event (); From 4f62721a1454de8f2a2284ac3523119b5230c1bd Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Sat, 5 May 2018 17:42:05 +0100 Subject: [PATCH 10/21] - fixed a bug in the input packet queue (using sum packet queue size!). - code cleanup related to DEBUG compilation flag. - changed a couple of misleading variable names (thrds_done to thrds_pend). --- c_code/comms_i.c | 2 +- c_code/comms_s.c | 2 +- c_code/comms_t.c | 4 ++-- c_code/comms_w.c | 6 +++--- c_code/input.c | 5 +++++ c_code/mlp_externs.h | 2 ++ c_code/mlp_params.h | 14 +++++++------- c_code/process_i.c | 4 ++-- c_code/process_s.c | 2 +- c_code/process_t.c | 11 +++++++++-- c_code/process_w.c | 2 +- c_code/threshold.c | 4 ++++ threshold_vertex.py | 6 +++--- 13 files changed, 41 insertions(+), 23 deletions(-) diff --git a/c_code/comms_i.c b/c_code/comms_i.c index 762ec7b..a1e27ae 100644 --- a/c_code/comms_i.c +++ b/c_code/comms_i.c @@ -31,7 +31,7 @@ void i_receivePacket (uint key, uint payload) io_printf (IO_BUF, "sc:%x\n", tick_stop); #endif - // check if all threads done + // check if all other threads done if (if_thrds_pend == 0) { // if done initialize semaphore, diff --git a/c_code/comms_s.c b/c_code/comms_s.c index 9cf03cb..6a0be8f 100644 --- a/c_code/comms_s.c +++ b/c_code/comms_s.c @@ -31,7 +31,7 @@ void s_receivePacket (uint key, uint payload) io_printf (IO_BUF, "sc:%x\n", tick_stop); #endif - // check if all threads done + // check if all other threads done if (sf_thrds_pend == 0) { // if done initialise semaphore diff --git a/c_code/comms_t.c b/c_code/comms_t.c index 71499a4..8e0592f 100644 --- a/c_code/comms_t.c +++ b/c_code/comms_t.c @@ -86,7 +86,7 @@ void t_stopPacket (uint key) io_printf (IO_BUF, "sc:%x\n", tick_stop); #endif - // check if all threads done + // check if all other threads done if (tf_thrds_pend == 0) { // initialise semaphore @@ -110,7 +110,7 @@ void t_stopPacket (uint key) void t_chainPacket (uint key) { #ifdef DEBUG - stp_recv++; + chn_recv++; #endif // STOP daisy chain partial decision arrived from previous core diff --git a/c_code/comms_w.c b/c_code/comms_w.c index 27953f1..d44da2a 100644 --- a/c_code/comms_w.c +++ b/c_code/comms_w.c @@ -84,7 +84,7 @@ void w_stopPacket (uint key) io_printf (IO_BUF, "sc:%x\n", tick_stop); #endif - // check if all threads done + // check if all other threads done if (wf_thrds_pend == 0) { // if done initialize synchronization semaphore, @@ -130,7 +130,7 @@ void w_ldsrPacket (uint payload) // the final link delta sum for the epoch arrived w_lds_final = (lds_t) payload; - // check if all threads done + // check if all other threads done if (wb_thrds_pend == 0) { //NOTE: no need to initialize semaphore @@ -189,7 +189,7 @@ void w_forwardPacket (uint key, uint payload) // update pointer to received unit outputs, wf_comms = 1 - wf_comms; - // and check if other threads are done, + // and check if all other threads are done, if (wf_thrds_pend == 0) { // if done initialize synchronization semaphore, diff --git a/c_code/input.c b/c_code/input.c index 167f784..cf791be 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -261,6 +261,11 @@ void done (uint ec) io_printf (IO_BUF, "(fd:%08x bd:%08x)\n", if_done, ib_done); #endif + break; + + case SPINN_PKT_QUEUE_ERROR: + io_printf (IO_BUF, "unexpected packet in queue - abort!\n"); + break; } diff --git a/c_code/mlp_externs.h b/c_code/mlp_externs.h index a24bc69..369bf2e 100644 --- a/c_code/mlp_externs.h +++ b/c_code/mlp_externs.h @@ -189,6 +189,8 @@ extern long_deriv_t * t_output_deriv_history; extern uint recv_bkp; // packets received in BACKPROP phase extern uint spk_sent; // sync packets sent extern uint spk_recv; // sync packets received + extern uint chn_sent; // chain packets sent + extern uint chn_recv; // chain packets received extern uint stp_sent; // stop packets sent extern uint stp_recv; // stop packets received extern uint stn_sent; // network_stop packets sent diff --git a/c_code/mlp_params.h b/c_code/mlp_params.h index 1d57957..1c38865 100644 --- a/c_code/mlp_params.h +++ b/c_code/mlp_params.h @@ -100,16 +100,17 @@ // multicast packet routing keys and masks // ------------------------------------------------------------------------ // packet type keys +#define SPINN_DATA_KEY 0x00000000 #define SPINN_SYNC_KEY 0x00001000 #define SPINN_LDST_KEY 0x00002000 #define SPINN_LDSA_KEY 0x00003000 -#define SPINN_LDSR_KEY 0x00002800 -#define SPINN_STPC_KEY 0x00003200 -#define SPINN_STPN_KEY 0x00003800 -#define SPINN_STOP_KEY 0x00003a00 +#define SPINN_LDSR_KEY 0x00004000 +#define SPINN_STPC_KEY 0x00005000 +#define SPINN_STPN_KEY 0x00006000 +#define SPINN_STOP_KEY 0x00007000 // packet type mask -#define SPINN_TYPE_MASK 0x00003a00 +#define SPINN_TYPE_MASK 0x0000f000 // packet condition keys #define SPINN_PHASE_KEY(p) (p << SPINN_PHASE_SHIFT) @@ -154,7 +155,7 @@ //TODO: check if size is appropriate #define SPINN_THLD_PQ_LEN 256 #define SPINN_WEIGHT_PQ_LEN 512 -#define SPINN_SUM_PQ_LEN 512 +#define SPINN_SUM_PQ_LEN 2048 #define SPINN_INPUT_PQ_LEN 512 // ------------------------------------------------------------------------ @@ -218,7 +219,6 @@ #define SPINN_QUEUE_FULL 2 #define SPINN_TIMEOUT_EXIT 3 #define SPINN_UNXPD_PKT 4 -#define SPINN_CORE_TYPE_ERROR 5 // ------------------------------------------------------------------------ #endif diff --git a/c_code/process_i.c b/c_code/process_i.c index e816be4..a0c7ea5 100644 --- a/c_code/process_i.c +++ b/c_code/process_i.c @@ -33,7 +33,7 @@ void i_process (uint null0, uint null1) // if not empty dequeue packet, uint key = i_pkt_queue.queue[i_pkt_queue.head].key; uint payload = i_pkt_queue.queue[i_pkt_queue.head].payload; - i_pkt_queue.head = (i_pkt_queue.head + 1) % SPINN_SUM_PQ_LEN; + i_pkt_queue.head = (i_pkt_queue.head + 1) % SPINN_INPUT_PQ_LEN; // restore interrupts after queue access, spin1_mode_restore (cpsr); @@ -240,7 +240,7 @@ void if_advance_tick (uint null0, uint null1) io_printf (IO_BUF, "if_tick: %d/%d\n", tick, tot_tick); #endif - // check if end of example's FORWARD phase + // check if end of event if (tick_stop) { if_advance_event (); diff --git a/c_code/process_s.c b/c_code/process_s.c index 8d74761..6de724a 100644 --- a/c_code/process_s.c +++ b/c_code/process_s.c @@ -425,7 +425,7 @@ void sf_advance_tick (uint null0, uint null1) io_printf (IO_BUF, "sf_tick: %d/%d\n", tick, tot_tick); #endif - // check if end of example's FORWARD phase + // check if end of event if (tick_stop) { sf_advance_event (); diff --git a/c_code/process_t.c b/c_code/process_t.c index 32b95d7..1a0ef69 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -331,7 +331,7 @@ void tf_advance_tick (uint null0, uint null1) } } - // and check if done with FORWARD phase + // and check if done with event if (tick_stop) { // update event criterion @@ -714,7 +714,14 @@ void tf_send_stop (uint null0, uint null1) ); #ifdef DEBUG - stp_sent++; + if (tcfg.is_last_output_group) + { + stp_sent++; + } + else + { + chn_sent++; + } #endif // and initialise criterion for next tick diff --git a/c_code/process_w.c b/c_code/process_w.c index 0fcde4a..5f243e0 100644 --- a/c_code/process_w.c +++ b/c_code/process_w.c @@ -716,7 +716,7 @@ void wf_advance_tick (uint null0, uint null1) // update pointer to processing unit outputs, wf_procs = 1 - wf_procs; - // and check if end of example's FORWARD phase + // and check if end of event if (tick_stop) { wf_advance_event (); diff --git a/c_code/threshold.c b/c_code/threshold.c index 892d1ec..07c4e54 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -186,6 +186,8 @@ long_deriv_t * t_output_deriv_history; uint recv_bkp = 0; // packets received in BACKPROP phase uint spk_sent = 0; // sync packets sent uint spk_recv = 0; // sync packets received + uint chn_sent = 0; // chain packets sent + uint chn_recv = 0; // chain packets received uint stp_sent = 0; // stop packets sent uint stp_recv = 0; // stop packets received uint stn_sent = 0; // network_stop packets sent @@ -399,6 +401,8 @@ void done (uint ec) io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "sync recv:%d\n", spk_recv); io_printf (IO_BUF, "sync sent:%d\n", spk_sent); + io_printf (IO_BUF, "chain recv:%d\n", chn_recv); + io_printf (IO_BUF, "chain sent:%d\n", chn_sent); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stop sent:%d\n", stp_sent); #endif diff --git a/threshold_vertex.py b/threshold_vertex.py index 3e887dc..65e492f 100644 --- a/threshold_vertex.py +++ b/threshold_vertex.py @@ -74,9 +74,9 @@ def __init__(self, self._is_last_output_group = 0 # forward, backprop and stop link partition names - self._fwd_link = "fwd_s{}".format (self.group.id) - self._bkp_link = "bkp_s{}".format (self.group.id) - self._stp_link = "stp_s{}".format (self.group.id) + self._fwd_link = "fwd_t{}".format (self.group.id) + self._bkp_link = "bkp_t{}".format (self.group.id) + self._stp_link = "stp_t{}".format (self.group.id) # threshold core-specific parameters # NOTE: if all-zero w cores are optimised out these need reviewing From d02a2873d80112300f730135cd0e9bdc0cb01209 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Sat, 5 May 2018 17:50:53 +0100 Subject: [PATCH 11/21] - removed debugging code left over. --- c_code/input.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/c_code/input.c b/c_code/input.c index cf791be..167f784 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -261,11 +261,6 @@ void done (uint ec) io_printf (IO_BUF, "(fd:%08x bd:%08x)\n", if_done, ib_done); #endif - break; - - case SPINN_PKT_QUEUE_ERROR: - io_printf (IO_BUF, "unexpected packet in queue - abort!\n"); - break; } From 54262770f19b193252c7aa35e27a17e4e17fe7e8 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Wed, 16 May 2018 12:42:45 +0100 Subject: [PATCH 12/21] - added a placement constraint on (0, 0) for OUTPUT group t cores. - request constant iptag = 2 to avoid having to modify C code. - modified a few parameter names for consistency. --- c_code/comms_t.c | 12 ++++++------ c_code/comms_t.h | 2 -- c_code/init_t.c | 4 ++-- c_code/mlp_params.h | 17 ++++++++--------- c_code/process_t.c | 8 +++++--- threshold_vertex.py | 11 ++++++++++- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/c_code/comms_t.c b/c_code/comms_t.c index 8e0592f..fe2ce73 100644 --- a/c_code/comms_t.c +++ b/c_code/comms_t.c @@ -123,7 +123,7 @@ void t_chainPacket (uint key) tf_chain_rdy = tf_chain_init; // and send stop packet - spin1_schedule_callback (tf_send_stop, NULL, NULL, SPINN_SEND_STOP_P); + spin1_schedule_callback (tf_send_stop, NULL, NULL, SPINN_T_SEND_STOP_P); // last group in the chain does not get a stop decision packet // so it's ready to advance tick @@ -183,7 +183,7 @@ void t_syncPacket (uint ph) // clear synchronisation flag, t_sync_rdy = FALSE; - // schedule sending of unit outputs, + // schedule sending of unit outputs to w cores, spin1_schedule_callback (t_init_outputs, NULL, NULL, SPINN_T_INIT_OUT_P ); @@ -192,7 +192,7 @@ void t_syncPacket (uint ph) if (tcfg.write_out) { spin1_schedule_callback (send_outputs_to_host, - SPINN_HOST_NORMAL, 0, SPINN_SEND_OUTS_P + SPINN_HOST_NORMAL, 0, SPINN_T_SEND_OUTS_P ); } } @@ -220,7 +220,7 @@ void t_syncPacket (uint ph) if (phase == SPINN_BACKPROP) { // schedule sending of deltas - //#spin1_schedule_callback (t_init_deltas, NULL, NULL, SPINN_SEND_DELTAS_P); + //#spin1_schedule_callback (t_init_deltas, NULL, NULL, SPINN_T_INIT_DLT_P); } else { @@ -380,8 +380,8 @@ void send_outputs_to_host (uint cmd, uint tick) // ------------------------------------------------------------------------ // send an sdp packet to the host with information related to // various parameters of the simulation: id of the output group sending the -// data, number of output units, number of units writing outputs an dnumber of -// ticks of simulation +// data, number of output units, number of groups writing outputs and number +// of ticks of simulation // ------------------------------------------------------------------------ void send_info_to_host (uint null0, uint null1) { diff --git a/c_code/comms_t.h b/c_code/comms_t.h index e695953..b2fb837 100644 --- a/c_code/comms_t.h +++ b/c_code/comms_t.h @@ -9,9 +9,7 @@ void t_chainPacket (uint); void t_syncPacket (uint); void t_networkStopPacket (void); -//#void t_sendDeltas (uint, uint); void send_info_to_host (uint, uint); -//#void send_outputs_to_host (ushort); void send_outputs_to_host (uint, uint); //#void send_weights_to_host (void); diff --git a/c_code/init_t.c b/c_code/init_t.c index c16f40b..c196130 100644 --- a/c_code/init_t.c +++ b/c_code/init_t.c @@ -263,7 +263,7 @@ uint t_init (void) } // schedule sending of initial data to host - spin1_schedule_callback (send_info_to_host, NULL, NULL, SPINN_T_INIT_OUT_P); + spin1_schedule_callback (send_info_to_host, NULL, NULL, SPINN_T_SEND_OUTS_P); } // initialise packet keys @@ -335,7 +335,7 @@ uint t_init (void) if (tcfg.write_out) { spin1_schedule_callback (send_outputs_to_host, - SPINN_HOST_NORMAL, 0, SPINN_SEND_OUTS_P + SPINN_HOST_NORMAL, 0, SPINN_T_SEND_OUTS_P ); } diff --git a/c_code/mlp_params.h b/c_code/mlp_params.h index 1c38865..ae8b947 100644 --- a/c_code/mlp_params.h +++ b/c_code/mlp_params.h @@ -168,7 +168,6 @@ #define SPINN_TIMER_P 0 // queueable callbacks -#define SPINN_UPDT_WEIGHT_P 1 #define SPINN_WF_TICK_P 1 #define SPINN_WB_TICK_P 1 #define SPINN_WF_PROCESS_P 2 @@ -181,14 +180,14 @@ #define SPINN_I_PROCESS_P 2 //TODO: review priorities -#define SPINN_T_INIT_OUT_P 1 -#define SPINN_SEND_OUTS_P 1 -#define SPINN_SEND_DELTAS_P 1 -#define SPINN_SEND_STOP_P 1 -#define SPINN_TF_TICK_P 1 -#define SPINN_TB_TICK_P 1 -#define SPINN_TF_PROCESS_P 2 -#define SPINN_TB_PROCESS_P 2 +#define SPINN_T_SEND_OUTS_P 1 +#define SPINN_T_SEND_STOP_P 2 +#define SPINN_T_INIT_OUT_P 2 +#define SPINN_T_INIT_DLT_P 2 +#define SPINN_TF_TICK_P 2 +#define SPINN_TB_TICK_P 2 +#define SPINN_TF_PROCESS_P 3 +#define SPINN_TB_PROCESS_P 3 // ------------------------------------------------------------------------ diff --git a/c_code/process_t.c b/c_code/process_t.c index 1a0ef69..c1b79a5 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -133,6 +133,7 @@ void tf_process (uint null0, uint null1) uint cpsr = spin1_int_disable (); // report processing thread done, + //NOTE: stop criterion cannot have arrived! tf_thrds_pend -= 1; // check if chain value can be forwarded @@ -314,7 +315,8 @@ void tf_advance_tick (uint null0, uint null1) // if requested report outputs to host, if (tcfg.write_out) { - spin1_delay_us (2000); //## + //spin1_delay_us (2000); //## + spin1_delay_us (100000); //## // is this the last report? if ((epoch == (ncfg.num_epochs - 1)) @@ -500,7 +502,7 @@ void t_advance_example (void) // we have decided to terminate training, so write out final data if (tcfg.write_out) { - spin1_delay_us (2000); //## + //spin1_delay_us (2000); //## send_outputs_to_host (SPINN_HOST_FINAL, 0); } @@ -594,7 +596,7 @@ void t_advance_example (void) if (tcfg.write_out) { spin1_schedule_callback (send_outputs_to_host, - SPINN_HOST_NORMAL, 0, SPINN_SEND_OUTS_P + SPINN_HOST_NORMAL, 0, SPINN_T_SEND_OUTS_P ); } } diff --git a/threshold_vertex.py b/threshold_vertex.py index 65e492f..cba42b3 100644 --- a/threshold_vertex.py +++ b/threshold_vertex.py @@ -5,7 +5,11 @@ from pacman.executor.injection_decorator import inject_items from pacman.model.graphs.machine.machine_vertex import MachineVertex + +from pacman.model.constraints.placer_constraints import ChipAndCoreConstraint + from pacman.model.decorators.overrides import overrides + from pacman.model.resources.resource_container import ResourceContainer from pacman.model.resources.sdram_resource import SDRAMResource from pacman.model.resources.iptag_resource import IPtagResource @@ -41,6 +45,10 @@ def __init__(self, MachineVertex.__init__(self, label =\ "t{} core".format (group.id)) + # add placement constraint if OUTPUT group + if group.output_grp: + self.add_constraint (ChipAndCoreConstraint (x = 0, y = 0)) + # application-level data self._network = network self._group = group @@ -49,7 +57,7 @@ def __init__(self, self._ev_cfg = network._ex_set.event_config # application parameters - self._out_integr_dt = 1.0 / network.ticks_per_int + self._out_integr_dt = 1.0 / network.ticks_per_int # choose appropriate group criterion if network.training: @@ -230,6 +238,7 @@ def resources_required (self): resources = ResourceContainer ( sdram = SDRAMResource (self._sdram_usage), iptags = [IPtagResource (ip_address = "localhost", + tag = 2, port = 17896, strip_sdp = False)] ) From ecc893e1d0a2c1a39f55683ffdcc9c779c54e343 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Wed, 16 May 2018 13:31:49 +0100 Subject: [PATCH 13/21] removed redundant statement --- mlp_group.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mlp_group.py b/mlp_group.py index 40277d4..b74b609 100644 --- a/mlp_group.py +++ b/mlp_group.py @@ -147,7 +147,6 @@ def __init__(self, self.out_procs_list [1] = MLPOutputProcs.OUT_NONE if (MLPGroupTypes.OUTPUT in self.type): - self.write_out = 1 self.group_criterion = MLPConstants.DEF_GRP_CRIT self.criterion_function = MLPStopCriteria.STOP_STD self.error_function = MLPErrorFuncs.ERR_CROSS_ENTROPY From 12faea5e1fe7d52c3d349a5f960b90f6d8a11a56 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Wed, 16 May 2018 16:28:49 +0100 Subject: [PATCH 14/21] reorganised code in tf_process. --- c_code/process_t.c | 101 ++++++++++++++++----------------------------- 1 file changed, 35 insertions(+), 66 deletions(-) diff --git a/c_code/process_t.c b/c_code/process_t.c index c1b79a5..1b2b156 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -91,80 +91,49 @@ void tf_process (uint null0, uint null1) // initialize scoreboard for next tick, tf_arrived = 0; - // if possible, FORWARD stop criterion + // access synchronisation flags with interrupts disabled + cpsr = spin1_int_disable (); + + // report processing done and forward stop criterion if OUTPUT group if (tcfg.output_grp) { - // last group in the chain does not get a stop decision - // it is ready to advance tick - if (tcfg.is_last_output_group) - { - // check flags status in critical section - uint cpsr = spin1_int_disable (); - - // check if chain value can be forwarded - if (tf_chain_rdy) - { - // initialise semaphore, - tf_chain_rdy = tf_chain_init; - - // restore interrupts after flag access, - spin1_mode_restore (cpsr); - - // send stop criterion packet, - //TODO: check if need to schedule or can simply call - tf_send_stop (NULL, NULL); - - // and advance tick + // report processing thread done, + //NOTE: stop criterion cannot have arrived! + tf_thrds_pend -= 1; + + // check if chain value can be forwarded + if (tf_chain_rdy) + { + // initialise semaphore, + tf_chain_rdy = tf_chain_init; + + // restore interrupts after flag access, + spin1_mode_restore (cpsr); + + // send stop criterion packet, + //TODO: check if need to schedule or can simply call + tf_send_stop (NULL, NULL); + + // and advance tick if last group + //NOTE: last group in the chain does not get a stop decision + if (tcfg.is_last_output_group) + { //TODO: check if need to schedule or can simply call tf_advance_tick (NULL, NULL); } - else - { - // if not, flag that local value is ready, - tf_chain_rdy = 1; - - // and restore interrupts after flag access - spin1_mode_restore (cpsr); - } - } - else - { - // check flags status in critical section - uint cpsr = spin1_int_disable (); - - // report processing thread done, - //NOTE: stop criterion cannot have arrived! - tf_thrds_pend -= 1; - - // check if chain value can be forwarded - if (tf_chain_rdy) - { - // initialise semaphore, - tf_chain_rdy = tf_chain_init; - - // restore interrupts after flag access, - spin1_mode_restore (cpsr); - - // and send stop criterion packet - //TODO: check if need to schedule or can simply call - tf_send_stop (NULL, NULL); - } - else - { - // if not, flag that local value is ready, - tf_chain_rdy = 1; - - // and restore interrupts after flag access - spin1_mode_restore (cpsr); - } - } + } + else + { + // flag that local value is ready, + tf_chain_rdy = 1; + + // and restore interrupts after flag access + spin1_mode_restore (cpsr); + } } else { - // access synchronisation semaphore with interrupts disabled - uint cpsr = spin1_int_disable (); - - // and check if all other threads done + // check if all other threads done if (tf_thrds_pend == 0) { // initialize semaphore, From dbad2a2af7f6728a8f9af5bbdf2889205314c014 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Thu, 17 May 2018 17:54:22 +0100 Subject: [PATCH 15/21] - modified the sending of outputs to host to make it less fragile. - modified 'output_mon_lens' to match C code modifications. --- c_code/comms_t.c | 56 +++++++++++++++------- c_code/init_t.c | 35 +++++++------- c_code/process_t.c | 55 +++++++++------------- output_mon_lens | 114 +++++++++++++++++++++++---------------------- 4 files changed, 136 insertions(+), 124 deletions(-) diff --git a/c_code/comms_t.c b/c_code/comms_t.c index fe2ce73..e78ff19 100644 --- a/c_code/comms_t.c +++ b/c_code/comms_t.c @@ -122,14 +122,21 @@ void t_chainPacket (uint key) // initialise flag, tf_chain_rdy = tf_chain_init; - // and send stop packet + // report outputs to host if requested, + if (tcfg.write_out) + { + spin1_schedule_callback (send_outputs_to_host, SPINN_HOST_NORMAL, tick, + SPINN_T_SEND_OUTS_P); + } + + // send stop packet, spin1_schedule_callback (tf_send_stop, NULL, NULL, SPINN_T_SEND_STOP_P); - // last group in the chain does not get a stop decision packet - // so it's ready to advance tick + // and advance tick if last_output_group + //NOTE: this group does not get a stop decision packet + // so it's ready to advance tick if (tcfg.is_last_output_group) { - // advance tick spin1_schedule_callback (tf_advance_tick, NULL, NULL, SPINN_TF_TICK_P); } } @@ -333,8 +340,11 @@ void t_backpropPacket (uint key, uint payload) // ------------------------------------------------------------------------ void send_outputs_to_host (uint cmd, uint tick) { - int le; - le = (tick == 0) ? -1 : (int) evt; + // spread SDP messages to avoid congestion and possible loss, + spin1_delay_us (1000); //## + + // adjust event according to Lens reporting, + int le = (tick == 0) ? -1 : (int) evt; // report epoch, example and tick, t_sdp_msg.cmd_rc = cmd; @@ -354,14 +364,18 @@ void send_outputs_to_host (uint cmd, uint tick) } else { - my_data[2 * i] = (short_activ_t) (t_outputs[i] >> (SPINN_ACTIV_SHIFT - SPINN_SHORT_ACTIV_SHIFT)); + my_data[2 * i] = (short_activ_t) (t_outputs[i] + >> (SPINN_ACTIV_SHIFT + - SPINN_SHORT_ACTIV_SHIFT)); if (tt[t_it_idx + i] == SPINN_ACTIV_ONE) { my_data[2 * i + 1] = SPINN_SHORT_ACTIV_MAX; } else { - my_data[2 * i + 1] = (short_activ_t) (tt[t_it_idx + i] >> (SPINN_ACTIV_SHIFT - SPINN_SHORT_ACTIV_SHIFT)); + my_data[2 * i + 1] = (short_activ_t) (tt[t_it_idx + i] + >> (SPINN_ACTIV_SHIFT + - SPINN_SHORT_ACTIV_SHIFT)); } } } @@ -371,8 +385,7 @@ void send_outputs_to_host (uint cmd, uint tick) t_sdp_msg.length = sizeof (sdp_hdr_t) + sizeof (cmd_hdr_t) + len; // and send message - while (!spin1_send_sdp_msg (&t_sdp_msg, SPINN_SDP_TMOUT)) - io_printf (IO_STD, "sdp!\n"); + while (!spin1_send_sdp_msg (&t_sdp_msg, SPINN_SDP_TMOUT)); } // ------------------------------------------------------------------------ @@ -393,17 +406,26 @@ void send_info_to_host (uint null0, uint null1) t_sdp_msg.arg2 = ncfg.num_write_blks; t_sdp_msg.arg3 = t_tot_ticks + 1; + // copy initial outputs and targets into msg buffer, + short_activ_t * my_data = (short_activ_t *) t_sdp_msg.data; + for (uint i = 0; i < tcfg.num_units; i++) + { + my_data[2 * i] = 0; + my_data[2 * i + 1] = 0; + } + // set message length, - t_sdp_msg.length = sizeof (sdp_hdr_t) + sizeof (cmd_hdr_t); + uint len = 2 * tcfg.num_units * sizeof(short_activ_t); + t_sdp_msg.length = sizeof (sdp_hdr_t) + sizeof (cmd_hdr_t) + len; // and send message while (!spin1_send_sdp_msg (&t_sdp_msg, SPINN_SDP_TMOUT)); - #ifdef DEBUG_VRB - io_printf (IO_BUF, "sent info to host: nb:%d wb:%d no:%d tt:%d\n", - ncfg.num_write_blks, tcfg.write_blk, - tcfg.num_units, t_tot_ticks - ); - #endif +#ifdef DEBUG_VRB + io_printf (IO_BUF, "sent info to host: nb:%d wb:%d no:%d tt:%d\n", + ncfg.num_write_blks, tcfg.write_blk, + tcfg.num_units, t_tot_ticks + ); +#endif } // ------------------------------------------------------------------------ diff --git a/c_code/init_t.c b/c_code/init_t.c index c196130..c9439e1 100644 --- a/c_code/init_t.c +++ b/c_code/init_t.c @@ -129,7 +129,8 @@ uint t_init (void) { // get max number of ticks for first event if (ev[event_idx].max_time != SPINN_FP_NaN) - max_ticks = (((ev[event_idx].max_time + SPINN_SMALL_VAL) * ncfg.ticks_per_int) + max_ticks = (((ev[event_idx].max_time + SPINN_SMALL_VAL) + * ncfg.ticks_per_int) + (1 << (SPINN_FPREAL_SHIFT - 1))) >> SPINN_FPREAL_SHIFT; else @@ -139,7 +140,8 @@ uint t_init (void) // get min number of ticks for first event if (ev[event_idx].min_time != SPINN_FP_NaN) - min_ticks = (((ev[event_idx].min_time + SPINN_SMALL_VAL) * ncfg.ticks_per_int) + min_ticks = (((ev[event_idx].min_time + SPINN_SMALL_VAL) + * ncfg.ticks_per_int) + (1 << (SPINN_FPREAL_SHIFT - 1))) >> SPINN_FPREAL_SHIFT; else @@ -172,8 +174,10 @@ uint t_init (void) // variables for stop criterion computation t_max_output_unit = -1; t_max_target_unit = -1; - t_max_output = SPINN_SHORT_ACTIV_MIN << (SPINN_ACTIV_SHIFT - SPINN_SHORT_ACTIV_SHIFT); - t_max_target = SPINN_SHORT_ACTIV_MIN << (SPINN_ACTIV_SHIFT - SPINN_SHORT_ACTIV_SHIFT); + t_max_output = SPINN_SHORT_ACTIV_MIN << (SPINN_ACTIV_SHIFT + - SPINN_SHORT_ACTIV_SHIFT); + t_max_target = SPINN_SHORT_ACTIV_MIN << (SPINN_ACTIV_SHIFT + - SPINN_SHORT_ACTIV_SHIFT); // no need to wait for previous value if first in chain if (tcfg.is_first_output_group) @@ -245,7 +249,8 @@ uint t_init (void) // update number of ticks for new event if (ev[event_idx + i].max_time != SPINN_FP_NaN) { - t_tot_ticks += (((ev[event_idx + i].max_time + SPINN_SMALL_VAL) * ncfg.ticks_per_int) + t_tot_ticks += (((ev[event_idx + i].max_time + SPINN_SMALL_VAL) + * ncfg.ticks_per_int) + (1 << (SPINN_FPREAL_SHIFT - 1))) >> SPINN_FPREAL_SHIFT; } @@ -287,7 +292,8 @@ uint t_init (void) // allocate memory in SDRAM for target history if ((t_target_history = ((activation_t *) sark_xalloc (sv->sdram_heap, - tcfg.num_units * ncfg.global_max_ticks * sizeof (activation_t), + tcfg.num_units * ncfg.global_max_ticks + * sizeof (activation_t), 0, ALLOC_LOCK) )) == NULL ) @@ -298,7 +304,8 @@ uint t_init (void) // allocate memory in SDRAM for output derivative history if ((t_output_deriv_history = ((long_deriv_t *) sark_xalloc (sv->sdram_heap, - tcfg.num_units * ncfg.global_max_ticks * sizeof (long_deriv_t), + tcfg.num_units * ncfg.global_max_ticks + * sizeof (long_deriv_t), 0, ALLOC_LOCK) )) == NULL ) @@ -320,7 +327,8 @@ uint t_init (void) // allocate memory in SDRAM for output history if ((t_output_history = ((activation_t *) sark_xalloc (sv->sdram_heap, - tcfg.num_units * ncfg.global_max_ticks * sizeof (activation_t), + tcfg.num_units * ncfg.global_max_ticks + * sizeof (activation_t), 0, ALLOC_LOCK) )) == NULL ) @@ -328,18 +336,9 @@ uint t_init (void) return (SPINN_MEM_UNAVAIL); } - // schedule initialization and sending of unit outputs + // and schedule initialization and sending of unit outputs spin1_schedule_callback (t_init_outputs, NULL, NULL, SPINN_T_INIT_OUT_P); - // and, if required, send outputs to host - if (tcfg.write_out) - { - spin1_schedule_callback (send_outputs_to_host, - SPINN_HOST_NORMAL, 0, SPINN_T_SEND_OUTS_P - ); - - } - return (SPINN_NO_ERROR); } // ------------------------------------------------------------------------ diff --git a/c_code/process_t.c b/c_code/process_t.c index 1b2b156..b00ff18 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -110,6 +110,13 @@ void tf_process (uint null0, uint null1) // restore interrupts after flag access, spin1_mode_restore (cpsr); + // report outputs to host if requested, + if (tcfg.write_out) + { + //TODO: check if need to schedule or can simply call + send_outputs_to_host (SPINN_HOST_NORMAL, tick); + } + // send stop criterion packet, //TODO: check if need to schedule or can simply call tf_send_stop (NULL, NULL); @@ -281,28 +288,7 @@ void tf_advance_tick (uint null0, uint null1) //TODO: dump outputs to SDRAM for record keeping, #endif - // if requested report outputs to host, - if (tcfg.write_out) - { - //spin1_delay_us (2000); //## - spin1_delay_us (100000); //## - - // is this the last report? - if ((epoch == (ncfg.num_epochs - 1)) - && (example == (ncfg.num_examples - 1)) - && ((evt == (num_events - 1)) || (tick == (ncfg.global_max_ticks - 1))) - && (tick_stop) - ) - { - send_outputs_to_host (SPINN_HOST_FINAL, tick); - } - else - { - send_outputs_to_host (SPINN_HOST_NORMAL, tick); - } - } - - // and check if done with event + // check if done with event if (tick_stop) { // update event criterion @@ -471,8 +457,6 @@ void t_advance_example (void) // we have decided to terminate training, so write out final data if (tcfg.write_out) { - //spin1_delay_us (2000); //## - send_outputs_to_host (SPINN_HOST_FINAL, 0); } @@ -496,7 +480,13 @@ void t_advance_example (void) // check if done with epochs if (++epoch >= ncfg.num_epochs) { - // done + // send final output packet to host, + if (tcfg.is_last_output_group && tcfg.write_out) + { + send_outputs_to_host (SPINN_HOST_FINAL, 0); + } + + // and exit spin1_exit (SPINN_NO_ERROR); return; } @@ -557,17 +547,16 @@ void t_advance_example (void) // restore interrupts, spin1_mode_restore (cpsr); - // schedule sending of unit outputs, - //TODO: check if need to schedule or can simply call - spin1_schedule_callback (t_init_outputs, NULL, NULL, SPINN_T_INIT_OUT_P); - - // and, if required, send outputs to host + // send outputs to host if requested, if (tcfg.write_out) { - spin1_schedule_callback (send_outputs_to_host, - SPINN_HOST_NORMAL, 0, SPINN_T_SEND_OUTS_P - ); + //TODO: check if need to schedule or can simply call + send_outputs_to_host (SPINN_HOST_NORMAL, 0); } + + // and send unit outputs to w cores + //TODO: check if need to schedule or can simply call + t_init_outputs (NULL, NULL); } else { diff --git a/output_mon_lens b/output_mon_lens index 27914da..626e0f6 100755 --- a/output_mon_lens +++ b/output_mon_lens @@ -153,6 +153,11 @@ sub main_loop #printf "c:%d g:%d t:%d s:%d\n", $cmd, $grp, $a3, scalar(@t); + if ($cmd == $HOST_FINAL) + { + last; + } + if ($cmd == $HOST_INFO) { $num_outs[$sq] = $a1; @@ -165,75 +170,72 @@ sub main_loop $info_ctr = 0; } + $a1 = 0; + $a2 = 0xffff0000; + $a3 = 0; } - else + + my $inx = $cur; + if ($tick[$cur] == 0xffffffff) { - my $inx = $cur; - if ($tick[$cur] == 0xffffffff) + if (($a3 != 0) && ($a3 != ($ltick + 1))) { - if (($a3 != 0) && ($a3 != ($ltick + 1))) - { - printf "missing tick e:%d r:%d\n", ($ltick + 1), $a3; - } - - $epoch[$cur] = $a1; - $example[$cur] = ($a2 & 0x0000ffff); - $event[$cur] = ($a2 >> 16); - if ($event[$cur] > 32767) { $event[$cur] = -1; } - $tick[$cur] = $a3; + printf "missing tick e:%d r:%d\n", ($ltick + 1), $a3; } - elsif ($tick[$cur] != $a3) + + $epoch[$cur] = $a1; + $example[$cur] = ($a2 & 0x0000ffff); + $event[$cur] = ($a2 >> 16); + if ($event[$cur] > 32767) { $event[$cur] = -1; } + $tick[$cur] = $a3; + } + elsif ($tick[$cur] != $a3) + { + if (($a3 != 0) && ($a3 != ($tick[$cur] + 1))) { - if (($a3 != 0) && ($a3 != ($tick[$cur] + 1))) - { - printf "missing tick e:%d r:%d\n", $tick[$cur] + 1, $a3; - } - - $inx = $nxt; - if ($tick[$nxt] == 0xffffffff) - { - $epoch[$nxt] = $a1; - $example[$nxt] = ($a2 & 0x0000ffff); - $event[$nxt] = ($a2 >> 16); - if ($event[$nxt] > 32767) { $event[$nxt] = -1; } - $tick[$nxt] = $a3; - } - elsif ($tick[$nxt] != $a3) - { - printf "wrong tick e:%d r:%d\n", $tick[$nxt], $a3; - } + printf "missing tick e:%d r:%d\n", $tick[$cur] + 1, $a3; } - my $bi = $bindex[$inx]; - - $bstart->[$inx][$grp] = $bi; - - for (my $i = 0; $i < @t/2; $i++) + $inx = $nxt; + if ($tick[$nxt] == 0xffffffff) + { + $epoch[$nxt] = $a1; + $example[$nxt] = ($a2 & 0x0000ffff); + $event[$nxt] = ($a2 >> 16); + if ($event[$nxt] > 32767) { $event[$nxt] = -1; } + $tick[$nxt] = $a3; + } + elsif ($tick[$nxt] != $a3) { - $outputs->[$inx][$bi] = $t[2 * $i]; - $targets->[$inx][$bi] = $t[(2 * $i) + 1]; - $bi++; + printf "wrong tick e:%d r:%d\n", $tick[$nxt], $a3; } + } + + my $bi = $bindex[$inx]; - $bindex[$inx] = $bi; - $grp_ctr[$inx]++; + $bstart->[$inx][$grp] = $bi; + + for (my $i = 0; $i < @t/2; $i++) + { + $outputs->[$inx][$bi] = $t[2 * $i]; + $targets->[$inx][$bi] = $t[(2 * $i) + 1]; + $bi++; + } + + $bindex[$inx] = $bi; + $grp_ctr[$inx]++; #printf "ix:%d gc:%d bi:%d\n", $inx, $grp_ctr[$inx], $bindex[$inx]; - if ($grp_ctr[$inx] == $num_grps) - { - print_outputs (); - - $grp_ctr[$inx] = 0; - $ltick = $tick[$inx]; - $tick[$inx] = 0xffffffff; - $nxt = $cur; - $cur = 1 - $cur; - if ($cmd == $HOST_FINAL) - { - $run = 0; - } - } + if ($grp_ctr[$inx] == $num_grps) + { + print_outputs (); + + $grp_ctr[$inx] = 0; + $ltick = $tick[$inx]; + $tick[$inx] = 0xffffffff; + $nxt = $cur; + $cur = 1 - $cur; } } } From 77c35384c320c04273666cbd1da051a82a2799d6 Mon Sep 17 00:00:00 2001 From: Joanna Moy Date: Fri, 8 Jun 2018 16:16:31 +0100 Subject: [PATCH 16/21] Ensure groups that are both input and output groups get assigned the correct input and output functions; makes sure the groups that are neither do not get assigned out_integr by default. --- mlp_group.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/mlp_group.py b/mlp_group.py index 40277d4..fa7a9dc 100644 --- a/mlp_group.py +++ b/mlp_group.py @@ -87,20 +87,22 @@ def __init__(self, self.weak_clamp_strength = MLPConstants.DEF_WEAK_CLMP if output_funcs is None: - # an input integrator removes the default output integrator - if (self.in_integr_en == 1): - self.out_integr_en = 0 - self.num_out_procs = MLPConstants.DEF_OUT_PROCS - 1 + # output groups have a default output integrator unless there is an input integrator + if (MLPGroupTypes.OUTPUT in self.type and self.in_integr_en == 0): + self.out_integr_en = 1 + self.num_out_procs = MLPConstants.DEF_OUT_PROCS self.out_procs_list = [MLPOutputProcs.OUT_LOGISTIC,\ - MLPOutputProcs.OUT_NONE,\ + MLPOutputProcs.OUT_INTEGR,\ MLPOutputProcs.OUT_NONE,\ MLPOutputProcs.OUT_NONE,\ MLPOutputProcs.OUT_NONE] + # an input integrator removes the default output integrator from an output group + # other groups have no integrator by default else: - self.out_integr_en = 1 - self.num_out_procs = MLPConstants.DEF_OUT_PROCS + self.out_integr_en = 0 + self.num_out_procs = MLPConstants.DEF_OUT_PROCS - 1 self.out_procs_list = [MLPOutputProcs.OUT_LOGISTIC,\ - MLPOutputProcs.OUT_INTEGR,\ + MLPOutputProcs.OUT_NONE,\ MLPOutputProcs.OUT_NONE,\ MLPOutputProcs.OUT_NONE,\ MLPOutputProcs.OUT_NONE] @@ -140,7 +142,7 @@ def __init__(self, self.init_output = MLPConstants.BIAS_INIT_OUT else: - if (MLPGroupTypes.INPUT in self.type): + if (MLPGroupTypes.INPUT in self.type and MLPGroupTypes.OUTPUT not in self.type): self.out_integr_en = 0 self.num_out_procs = 1 self.out_procs_list [0] = MLPOutputProcs.OUT_HARD_CLAMP From 29abe43d6b7051ec69296c355d879b051e442e03 Mon Sep 17 00:00:00 2001 From: "Luis A. Plana" Date: Sat, 11 Apr 2020 14:23:43 +0100 Subject: [PATCH 17/21] added debugging output to help diagnose timeout. Fixed output_mon_lens. --- c_code/input.c | 10 +++++----- c_code/sum.c | 12 ++++++------ c_code/threshold.c | 17 ++++++++++------- c_code/weight.c | 10 +++++----- output_mon_lens | 5 ++++- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/c_code/input.c b/c_code/input.c index 167f784..4e3e9bb 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -239,17 +239,17 @@ void done (uint ec) case SPINN_QUEUE_FULL: io_printf (IO_BUF, "packet queue full\n"); - + rt_error(RTE_SWERR); break; case SPINN_MEM_UNAVAIL: io_printf (IO_BUF, "malloc failed\n"); - + rt_error(RTE_SWERR); break; case SPINN_UNXPD_PKT: io_printf (IO_BUF, "unexpected packet received - abort!\n"); - + rt_error(RTE_SWERR); break; case SPINN_TIMEOUT_EXIT: @@ -257,8 +257,8 @@ void done (uint ec) epoch, example, phase, tick ); - #ifdef DEBUG_VRB - io_printf (IO_BUF, "(fd:%08x bd:%08x)\n", if_done, ib_done); + #ifdef DEBUG_TO + io_printf (IO_BUF, "(fd:%u bd:%u)\n", if_done, ib_done); #endif break; diff --git a/c_code/sum.c b/c_code/sum.c index 4088161..d29ad46 100644 --- a/c_code/sum.c +++ b/c_code/sum.c @@ -190,17 +190,17 @@ void done (uint ec) case SPINN_QUEUE_FULL: io_printf (IO_BUF, "packet queue full\n"); - + rt_error(RTE_SWERR); break; case SPINN_MEM_UNAVAIL: io_printf (IO_BUF, "malloc failed\n"); - + rt_error(RTE_SWERR); break; case SPINN_UNXPD_PKT: io_printf (IO_BUF, "unexpected packet received - abort!\n"); - + rt_error(RTE_SWERR); break; case SPINN_TIMEOUT_EXIT: @@ -208,12 +208,12 @@ void done (uint ec) epoch, example, phase, tick ); - #ifdef DEBUG_VRB - io_printf (IO_BUF, "(fd:%08x bd:%08x)\n", sf_done, sb_done); + #ifdef DEBUG_TO + io_printf (IO_BUF, "(fd:%u bd:%u)\n", sf_done, sb_done); for (uint i = 0; i < scfg.num_units; i++) { - io_printf (IO_BUF, "(fa:%08x ba:%08x)\n", + io_printf (IO_BUF, "(fa:%u ba:%u)\n", sf_arrived[i], sb_arrived[i] ); } diff --git a/c_code/threshold.c b/c_code/threshold.c index 07c4e54..6d8512a 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -355,17 +355,17 @@ void done (uint ec) case SPINN_QUEUE_FULL: io_printf (IO_BUF, "packet queue full\n"); - + rt_error(RTE_SWERR); break; case SPINN_MEM_UNAVAIL: io_printf (IO_BUF, "malloc failed\n"); - + rt_error(RTE_SWERR); break; case SPINN_UNXPD_PKT: io_printf (IO_BUF, "unexpected packet received - abort!\n"); - + rt_error(RTE_SWERR); break; case SPINN_TIMEOUT_EXIT: @@ -373,13 +373,16 @@ void done (uint ec) epoch, example, phase, tick ); - #ifdef DEBUG_VRB - io_printf (IO_BUF, "(tactive:%u ta:0x%08x/0x%08x tb:0x%08x/0x%08x)\n", + #ifdef DEBUG_TO + io_printf (IO_BUF, "(tactive:%u ta:%u/%u tb:%u/%u)\n", t_active, tf_arrived, tcfg.num_units, tb_arrived, tcfg.num_units ); - io_printf (IO_BUF, "(tsd:%u tsa:0x%08x/0x%08x)\n", - t_sync_done, t_sync_arrived, tcfg.fwd_sync_expected + io_printf (IO_BUF, "(tsr:%u tsa:%u/%u)\n", + t_sync_rdy, t_sync_arrived, tcfg.fwd_sync_expected + ); + io_printf (IO_BUF, "(tcr:%u)\n", + tf_chain_rdy ); #endif diff --git a/c_code/weight.c b/c_code/weight.c index 1501790..718adf4 100644 --- a/c_code/weight.c +++ b/c_code/weight.c @@ -219,17 +219,17 @@ void done (uint ec) case SPINN_QUEUE_FULL: io_printf (IO_BUF, "packet queue full\n"); - + rt_error(RTE_SWERR); break; case SPINN_MEM_UNAVAIL: io_printf (IO_BUF, "malloc failed\n"); - + rt_error(RTE_SWERR); break; case SPINN_UNXPD_PKT: io_printf (IO_BUF, "unexpected packet received - abort!\n"); - + rt_error(RTE_SWERR); break; case SPINN_TIMEOUT_EXIT: @@ -237,11 +237,11 @@ void done (uint ec) epoch, example, phase, tick ); - #ifdef DEBUG_VRB + #ifdef DEBUG_TO io_printf (IO_BUF, "(fp:%u fc:%u)\n", wf_procs, wf_comms); io_printf (IO_BUF, "(fptd:%u)\n", wf_thrds_pend); - io_printf (IO_BUF, "(fa:0x%08x ba:0x%08x)\n", + io_printf (IO_BUF, "(fa:%u ba:%u)\n", wf_arrived, wb_arrived ); #endif diff --git a/output_mon_lens b/output_mon_lens index 626e0f6..125bbd2 100755 --- a/output_mon_lens +++ b/output_mon_lens @@ -126,6 +126,7 @@ sub print_outputs sub main_loop { my $run = 1; + my $last_tick = 0; while ($run != 0) { @@ -155,7 +156,7 @@ sub main_loop if ($cmd == $HOST_FINAL) { - last; + $last_tick = 1;; } if ($cmd == $HOST_INFO) @@ -231,6 +232,8 @@ sub main_loop { print_outputs (); + last if $last_tick; + $grp_ctr[$inx] = 0; $ltick = $tick[$inx]; $tick[$inx] = 0xffffffff; From 4874bed8163244692e9cd855f633cf431073f788 Mon Sep 17 00:00:00 2001 From: "Luis A. Plana" Date: Mon, 13 Apr 2020 16:41:44 +0100 Subject: [PATCH 18/21] updated DEBUG data collecting and reporting. --- c_code/comms_i.c | 4 ++++ c_code/comms_s.c | 4 ++++ c_code/comms_t.c | 6 ++++-- c_code/comms_w.c | 22 ++++++++++++++++++++-- c_code/input.c | 10 +++++----- c_code/mlp_externs.h | 6 +++++- c_code/process_i.c | 8 ++++++-- c_code/process_s.c | 14 ++++++++++++-- c_code/process_t.c | 10 +++++++++- c_code/process_w.c | 13 +++++++++++++ c_code/sum.c | 30 +++++++++++++++++++----------- c_code/threshold.c | 30 +++++++++++++++++++++++------- c_code/weight.c | 16 +++++++++++----- 13 files changed, 135 insertions(+), 38 deletions(-) diff --git a/c_code/comms_i.c b/c_code/comms_i.c index a1e27ae..c87193f 100644 --- a/c_code/comms_i.c +++ b/c_code/comms_i.c @@ -16,6 +16,10 @@ // ------------------------------------------------------------------------ void i_receivePacket (uint key, uint payload) { +#ifdef DEBUG + pkt_recv++; +#endif + // check if stop packet if ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY) { diff --git a/c_code/comms_s.c b/c_code/comms_s.c index 6a0be8f..bdd74aa 100644 --- a/c_code/comms_s.c +++ b/c_code/comms_s.c @@ -16,6 +16,10 @@ // ------------------------------------------------------------------------ void s_receivePacket (uint key, uint payload) { +#ifdef DEBUG + pkt_recv++; +#endif + // check if stop packet if ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY) { diff --git a/c_code/comms_t.c b/c_code/comms_t.c index e78ff19..3eb530c 100644 --- a/c_code/comms_t.c +++ b/c_code/comms_t.c @@ -16,6 +16,10 @@ // ------------------------------------------------------------------------ void t_receivePacket (uint key, uint payload) { +#ifdef DEBUG + pkt_recv++; +#endif + // check if packet is stop type uint stop = ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY); if (stop) @@ -247,7 +251,6 @@ void t_syncPacket (uint ph) void t_forwardPacket (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_fwd++; if (phase == SPINN_BACKPROP) wrng_phs++; @@ -287,7 +290,6 @@ void t_forwardPacket (uint key, uint payload) void t_backpropPacket (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_bkp++; if (phase == SPINN_FORWARD) wrng_phs++; diff --git a/c_code/comms_w.c b/c_code/comms_w.c index d44da2a..e259209 100644 --- a/c_code/comms_w.c +++ b/c_code/comms_w.c @@ -16,6 +16,10 @@ // ------------------------------------------------------------------------ void w_receivePacket (uint key, uint payload) { +#ifdef DEBUG + pkt_recv++; +#endif + // check if packet is stop type uint stop = ((key & SPINN_TYPE_MASK) == SPINN_STOP_KEY); if (stop) @@ -53,6 +57,12 @@ void w_receivePacket (uint key, uint payload) { w_forwardPacket (key, payload); } +#ifdef DEBUG + else + { + pkt_fwbk++; + } +#endif } else { @@ -61,6 +71,12 @@ void w_receivePacket (uint key, uint payload) { w_backpropPacket (key, payload); } +#ifdef DEBUG + else + { + pkt_bwbk++; + } +#endif } } // ------------------------------------------------------------------------ @@ -127,6 +143,10 @@ void w_networkStopPacket (void) // ------------------------------------------------------------------------ void w_ldsrPacket (uint payload) { +#ifdef DEBUG + ldr_recv++; +#endif + // the final link delta sum for the epoch arrived w_lds_final = (lds_t) payload; @@ -159,7 +179,6 @@ void w_ldsrPacket (uint payload) void w_forwardPacket (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_fwd++; if (phase == SPINN_BACKPROP) wrng_phs++; @@ -218,7 +237,6 @@ void w_forwardPacket (uint key, uint payload) void w_backpropPacket (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_bkp++; if (phase == SPINN_FORWARD) wrng_phs++; diff --git a/c_code/input.c b/c_code/input.c index 4e3e9bb..670f375 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -267,15 +267,15 @@ void done (uint ec) // report diagnostics #ifdef DEBUG io_printf (IO_BUF, "total ticks:%d\n", tot_tick); - io_printf (IO_BUF, "recv:%d fwd:%d bkp:%d\n", pkt_recv, recv_fwd, recv_bkp); - io_printf (IO_BUF, "sent:%d fwd:%d bkp:%d\n", pkt_sent, sent_fwd, sent_bkp); + io_printf (IO_BUF, "total recv:%d\n", pkt_recv); + io_printf (IO_BUF, "total sent:%d\n", pkt_sent); + io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); + io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); - io_printf (IO_BUF, "sync recv:%d\n", spk_recv); - io_printf (IO_BUF, "sync sent:%d\n", spk_sent); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); - io_printf (IO_BUF, "stop sent:%d\n", stp_sent); + io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); #endif } // ------------------------------------------------------------------------ diff --git a/c_code/mlp_externs.h b/c_code/mlp_externs.h index 369bf2e..c950516 100644 --- a/c_code/mlp_externs.h +++ b/c_code/mlp_externs.h @@ -186,7 +186,9 @@ extern long_deriv_t * t_output_deriv_history; extern uint sent_bkp; // packets sent in BACKPROP phase extern uint pkt_recv; // total packets received extern uint recv_fwd; // packets received in FORWARD phase - extern uint recv_bkp; // packets received in BACKPROP phase + extern uint recv_bkp; // packets received in FORWARD phase + extern uint pkt_fwbk; // unused packets received in FORWARD phase + extern uint pkt_bwbk; // unused packets received in FORWARD phase extern uint spk_sent; // sync packets sent extern uint spk_recv; // sync packets received extern uint chn_sent; // chain packets sent @@ -199,6 +201,8 @@ extern long_deriv_t * t_output_deriv_history; extern uint lda_recv; // partial link_delta packets received extern uint ldt_sent; // total link_delta packets sent extern uint ldt_recv; // total link_delta packets received + extern uint ldr_sent; // link_delta packets sent + extern uint ldr_recv; // link_delta packets received extern uint tot_tick; // total number of ticks executed extern uint wght_ups; // number of weight updates done extern uint wrng_phs; // packets received in wrong phase diff --git a/c_code/process_i.c b/c_code/process_i.c index a0c7ea5..63037bf 100644 --- a/c_code/process_i.c +++ b/c_code/process_i.c @@ -70,7 +70,6 @@ void i_process (uint null0, uint null1) void i_forward_packet (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_fwd++; if (phase != SPINN_FORWARD) wrng_phs++; @@ -159,7 +158,6 @@ void i_forward_packet (uint key, uint payload) void i_backprop_packet (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_bkp++; if (phase != SPINN_BACKPROP) wrng_phs++; @@ -237,6 +235,9 @@ void if_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "if_tick: %d/%d\n", tick, tot_tick); #endif @@ -266,6 +267,9 @@ void ib_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "ib_tick: %d/%d\n", tick, tot_tick); #endif diff --git a/c_code/process_s.c b/c_code/process_s.c index 6de724a..412734f 100644 --- a/c_code/process_s.c +++ b/c_code/process_s.c @@ -102,6 +102,7 @@ void s_ldsa_packet (uint payload) while (!spin1_send_mc_packet (ldstKey, s_lds_part, WITH_PAYLOAD)); #ifdef DEBUG + pkt_sent++; ldt_sent++; #endif } @@ -156,6 +157,11 @@ void s_ldst_packet (uint payload) // send the final value of s_lds_part back to the w cores while (!spin1_send_mc_packet (ldsrKey, s_lds_part, WITH_PAYLOAD)); +#ifdef DEBUG + pkt_sent++; + ldr_sent++; +#endif + // access synchronisation semaphore with interrupts disabled uint cpsr = spin1_int_disable (); @@ -191,7 +197,6 @@ void s_ldst_packet (uint payload) void s_forward_packet (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_fwd++; if (phase != SPINN_FORWARD) wrng_phs++; @@ -290,7 +295,6 @@ void s_forward_packet (uint key, uint payload) void s_backprop_packet (uint key, uint payload) { #ifdef DEBUG - pkt_recv++; recv_bkp++; if (phase != SPINN_BACKPROP) wrng_phs++; @@ -422,6 +426,9 @@ void sf_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "sf_tick: %d/%d\n", tick, tot_tick); #endif @@ -451,6 +458,9 @@ void sb_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "sb_tick: %d/%d\n", tick, tot_tick); #endif diff --git a/c_code/process_t.c b/c_code/process_t.c index b00ff18..8a1d941 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -281,6 +281,9 @@ void tf_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "tf_tick: %d/%d\n", tick, tot_tick); #endif @@ -323,6 +326,9 @@ void tb_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "tb_tick: %d/%d\n", tick, tot_tick); #endif @@ -446,7 +452,7 @@ void t_advance_example (void) #endif // check if done with examples - //TODO: alternative algorithms for chosing example order! + //TODO: alternative algorithms for choosing example order! if (++example >= ncfg.num_examples) { if (tcfg.is_last_output_group) @@ -468,6 +474,7 @@ void t_advance_example (void) ); #ifdef DEBUG + pkt_sent++; stn_sent++; #endif @@ -674,6 +681,7 @@ void tf_send_stop (uint null0, uint null1) ); #ifdef DEBUG + pkt_sent++; if (tcfg.is_last_output_group) { stp_sent++; diff --git a/c_code/process_w.c b/c_code/process_w.c index 5f243e0..7fbe886 100644 --- a/c_code/process_w.c +++ b/c_code/process_w.c @@ -218,6 +218,7 @@ void wb_process (uint null0, uint null1) (uint) link_delta_sum_short, WITH_PAYLOAD) ); #ifdef DEBUG + pkt_sent++; lda_sent++; #endif } @@ -706,6 +707,13 @@ void wf_advance_tick (uint null0, uint null1) if (tick) { tot_tick++; + } +#endif + +#ifdef DEBUG_TICK + //NOTE: tick 0 is not a computation tick + if (tick) + { io_printf (IO_BUF, "wf_tick: %d/%d\n", tick, tot_tick); } #endif @@ -745,6 +753,9 @@ void wb_advance_tick (uint null0, uint null1) #ifdef DEBUG tot_tick++; +#endif + +#ifdef DEBUG_TICK io_printf (IO_BUF, "wb_tick: %d/%d\n", tick, tot_tick); #endif @@ -895,6 +906,7 @@ void w_advance_example (void) while (!spin1_send_mc_packet (wf_sync_key, 0, NO_PAYLOAD)); #ifdef DEBUG + pkt_sent++; spk_sent++; #endif } @@ -935,6 +947,7 @@ void w_switch_to_bp (void) //# while (!spin1_send_mc_packet (wb_sync_key, 0, NO_PAYLOAD)); //# #ifdef DEBUG +//# pkt_sent++; //# spk_sent++; //# #endif } diff --git a/c_code/sum.c b/c_code/sum.c index d29ad46..86aa5fc 100644 --- a/c_code/sum.c +++ b/c_code/sum.c @@ -96,8 +96,9 @@ scoreboard_t s_ldst_arrived; // keep track of the number of link delta su uint stp_recv = 0; // stop packets received uint stn_recv = 0; // network_stop packets received uint lda_recv = 0; // partial link_delta packets received - uint ldt_sent = 0; // total link_delta packets snet + uint ldt_sent = 0; // total link_delta packets sent uint ldt_recv = 0; // total link_delta packets received + uint ldr_sent = 0; // link_delta packets sent uint wrng_phs = 0; // packets received in wrong phase uint wrng_tck = 0; // FORWARD packets received in wrong tick uint wrng_btk = 0; // BACKPROP packets received in wrong tick @@ -213,8 +214,9 @@ void done (uint ec) for (uint i = 0; i < scfg.num_units; i++) { - io_printf (IO_BUF, "(fa:%u ba:%u)\n", - sf_arrived[i], sb_arrived[i] + io_printf (IO_BUF, "(fa[0]:%u ba[0]:%u fa[1]:%u ba[1]:%u)\n", + sf_arrived[0][i], sb_arrived[0][i], + sf_arrived[1][i], sb_arrived[1][i] ); } #endif @@ -225,19 +227,25 @@ void done (uint ec) // report diagnostics #ifdef DEBUG io_printf (IO_BUF, "total ticks:%d\n", tot_tick); - io_printf (IO_BUF, "recv:%d fwd:%d bkp:%d\n", pkt_recv, recv_fwd, recv_bkp); - io_printf (IO_BUF, "sent:%d fwd:%d bkp:%d\n", pkt_sent, sent_fwd, sent_bkp); + io_printf (IO_BUF, "total recv:%d\n", pkt_recv); + io_printf (IO_BUF, "total sent:%d\n", pkt_sent); + io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); + io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); - io_printf (IO_BUF, "sync recv:%d\n", spk_recv); - io_printf (IO_BUF, "sync sent:%d\n", spk_sent); + io_printf (IO_BUF, "ldsa recv:%d\n", lda_recv); + if (scfg.is_first_group) + { + io_printf (IO_BUF, "ldst recv:%d\n", ldt_recv); + io_printf (IO_BUF, "ldsr sent:%d\n", ldr_sent); + } + else + { + io_printf (IO_BUF, "ldst sent:%d\n", ldt_sent); + } io_printf (IO_BUF, "stop recv:%d\n", stp_recv); - io_printf (IO_BUF, "stop sent:%d\n", stp_sent); io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); - io_printf (IO_BUF, "ldsa recv:%d\n", lda_recv); - io_printf (IO_BUF, "ldst sent:%d\n", ldt_sent); - io_printf (IO_BUF, "ldst recv:%d\n", ldt_recv); #endif } // ------------------------------------------------------------------------ diff --git a/c_code/threshold.c b/c_code/threshold.c index 6d8512a..5af210a 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -397,17 +397,33 @@ void done (uint ec) // report diagnostics #ifdef DEBUG io_printf (IO_BUF, "total ticks:%d\n", tot_tick); - io_printf (IO_BUF, "recv:%d fwd:%d bkp:%d\n", pkt_recv, recv_fwd, recv_bkp); - io_printf (IO_BUF, "sent:%d fwd:%d bkp:%d\n", pkt_sent, sent_fwd, sent_bkp); + io_printf (IO_BUF, "total recv:%d\n", pkt_recv); + io_printf (IO_BUF, "total sent:%d\n", pkt_sent); + io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); + io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "sync recv:%d\n", spk_recv); - io_printf (IO_BUF, "sync sent:%d\n", spk_sent); - io_printf (IO_BUF, "chain recv:%d\n", chn_recv); - io_printf (IO_BUF, "chain sent:%d\n", chn_sent); - io_printf (IO_BUF, "stop recv:%d\n", stp_recv); - io_printf (IO_BUF, "stop sent:%d\n", stp_sent); + if (tcfg.is_first_output_group) + { + io_printf (IO_BUF, "chain recv: first\n"); + } + else + { + io_printf (IO_BUF, "chain recv:%d\n", chn_recv); + } + if (tcfg.is_last_output_group) + { + io_printf (IO_BUF, "stop sent:%d\n", stp_sent); + io_printf (IO_BUF, "stpn sent:%d\n", stn_sent); + } + else + { + io_printf (IO_BUF, "chain sent:%d\n", chn_sent); + io_printf (IO_BUF, "stop recv:%d\n", stp_recv); + io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); + } #endif } // ------------------------------------------------------------------------ diff --git a/c_code/weight.c b/c_code/weight.c index 718adf4..838dff5 100644 --- a/c_code/weight.c +++ b/c_code/weight.c @@ -116,12 +116,15 @@ activation_t * w_output_history; // history array for outputs uint pkt_recv = 0; // total packets received uint recv_fwd = 0; // packets received in FORWARD phase uint recv_bkp = 0; // packets received in BACKPROP phase + uint pkt_fwbk = 0; // unused packets received in FORWARD phase + uint pkt_bwbk = 0; // unused packets received in BACKPROP phase uint spk_sent = 0; // sync packets sent uint spk_recv = 0; // sync packets received uint stp_sent = 0; // stop packets sent uint stp_recv = 0; // stop packets received uint stn_recv = 0; // network_stop packets received uint lda_sent = 0; // partial link_delta packets sent + uint ldr_recv = 0; // link_delta packets received uint wrng_phs = 0; // packets received in wrong phase uint wrng_tck = 0; // FORWARD packets received in wrong tick uint wrng_btk = 0; // BACKPROP packets received in wrong tick @@ -252,17 +255,20 @@ void done (uint ec) // report diagnostics #ifdef DEBUG io_printf (IO_BUF, "total ticks:%d\n", tot_tick); - io_printf (IO_BUF, "recv:%d fwd:%d bkp:%d\n", pkt_recv, recv_fwd, recv_bkp); - io_printf (IO_BUF, "sent:%d fwd:%d bkp:%d\n", pkt_sent, sent_fwd, sent_bkp); + io_printf (IO_BUF, "total recv:%d\n", pkt_recv); + io_printf (IO_BUF, "total sent:%d\n", pkt_sent); + io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); + io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); + io_printf (IO_BUF, "unused recv: fwd:%d bkp:%d\n", pkt_fwbk, pkt_bwbk); io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); - io_printf (IO_BUF, "sync recv:%d\n", spk_recv); io_printf (IO_BUF, "sync sent:%d\n", spk_sent); + io_printf (IO_BUF, "ldsa sent:%d\n", lda_sent); + io_printf (IO_BUF, "ldsr recv:%d\n", ldr_recv); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); - io_printf (IO_BUF, "stop sent:%d\n", stp_sent); io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); - io_printf (IO_BUF, "ldsa sent:%d\n", lda_sent); + io_printf (IO_BUF, "------\n"); io_printf (IO_BUF, "weight updates:%d\n", wght_ups); #endif } From fbc750b7d0e50550af718640ca9692ef702e6935 Mon Sep 17 00:00:00 2001 From: "Luis A. Plana" Date: Mon, 13 Apr 2020 18:42:10 +0100 Subject: [PATCH 19/21] further changes to DEBUG reporting. --- c_code/input.c | 6 +++--- c_code/sum.c | 8 ++++---- c_code/threshold.c | 6 +++--- c_code/weight.c | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/c_code/input.c b/c_code/input.c index 670f375..ef82c36 100644 --- a/c_code/input.c +++ b/c_code/input.c @@ -271,11 +271,11 @@ void done (uint ec) io_printf (IO_BUF, "total sent:%d\n", pkt_sent); io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); - io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); - io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); - io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); + if (wrng_phs) io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); + if (wrng_tck) io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); + if (wrng_btk) io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); #endif } // ------------------------------------------------------------------------ diff --git a/c_code/sum.c b/c_code/sum.c index 86aa5fc..3726565 100644 --- a/c_code/sum.c +++ b/c_code/sum.c @@ -214,7 +214,7 @@ void done (uint ec) for (uint i = 0; i < scfg.num_units; i++) { - io_printf (IO_BUF, "(fa[0]:%u ba[0]:%u fa[1]:%u ba[1]:%u)\n", + io_printf (IO_BUF, "%2d: (fa[0]:%u ba[0]:%u fa[1]:%u ba[1]:%u)\n", i, sf_arrived[0][i], sb_arrived[0][i], sf_arrived[1][i], sb_arrived[1][i] ); @@ -231,9 +231,6 @@ void done (uint ec) io_printf (IO_BUF, "total sent:%d\n", pkt_sent); io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); - io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); - io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); - io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "ldsa recv:%d\n", lda_recv); if (scfg.is_first_group) { @@ -246,6 +243,9 @@ void done (uint ec) } io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); + if (wrng_phs) io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); + if (wrng_tck) io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); + if (wrng_btk) io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); #endif } // ------------------------------------------------------------------------ diff --git a/c_code/threshold.c b/c_code/threshold.c index 5af210a..9405353 100644 --- a/c_code/threshold.c +++ b/c_code/threshold.c @@ -401,9 +401,6 @@ void done (uint ec) io_printf (IO_BUF, "total sent:%d\n", pkt_sent); io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); - io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); - io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); - io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "sync recv:%d\n", spk_recv); if (tcfg.is_first_output_group) { @@ -424,6 +421,9 @@ void done (uint ec) io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); } + if (wrng_phs) io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); + if (wrng_tck) io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); + if (wrng_btk) io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); #endif } // ------------------------------------------------------------------------ diff --git a/c_code/weight.c b/c_code/weight.c index 838dff5..6015a87 100644 --- a/c_code/weight.c +++ b/c_code/weight.c @@ -260,14 +260,14 @@ void done (uint ec) io_printf (IO_BUF, "recv: fwd:%d bkp:%d\n", recv_fwd, recv_bkp); io_printf (IO_BUF, "sent: fwd:%d bkp:%d\n", sent_fwd, sent_bkp); io_printf (IO_BUF, "unused recv: fwd:%d bkp:%d\n", pkt_fwbk, pkt_bwbk); - io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); - io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); - io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "sync sent:%d\n", spk_sent); io_printf (IO_BUF, "ldsa sent:%d\n", lda_sent); io_printf (IO_BUF, "ldsr recv:%d\n", ldr_recv); io_printf (IO_BUF, "stop recv:%d\n", stp_recv); io_printf (IO_BUF, "stpn recv:%d\n", stn_recv); + if (wrng_phs) io_printf (IO_BUF, "wrong phase:%d\n", wrng_phs); + if (wrng_tck) io_printf (IO_BUF, "wrong tick:%d\n", wrng_tck); + if (wrng_btk) io_printf (IO_BUF, "wrong btick:%d\n", wrng_btk); io_printf (IO_BUF, "------\n"); io_printf (IO_BUF, "weight updates:%d\n", wght_ups); #endif From 53d6a6b4c2f90751327b91cd0232c2ba861362c3 Mon Sep 17 00:00:00 2001 From: Luis Plana Date: Tue, 14 Apr 2020 18:57:23 +0100 Subject: [PATCH 20/21] minor changes to make sure that last outputs are sent and shown. --- c_code/process_t.c | 2 +- output_mon_lens | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/c_code/process_t.c b/c_code/process_t.c index 8a1d941..a837ecc 100644 --- a/c_code/process_t.c +++ b/c_code/process_t.c @@ -488,7 +488,7 @@ void t_advance_example (void) if (++epoch >= ncfg.num_epochs) { // send final output packet to host, - if (tcfg.is_last_output_group && tcfg.write_out) + if (tcfg.write_out) { send_outputs_to_host (SPINN_HOST_FINAL, 0); } diff --git a/output_mon_lens b/output_mon_lens index 125bbd2..eed5ff3 100755 --- a/output_mon_lens +++ b/output_mon_lens @@ -9,6 +9,10 @@ use warnings; use IO::Socket::INET; +use IO::Handle; + +STDOUT->autoflush(1); + my $version = '0.99'; my $port = 17896; @@ -125,10 +129,9 @@ sub print_outputs sub main_loop { - my $run = 1; - my $last_tick = 0; + my $finish = 0; - while ($run != 0) + while (1) { my $rm = ''; vec ($rm, $socket_fd, 1) = 1; @@ -156,7 +159,7 @@ sub main_loop if ($cmd == $HOST_FINAL) { - $last_tick = 1;; + $finish = 1; } if ($cmd == $HOST_INFO) @@ -232,7 +235,7 @@ sub main_loop { print_outputs (); - last if $last_tick; + last if $finish; $grp_ctr[$inx] = 0; $ltick = $tick[$inx]; From 72099812fc7dfdd6203f10f57ae610bc20e9a8a8 Mon Sep 17 00:00:00 2001 From: "Luis A. Plana" Date: Tue, 14 Apr 2020 19:23:54 +0100 Subject: [PATCH 21/21] fixed a couple of typos in comments. --- c_code/mlp_externs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/c_code/mlp_externs.h b/c_code/mlp_externs.h index c950516..d1eba4a 100644 --- a/c_code/mlp_externs.h +++ b/c_code/mlp_externs.h @@ -186,9 +186,9 @@ extern long_deriv_t * t_output_deriv_history; extern uint sent_bkp; // packets sent in BACKPROP phase extern uint pkt_recv; // total packets received extern uint recv_fwd; // packets received in FORWARD phase - extern uint recv_bkp; // packets received in FORWARD phase + extern uint recv_bkp; // packets received in BACKPROP phase extern uint pkt_fwbk; // unused packets received in FORWARD phase - extern uint pkt_bwbk; // unused packets received in FORWARD phase + extern uint pkt_bwbk; // unused packets received in BACKPROP phase extern uint spk_sent; // sync packets sent extern uint spk_recv; // sync packets received extern uint chn_sent; // chain packets sent