From 3f66b56192ce781ce21bb027e308f7ed1640c57a Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 11:55:40 -0500 Subject: [PATCH 01/37] Hotfix: Fixed an issue wher TAN was being sent incorrectly --- core/federated/federate.c | 44 +++++++++++++++++++++++++++++---------- core/reactor_common.c | 5 ++++- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index 3a8ebf9fa..e01e73028 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1142,6 +1142,12 @@ void update_last_known_status_on_input_ports(tag_t tag) { // we do not update the last known status tag. if (compare_tags(tag, input_port_action->last_known_status_tag) >= 0) { + DEBUG_PRINT( + "Updating the last known status tag of port %d to (%ld, %u).", + i, + tag.time - get_start_time(), + tag.microstep + ); input_port_action->last_known_status_tag = tag; if (input_port_action->is_a_control_reaction_waiting) { notify = true; @@ -1183,6 +1189,12 @@ void update_last_known_status_on_input_port(tag_t tag, int port_id) { // semantics in tag.h. tag.microstep++; } + DEBUG_PRINT( + "Updating the last known status tag of port %d to (%ld, %u).", + port_id, + tag.time - get_start_time(), + tag.microstep + ); input_port_action->last_known_status_tag = tag; // If any control reaction is waiting, notify them that the status has changed if (input_port_action->is_a_control_reaction_waiting) { @@ -1601,7 +1613,7 @@ void handle_port_absent_message(int socket, int fed_id) { tag_t intended_tag = extract_tag(&(buffer[sizeof(uint16_t)+sizeof(uint16_t)])); LOG_PRINT("Handling port absent for tag (%lld, %u) for port %d.", - intended_tag.time - start_time, + intended_tag.time - get_start_time(), intended_tag.microstep, port_id, fed_id @@ -1712,7 +1724,7 @@ void handle_tagged_message(int socket, int fed_id) { if (action->is_physical) { // Messages sent on physical connections should be handled via handle_message(). - warning_print("Received a timed message on a physical connection. Time stamp will be lost."); + error_print_and_exit("Received a timed message on a physical connection."); } #ifdef FEDERATED_DECENTRALIZED // Only applicable for federated programs with decentralized coordination @@ -1789,7 +1801,15 @@ void handle_tagged_message(int socket, int fed_id) { // Since the message is intended for the current tag and a control reaction // was waiting for the message, trigger the corresponding reactions for this // message. - LOG_PRINT("Inserting reactions directly at tag (%lld, %u).", intended_tag.time - start_time, intended_tag.microstep); + tag_t current_tag = get_current_tag(); + DEBUG_PRINT( + "Inserting reactions directly at tag (%lld, %u). " + "Current tag: (%ld, %u).", + intended_tag.time - get_start_time(), + intended_tag.microstep, + current_tag.time - get_start_time(), + current_tag.microstep + ); action->intended_tag = intended_tag; _lf_insert_reactions_for_trigger(action, message_token); @@ -2612,14 +2632,16 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { } } } - // Next tag is greater than physical time and this fed has downstream - // federates. Need to send TAN rather than NET. - // TAN does not include a microstep and expects no reply. - // It is sent to enable downstream federates to advance. - _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); - _fed.last_sent_NET = tag; - LOG_PRINT("Sent Time Advance Notice (TAN) %lld to RTI.", - tag.time - start_time); + if(_lf_bounded_NET(&tag)) { + // Next tag is greater than physical time and this fed has downstream + // federates. Need to send TAN rather than NET. + // TAN does not include a microstep and expects no reply. + // It is sent to enable downstream federates to advance. + _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); + _fed.last_sent_NET = tag; + LOG_PRINT("Sent Time Advance Notice (TAN) %lld to RTI.", + tag.time - get_start_time()); + } if (!wait_for_reply) { LOG_PRINT("Not waiting physical time to advance further."); diff --git a/core/reactor_common.c b/core/reactor_common.c index b3b1db650..3723e15c2 100644 --- a/core/reactor_common.c +++ b/core/reactor_common.c @@ -1374,8 +1374,11 @@ trigger_handle_t _lf_insert_reactions_for_trigger(trigger_t* trigger, lf_token_t // Check for STP violation in the centralized coordination, which is a // critical error. if (is_STP_violated) { + tag_t tag = get_current_tag(); error_print_and_exit("Attempted to insert reactions for a trigger that had violated the STP offset" - " in centralized coordination."); + " in centralized coordination. Intended tag: (%ld, %u). Current tag: (%ld, %u).", + trigger->intended_tag.time - get_start_time(), trigger->intended_tag.microstep, + tag.time - get_start_time(), tag.microstep); } #endif #endif From 972fffbd4b421d3db491c7ee04111f8346757bfa Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 12:09:20 -0500 Subject: [PATCH 02/37] Slightly adjusted debug message --- core/federated/federate.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index e01e73028..505fb7e87 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1804,11 +1804,11 @@ void handle_tagged_message(int socket, int fed_id) { tag_t current_tag = get_current_tag(); DEBUG_PRINT( "Inserting reactions directly at tag (%lld, %u). " - "Current tag: (%ld, %u).", - intended_tag.time - get_start_time(), - intended_tag.microstep, + "Intended tag: (%ld, %u).", current_tag.time - get_start_time(), - current_tag.microstep + current_tag.microstep, + intended_tag.time - get_start_time(), + intended_tag.microstep ); action->intended_tag = intended_tag; _lf_insert_reactions_for_trigger(action, message_token); From 83d8d00fe6df82bc63a52e61afa425a023c551f4 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 14:15:21 -0500 Subject: [PATCH 03/37] Prevent transitive TAGs upon receiving a TAN --- core/federated/RTI/rti.c | 6 ++++-- core/federated/federate.c | 23 +++++++++++----------- core/reactor_common.c | 4 ++-- core/threaded/scheduler_sync_tag_advance.c | 10 ++++++---- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 1f0d693dd..b1a649b00 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -826,8 +826,10 @@ void handle_time_advance_notice(federate_t* fed) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); + for (int i = 0; i < fed->num_downstream; i++) { + federate_t* downstream = &_RTI.federates[fed->downstream[i]]; + send_advance_grant_if_safe(downstream); + } pthread_mutex_unlock(&_RTI.rti_mutex); } diff --git a/core/federated/federate.c b/core/federated/federate.c index 505fb7e87..66e87260e 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1793,7 +1793,8 @@ void handle_tagged_message(int socket, int fed_id) { // a message with intended_tag of 9 could arrive before the control reaction has had a chance // to exit. The port status is on the other hand changed in this thread, and thus, // can be checked in this scenario without this race condition. The message with - // intended_tag of 9 in this case needs to wait one microstep to be processed. + // intended_tag of 9 in this case needs to wait one microstep to be + // processed. if (compare_tags(intended_tag, get_current_tag()) <= 0 && action->is_a_control_reaction_waiting && // Check if a control reaction is waiting action->status == unknown // Check if the status of the port is still unknown @@ -2632,16 +2633,16 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { } } } - if(_lf_bounded_NET(&tag)) { - // Next tag is greater than physical time and this fed has downstream - // federates. Need to send TAN rather than NET. - // TAN does not include a microstep and expects no reply. - // It is sent to enable downstream federates to advance. - _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); - _fed.last_sent_NET = tag; - LOG_PRINT("Sent Time Advance Notice (TAN) %lld to RTI.", - tag.time - get_start_time()); - } + + + // Next tag is greater than physical time and this fed has downstream + // federates. Need to send TAN rather than NET. + // TAN does not include a microstep and expects no reply. + // It is sent to enable downstream federates to advance. + _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); + _fed.last_sent_NET = tag; + DEBUG_PRINT("Sent Time Advance Notice (TAN) %lld to RTI.", + tag.time - get_start_time()); if (!wait_for_reply) { LOG_PRINT("Not waiting physical time to advance further."); diff --git a/core/reactor_common.c b/core/reactor_common.c index 3723e15c2..bc405e8b1 100644 --- a/core/reactor_common.c +++ b/core/reactor_common.c @@ -1375,8 +1375,8 @@ trigger_handle_t _lf_insert_reactions_for_trigger(trigger_t* trigger, lf_token_t // critical error. if (is_STP_violated) { tag_t tag = get_current_tag(); - error_print_and_exit("Attempted to insert reactions for a trigger that had violated the STP offset" - " in centralized coordination. Intended tag: (%ld, %u). Current tag: (%ld, %u).", + error_print_and_exit("Attempted to insert reactions for a trigger that had an intended tag that was in the past. " + "This should not happen under centralized coordination. Intended tag: (%ld, %u). Current tag: (%ld, %u).", trigger->intended_tag.time - get_start_time(), trigger->intended_tag.microstep, tag.time - get_start_time(), tag.microstep); } diff --git a/core/threaded/scheduler_sync_tag_advance.c b/core/threaded/scheduler_sync_tag_advance.c index 4572be326..a36c615a9 100644 --- a/core/threaded/scheduler_sync_tag_advance.c +++ b/core/threaded/scheduler_sync_tag_advance.c @@ -68,10 +68,8 @@ bool _lf_logical_tag_completed = false; * This function assumes the caller holds the mutex lock. */ bool _lf_sched_should_stop_locked() { - // If this is not the very first step, notify that the previous step is complete - // and check against the stop tag to see whether this is the last step. + // If this is not the very first step, check against the stop tag to see whether this is the last step. if (_lf_logical_tag_completed) { - logical_tag_complete(current_tag); // If we are at the stop tag, do not call _lf_next_locked() // to prevent advancing the logical time. if (compare_tags(current_tag, stop_tag) >= 0) { @@ -90,6 +88,10 @@ bool _lf_sched_should_stop_locked() { * @return should_exit True if the worker thread should exit. False otherwise. */ bool _lf_sched_advance_tag_locked() { + // If this is not the very first step, notify that the previous step is complete. + if (_lf_logical_tag_completed) { + logical_tag_complete(current_tag); + } if (_lf_sched_should_stop_locked()) { return true; @@ -107,4 +109,4 @@ bool _lf_sched_advance_tag_locked() { DEBUG_PRINT("Scheduler: Done waiting for _lf_next_locked()."); return false; -} \ No newline at end of file +} From 8fe8d6048236b90a045ce8705b606d87136b2639 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 15:10:24 -0500 Subject: [PATCH 04/37] Update completed based on TAN not next_event --- core/federated/RTI/rti.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index b1a649b00..e8c590430 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -807,20 +807,15 @@ void handle_time_advance_notice(federate_t* fed) { LOG_PRINT("RTI received from federate %d the Time Advance Notice (TAN) %lld.", fed->id, fed->time_advance - start_time); - // If the TAN is greater than the most recently received NET, then - // update the NET to match the TAN. The NET is a promise that, absent - // network inputs, the federate will not produce an output with tag - // less than the NET. + // Update the completed tag to match the TAN. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; - if (compare_tags(ta, fed->next_event) > 0) { - fed->next_event = ta; - // We need to reply just as if this were a NET because it could unblock - // network input port control reactions. - // This is a side-effect of the combination of distributed cycles and - // physical actions in federates. FIXME: More explanation is needed. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); - } + fed->completed = ta; + // We need to reply just as if this were a NET because it could unblock + // network input port control reactions. + // This is a side-effect of the combination of distributed cycles and + // physical actions in federates. FIXME: More explanation is needed. + if (fed->num_upstream > 0) { + send_advance_grant_if_safe(fed); } // Check downstream federates to see whether they should now be granted a TAG. From 54b31c837c523036572c4cc44e1d4cc693b7bc80 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 15:20:42 -0500 Subject: [PATCH 05/37] Update next_event unconditionally --- core/federated/RTI/rti.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index e8c590430..48ca1b4af 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -807,9 +807,11 @@ void handle_time_advance_notice(federate_t* fed) { LOG_PRINT("RTI received from federate %d the Time Advance Notice (TAN) %lld.", fed->id, fed->time_advance - start_time); - // Update the completed tag to match the TAN. + // Update the NET to match the TAN. The NET is a promise that, absent + // network inputs, the federate will not produce an output with tag + // less than the NET. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; - fed->completed = ta; + fed->next_event = ta; // We need to reply just as if this were a NET because it could unblock // network input port control reactions. // This is a side-effect of the combination of distributed cycles and From ebb19dc95564971f7587b0221c94c730a9fc874a Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 15:40:31 -0500 Subject: [PATCH 06/37] Revert "Update next_event unconditionally" This reverts commit 54b31c837c523036572c4cc44e1d4cc693b7bc80. --- core/federated/RTI/rti.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 48ca1b4af..e8c590430 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -807,11 +807,9 @@ void handle_time_advance_notice(federate_t* fed) { LOG_PRINT("RTI received from federate %d the Time Advance Notice (TAN) %lld.", fed->id, fed->time_advance - start_time); - // Update the NET to match the TAN. The NET is a promise that, absent - // network inputs, the federate will not produce an output with tag - // less than the NET. + // Update the completed tag to match the TAN. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; - fed->next_event = ta; + fed->completed = ta; // We need to reply just as if this were a NET because it could unblock // network input port control reactions. // This is a side-effect of the combination of distributed cycles and From b0e85002819c5239bf43feaa3af436200af3aadd Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 4 Apr 2022 15:40:46 -0500 Subject: [PATCH 07/37] Revert "Update completed based on TAN not next_event" This reverts commit 8fe8d6048236b90a045ce8705b606d87136b2639. --- core/federated/RTI/rti.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index e8c590430..b1a649b00 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -807,15 +807,20 @@ void handle_time_advance_notice(federate_t* fed) { LOG_PRINT("RTI received from federate %d the Time Advance Notice (TAN) %lld.", fed->id, fed->time_advance - start_time); - // Update the completed tag to match the TAN. + // If the TAN is greater than the most recently received NET, then + // update the NET to match the TAN. The NET is a promise that, absent + // network inputs, the federate will not produce an output with tag + // less than the NET. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; - fed->completed = ta; - // We need to reply just as if this were a NET because it could unblock - // network input port control reactions. - // This is a side-effect of the combination of distributed cycles and - // physical actions in federates. FIXME: More explanation is needed. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); + if (compare_tags(ta, fed->next_event) > 0) { + fed->next_event = ta; + // We need to reply just as if this were a NET because it could unblock + // network input port control reactions. + // This is a side-effect of the combination of distributed cycles and + // physical actions in federates. FIXME: More explanation is needed. + if (fed->num_upstream > 0) { + send_advance_grant_if_safe(fed); + } } // Check downstream federates to see whether they should now be granted a TAG. From 808a189fdabd3bb88550b887ed97d2630aebf981 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Tue, 5 Apr 2022 13:02:02 -0500 Subject: [PATCH 08/37] Revert back to the transitive logic for TAN messages in the RTI --- core/federated/RTI/rti.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index b1a649b00..1f0d693dd 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -826,10 +826,8 @@ void handle_time_advance_notice(federate_t* fed) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - for (int i = 0; i < fed->num_downstream; i++) { - federate_t* downstream = &_RTI.federates[fed->downstream[i]]; - send_advance_grant_if_safe(downstream); - } + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(fed, visited); pthread_mutex_unlock(&_RTI.rti_mutex); } From 5ac67c4913120026f3659afb60d480b6dd543399 Mon Sep 17 00:00:00 2001 From: eal Date: Tue, 5 Apr 2022 17:23:52 -0700 Subject: [PATCH 09/37] Address a race condition Race condition is where a NET message from a federate from a previous cycle crosses a message to the federate being forwarded by the RTI, which causes the RTI's view of the NET of the federate to be incorrect. --- core/federated/RTI/rti.c | 119 ++++++++++++++++++++++++++++---------- core/federated/RTI/rti.h | 1 + core/federated/federate.c | 18 ++++-- 3 files changed, 102 insertions(+), 36 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 1f0d693dd..9e2d6d0a9 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -237,7 +237,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); read_from_socket_errexit(sending_federate->socket, message_size, &(buffer[1]), - " RTI failed ot read port absent message from federate %u.", + " RTI failed to read port absent message from federate %u.", sending_federate->id); // Need to acquire the mutex lock to ensure that the thread handling @@ -254,6 +254,20 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf pthread_mutex_unlock(&_RTI.rti_mutex); warning_print("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); + // FIXME: Do not exit! + error_print_and_exit("Fed status: next_event (%lld, %d), " + "completed (%lld, %d), " + "last_granted (%lld, %d), " + "last_provisionally_granted (%lld, %d).", + _RTI.federates[federate_id].next_event.time - start_time, + _RTI.federates[federate_id].next_event.microstep, + _RTI.federates[federate_id].completed.time - start_time, + _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].last_granted.time - start_time, + _RTI.federates[federate_id].last_granted.microstep, + _RTI.federates[federate_id].last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].last_provisionally_granted.microstep + ); return; } @@ -330,6 +344,20 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { pthread_mutex_unlock(&_RTI.rti_mutex); warning_print("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); + // FIXME: Do not exit! + error_print_and_exit("Fed status: next_event (%lld, %d), " + "completed (%lld, %d), " + "last_granted (%lld, %d), " + "last_provisionally_granted (%lld, %d).", + _RTI.federates[federate_id].next_event.time - start_time, + _RTI.federates[federate_id].next_event.microstep, + _RTI.federates[federate_id].completed.time - start_time, + _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].last_granted.time - start_time, + _RTI.federates[federate_id].last_granted.microstep, + _RTI.federates[federate_id].last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].last_provisionally_granted.microstep + ); return; } @@ -341,6 +369,10 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // updated. if (compare_tags(_RTI.federates[federate_id].next_event, intended_tag) > 0) { _RTI.federates[federate_id].next_event = intended_tag; + // Flag the next_field as being the result of an in-transit message. + // This will prevent overwriting the field until a LTC message is + // received with at least as large a tag. + _RTI.federates[federate_id].in_transit_message = true; } // Forward the message or message chunk. @@ -764,25 +796,35 @@ void handle_next_event_tag(federate_t* fed) { // select() mechanism to read and process // federates' buffers in an orderly fashion - - fed->next_event = extract_tag(buffer); - - LOG_PRINT("RTI received from federate %d the Next Event Tag (NET) (%lld, %u).", - fed->id, fed->next_event.time - start_time, - fed->next_event.microstep); - - // Check to see whether we can reply now with a time advance grant. - // If the federate has no upstream federates, then it does not wait for - // nor expect a reply. It just proceeds to advance time. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); + // Before overwriting next_event, if the current next_event value is due + // to a message in transit, then make sure an LTC has been received that is + // at least as large. + tag_t received_next_event = extract_tag(buffer); + if (!fed->in_transit_message || compare_tags(fed->completed, received_next_event) >= 0) { + fed->next_event = received_next_event; + fed->in_transit_message = false; + + LOG_PRINT("RTI received from federate %d the Next Event Tag (NET) (%lld, %u).", + fed->id, fed->next_event.time - start_time, + fed->next_event.microstep); + + // Check to see whether we can reply now with a time advance grant. + // If the federate has no upstream federates, then it does not wait for + // nor expect a reply. It just proceeds to advance time. + if (fed->num_upstream > 0) { + send_advance_grant_if_safe(fed); + } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(fed, visited); + } else { + LOG_PRINT("RTI received from federate %d the Next Event Tag (NET) (%lld, %u), " + "but ignoring it because there is a message in transit to the federate.", + fed->id, fed->next_event.time - start_time, + fed->next_event.microstep); } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); - pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -813,22 +855,31 @@ void handle_time_advance_notice(federate_t* fed) { // less than the NET. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; if (compare_tags(ta, fed->next_event) > 0) { - fed->next_event = ta; - // We need to reply just as if this were a NET because it could unblock - // network input port control reactions. - // This is a side-effect of the combination of distributed cycles and - // physical actions in federates. FIXME: More explanation is needed. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); + if (!fed->in_transit_message || compare_tags(fed->completed, ta) >= 0) { + fed->next_event = ta; + fed->in_transit_message = false; + // We need to reply just as if this were a NET because it could unblock + // network input port control reactions. + // This is a side-effect of the combination of distributed cycles and + // physical actions in federates. FIXME: More explanation is needed. + if (fed->num_upstream > 0) { + send_advance_grant_if_safe(fed); + } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(fed, visited); + } else { + LOG_PRINT("RTI ignoring TAN %lld from federate %d " + "because a message is in transit to the federate.", + fed->time_advance - start_time, fed->id); } + } else { + warning_print("RTI: Federate %d has reported a NET ahead of physical time, " + "but now it is sending a TAN, which indicates it shouldn't have.", fed->id); } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); - pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -863,6 +914,7 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { if (compare_tags(_RTI.federates[i].next_event, _RTI.max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. _RTI.federates[i].next_event = _RTI.max_stop_tag; + _RTI.federates[i].in_transit_message = false; } write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", _RTI.federates[i].id); @@ -1360,6 +1412,8 @@ void handle_federate_resign(federate_t *my_fed) { // Indicate that there will no further events from this federate. my_fed->next_event = FOREVER_TAG; + my_fed->in_transit_message = false; + my_fed->time_advance = FOREVER; // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket, @@ -1845,6 +1899,7 @@ void initialize_federate(uint16_t id) { _RTI.federates[id].last_granted = NEVER_TAG; _RTI.federates[id].last_provisionally_granted = NEVER_TAG; _RTI.federates[id].next_event = NEVER_TAG; + _RTI.federates[id].in_transit_message = false; _RTI.federates[id].time_advance = NEVER; _RTI.federates[id].state = NOT_CONNECTED; _RTI.federates[id].upstream = NULL; diff --git a/core/federated/RTI/rti.h b/core/federated/RTI/rti.h index 73391339c..ed5023790 100644 --- a/core/federated/RTI/rti.h +++ b/core/federated/RTI/rti.h @@ -76,6 +76,7 @@ typedef struct federate_t { tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). + bool in_transit_message;// Indicator that next_event was set by the RTI because of a message destined to the federate. instant_t time_advance; // Most recent TAN received from the federate (or NEVER if none received). fed_state_t state; // State of the federate. int* upstream; // Array of upstream federate ids. diff --git a/core/federated/federate.c b/core/federated/federate.c index 66e87260e..76142080b 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2585,6 +2585,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // What we do next depends on whether the NET has been bounded by // physical time or by an event on the event queue. if (!tag_bounded_by_physical_time) { + // This if statement does not fall through but rather returns. // NET is not bounded by physical time or has no downstream federates. // Normal case. _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag, wait_for_reply); @@ -2619,7 +2620,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { if (lf_cond_wait(&event_q_changed, &mutex) != 0) { error_print("Wait error."); } - // Either a TAG or PTAG arrived. + // Either a TAG or PTAG arrived or something appeared on the event queue. if (!_fed.waiting_for_TAG) { // _fed.last_TAG will have been set by the thread receiving the TAG message that // set _fed.waiting_for_TAG to false. @@ -2628,12 +2629,21 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // Check whether the new event on the event queue requires sending a new NET. tag_t next_tag = get_next_event_tag(); if (compare_tags(next_tag, tag) != 0) { - _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); - _fed.last_sent_NET = next_tag; + // The next tag may also have to be bounded by physical time. + if (_lf_bounded_NET(&next_tag)) { + _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, next_tag.time, wait_for_reply); + _fed.last_sent_NET = next_tag; + DEBUG_PRINT("Sent Time Advance Notice (TAN) %lld to RTI.", + next_tag.time - get_start_time()); + } else { + _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); + _fed.last_sent_NET = next_tag; + LOG_PRINT("Sent next event tag (NET) (%lld, %u) to RTI.", + next_tag.time - start_time, next_tag.microstep); + } } } } - // Next tag is greater than physical time and this fed has downstream // federates. Need to send TAN rather than NET. From e59095bfc978290d9e74bacea81cf2b54466afb8 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 27 May 2022 09:38:03 -0500 Subject: [PATCH 10/37] Fix merge artifacts --- core/federated/RTI/rti.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index c6379a15c..4cd00ba45 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -255,7 +255,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); // FIXME: Do not exit! - error_print_and_exit("Fed status: next_event (%lld, %d), " + lf_print_error_and_exit("Fed status: next_event (%lld, %d), " "completed (%lld, %d), " "last_granted (%lld, %d), " "last_provisionally_granted (%lld, %d).", @@ -345,7 +345,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); // FIXME: Do not exit! - error_print_and_exit("Fed status: next_event (%lld, %d), " + lf_print_error_and_exit("Fed status: next_event (%lld, %d), " "completed (%lld, %d), " "last_granted (%lld, %d), " "last_provisionally_granted (%lld, %d).", @@ -882,12 +882,12 @@ void handle_time_advance_notice(federate_t* fed) { bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. send_downstream_advance_grants_if_safe(fed, visited); } else { - LOG_PRINT("RTI ignoring TAN %lld from federate %d " + LF_PRINT_LOG("RTI ignoring TAN %lld from federate %d " "because a message is in transit to the federate.", fed->time_advance - start_time, fed->id); } } else { - warning_print("RTI: Federate %d has reported a NET ahead of physical time, " + lf_print_warning("RTI: Federate %d has reported a NET ahead of physical time, " "but now it is sending a TAN, which indicates it shouldn't have.", fed->id); } From 29c3f042c474e3c7f3adaf8db228696c21bd83ad Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 27 May 2022 10:35:15 -0500 Subject: [PATCH 11/37] Fixed more merge artifacts --- core/federated/federate.c | 10 +++++----- core/reactor_common.c | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index 97f050c1d..42ea399a7 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1142,7 +1142,7 @@ void update_last_known_status_on_input_ports(tag_t tag) { // we do not update the last known status tag. if (lf_tag_compare(tag, input_port_action->last_known_status_tag) >= 0) { - DEBUG_PRINT( + LF_PRINT_DEBUG( "Updating the last known status tag of port %d to (%ld, %u).", i, tag.time - lf_time_start(), @@ -1189,7 +1189,7 @@ void update_last_known_status_on_input_port(tag_t tag, int port_id) { // semantics in tag.h. tag.microstep++; } - DEBUG_PRINT( + LF_PRINT_DEBUG( "Updating the last known status tag of port %d to (%ld, %u).", port_id, tag.time - lf_time_start(), @@ -2643,12 +2643,12 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { if (_lf_bounded_NET(&next_tag)) { _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, next_tag.time, wait_for_reply); _fed.last_sent_NET = next_tag; - DEBUG_PRINT("Sent Time Advance Notice (TAN) %lld to RTI.", - next_tag.time - get_start_time()); + LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", + next_tag.time - lf_time_start()); } else { _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); _fed.last_sent_NET = next_tag; - LOG_PRINT("Sent next event tag (NET) (%lld, %u) to RTI.", + LF_PRINT_LOG("Sent next event tag (NET) (%lld, %u) to RTI.", next_tag.time - start_time, next_tag.microstep); } } diff --git a/core/reactor_common.c b/core/reactor_common.c index 8217a3a62..0c0fb03fc 100644 --- a/core/reactor_common.c +++ b/core/reactor_common.c @@ -1374,11 +1374,11 @@ trigger_handle_t _lf_insert_reactions_for_trigger(trigger_t* trigger, lf_token_t if (is_STP_violated) { lf_print_error_and_exit("Attempted to insert reactions for a trigger that had an intended tag that was in the past. " "This should not happen under centralized coordination. Intended tag: (%ld, %u). Current tag: (%ld, %u).", - trigger->intended_tag.time - get_start_time(), + trigger->intended_tag.time - lf_time_start(), trigger->intended_tag.microstep, lf_time_logical_elapsed(), lf_tag().microstep); -======= + } #endif #endif From 40b190eab9cd8fef8caa1ea2f98274c508bacfc2 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Sun, 29 May 2022 17:15:24 -0500 Subject: [PATCH 12/37] Do not exit --- core/federated/RTI/rti.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 4cd00ba45..30697ff87 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -254,8 +254,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf pthread_mutex_unlock(&_RTI.rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); - // FIXME: Do not exit! - lf_print_error_and_exit("Fed status: next_event (%lld, %d), " + LF_PRINT_LOG("Fed status: next_event (%lld, %d), " "completed (%lld, %d), " "last_granted (%lld, %d), " "last_provisionally_granted (%lld, %d).", @@ -344,8 +343,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { pthread_mutex_unlock(&_RTI.rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); - // FIXME: Do not exit! - lf_print_error_and_exit("Fed status: next_event (%lld, %d), " + LF_PRINT_LOG("Fed status: next_event (%lld, %d), " "completed (%lld, %d), " "last_granted (%lld, %d), " "last_provisionally_granted (%lld, %d).", From 77449b9e260fefabf376a4d2be9097e4d312eaa8 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Sun, 29 May 2022 17:26:48 -0500 Subject: [PATCH 13/37] Removed in_transit_message because it's too coarse-grained --- core/federated/RTI/rti.c | 96 +++++++++++++--------------------------- 1 file changed, 30 insertions(+), 66 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 30697ff87..807ee9439 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -359,20 +359,6 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { return; } - // If the destination federate has previously sent a NET that is larger - // than the indended tag of this message, then reset that NET to be equal - // to the indended tag of this message. This is needed because the NET - // is a promise that is valid only in the absence of network inputs, - // and now there is a network input. Hence, the promise needs to be - // updated. - if (lf_tag_compare(_RTI.federates[federate_id].next_event, intended_tag) > 0) { - _RTI.federates[federate_id].next_event = intended_tag; - // Flag the next_field as being the result of an in-transit message. - // This will prevent overwriting the field until a LTC message is - // received with at least as large a tag. - _RTI.federates[federate_id].in_transit_message = true; - } - // Forward the message or message chunk. int destination_socket = _RTI.federates[federate_id].socket; @@ -805,35 +791,24 @@ void handle_next_event_tag(federate_t* fed) { // select() mechanism to read and process // federates' buffers in an orderly fashion - // Before overwriting next_event, if the current next_event value is due - // to a message in transit, then make sure an LTC has been received that is - // at least as large. - tag_t received_next_event = extract_tag(buffer); - if (!fed->in_transit_message || lf_tag_compare(fed->completed, received_next_event) >= 0) { - fed->next_event = received_next_event; - fed->in_transit_message = false; - - LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%lld, %u).", - fed->id, fed->next_event.time - start_time, - fed->next_event.microstep); - - // Check to see whether we can reply now with a time advance grant. - // If the federate has no upstream federates, then it does not wait for - // nor expect a reply. It just proceeds to advance time. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); - } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); - } else { - LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%lld, %u), " - "but ignoring it because there is a message in transit to the federate.", - fed->id, fed->next_event.time - start_time, - fed->next_event.microstep); + fed->next_event = extract_tag(buffer); + + LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%lld, %u).", + fed->id, fed->next_event.time - start_time, + fed->next_event.microstep); + + // Check to see whether we can reply now with a time advance grant. + // If the federate has no upstream federates, then it does not wait for + // nor expect a reply. It just proceeds to advance time. + if (fed->num_upstream > 0) { + send_advance_grant_if_safe(fed); } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(fed, visited); + pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -864,31 +839,22 @@ void handle_time_advance_notice(federate_t* fed) { // less than the NET. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; if (lf_tag_compare(ta, fed->next_event) > 0) { - if (!fed->in_transit_message || lf_tag_compare(fed->completed, ta) >= 0) { - fed->next_event = ta; - fed->in_transit_message = false; - // We need to reply just as if this were a NET because it could unblock - // network input port control reactions. - // This is a side-effect of the combination of distributed cycles and - // physical actions in federates. FIXME: More explanation is needed. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); - } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); - } else { - LF_PRINT_LOG("RTI ignoring TAN %lld from federate %d " - "because a message is in transit to the federate.", - fed->time_advance - start_time, fed->id); + fed->next_event = ta; + // We need to reply just as if this were a NET because it could unblock + // network input port control reactions. + // This is a side-effect of the combination of distributed cycles and + // physical actions in federates. FIXME: More explanation is needed. + if (fed->num_upstream > 0) { + send_advance_grant_if_safe(fed); } - } else { - lf_print_warning("RTI: Federate %d has reported a NET ahead of physical time, " - "but now it is sending a TAN, which indicates it shouldn't have.", fed->id); } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(fed, visited); + pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -1421,7 +1387,6 @@ void handle_federate_resign(federate_t *my_fed) { // Indicate that there will no further events from this federate. my_fed->next_event = FOREVER_TAG; - my_fed->in_transit_message = false; my_fed->time_advance = FOREVER; @@ -1908,7 +1873,6 @@ void initialize_federate(uint16_t id) { _RTI.federates[id].last_granted = NEVER_TAG; _RTI.federates[id].last_provisionally_granted = NEVER_TAG; _RTI.federates[id].next_event = NEVER_TAG; - _RTI.federates[id].in_transit_message = false; _RTI.federates[id].time_advance = NEVER; _RTI.federates[id].state = NOT_CONNECTED; _RTI.federates[id].upstream = NULL; From 8e18f27b27a4dbe8f69edf4128c86e08a1de9d2e Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 00:17:46 -0500 Subject: [PATCH 14/37] Moved handle_timed_message and adjusted logic a bit --- core/federated/RTI/rti.c | 300 +++++++++++++-------- core/threaded/scheduler_sync_tag_advance.c | 5 +- 2 files changed, 183 insertions(+), 122 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 807ee9439..629ccd792 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -289,118 +289,6 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf pthread_mutex_unlock(&_RTI.rti_mutex); } -/** - * Handle a timed message being received from a federate by the RTI to relay to another federate. - * - * This function assumes the caller does not hold the mutex. - * - * @param sending_federate The sending federate. - * @param buffer The buffer to read into (the first byte is already there). - */ -void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { - size_t header_size = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(int64_t) + sizeof(uint32_t); - // Read the header, minus the first byte which has already been read. - read_from_socket_errexit(sending_federate->socket, header_size - 1, &(buffer[1]), "RTI failed to read the timed message header from remote federate."); - // Extract the header information. of the sender - uint16_t reactor_port_id; - uint16_t federate_id; - size_t length; - tag_t intended_tag; - // Extract information from the header. - extract_timed_header(&(buffer[1]), &reactor_port_id, &federate_id, &length, &intended_tag); - - size_t total_bytes_to_read = length + header_size; - size_t bytes_to_read = length; - - if (FED_COM_BUFFER_SIZE < header_size + 1) { - lf_print_error_and_exit("Buffer size (%d) is not large enough to " - "read the header plus one byte.", - FED_COM_BUFFER_SIZE); - } - - // Cut up the payload in chunks. - if (bytes_to_read > FED_COM_BUFFER_SIZE - header_size) { - bytes_to_read = FED_COM_BUFFER_SIZE - header_size; - } - - LF_PRINT_LOG("RTI received message from federate %d for federate %u port %u. Forwarding.", - sending_federate->id, federate_id, reactor_port_id); - - read_from_socket_errexit(sending_federate->socket, bytes_to_read, &(buffer[header_size]), - "RTI failed to read timed message from federate %d.", federate_id); - size_t bytes_read = bytes_to_read + header_size; - // Following only works for string messages. - // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); - - // Need to acquire the mutex lock to ensure that the thread handling - // messages coming from the socket connected to the destination does not - // issue a TAG before this message has been forwarded. - pthread_mutex_lock(&_RTI.rti_mutex); - - // If the destination federate is no longer connected, issue a warning - // and return. - if (_RTI.federates[federate_id].state == NOT_CONNECTED) { - pthread_mutex_unlock(&_RTI.rti_mutex); - lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", - federate_id); - LF_PRINT_LOG("Fed status: next_event (%lld, %d), " - "completed (%lld, %d), " - "last_granted (%lld, %d), " - "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].next_event.time - start_time, - _RTI.federates[federate_id].next_event.microstep, - _RTI.federates[federate_id].completed.time - start_time, - _RTI.federates[federate_id].completed.microstep, - _RTI.federates[federate_id].last_granted.time - start_time, - _RTI.federates[federate_id].last_granted.microstep, - _RTI.federates[federate_id].last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].last_provisionally_granted.microstep - ); - return; - } - - // Forward the message or message chunk. - int destination_socket = _RTI.federates[federate_id].socket; - - LF_PRINT_DEBUG( - "RTI forwarding message to port %d of federate %d of length %d.", - reactor_port_id, - federate_id, - length - ); - - // Need to make sure that the destination federate's thread has already - // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[federate_id].state == PENDING) { - // Need to wait here. - pthread_cond_wait(&_RTI.sent_start_time, &_RTI.rti_mutex); - } - write_to_socket_errexit(destination_socket, bytes_read, buffer, - "RTI failed to forward message to federate %d.", federate_id); - - // The message length may be longer than the buffer, - // in which case we have to handle it in chunks. - size_t total_bytes_read = bytes_read; - while (total_bytes_read < total_bytes_to_read) { - LF_PRINT_DEBUG("Forwarding message in chunks."); - bytes_to_read = total_bytes_to_read - total_bytes_read; - if (bytes_to_read > FED_COM_BUFFER_SIZE) { - bytes_to_read = FED_COM_BUFFER_SIZE; - } - read_from_socket_errexit(sending_federate->socket, bytes_to_read, buffer, - "RTI failed to read message chunks."); - total_bytes_read += bytes_to_read; - - // FIXME: a mutex needs to be held for this so that other threads - // do not write to destination_socket and cause interleaving. However, - // holding the _RTI.rti_mutex might be very expensive. Instead, each outgoing - // socket should probably have its own mutex. - write_to_socket_errexit(destination_socket, bytes_to_read, buffer, - "RTI failed to send message chunks."); - } - pthread_mutex_unlock(&_RTI.rti_mutex); -} - /** * Send a tag advance grant (TAG) message to the specified federate. * Do not send it if a previously sent PTAG was greater or if a @@ -739,6 +627,145 @@ void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { } } + +/** + * Handle a timed message being received from a federate by the RTI to relay to another federate. + * + * This function assumes the caller does not hold the mutex. + * + * @param sending_federate The sending federate. + * @param buffer The buffer to read into (the first byte is already there). + */ +void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { + size_t header_size = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(int64_t) + sizeof(uint32_t); + // Read the header, minus the first byte which has already been read. + read_from_socket_errexit(sending_federate->socket, header_size - 1, &(buffer[1]), "RTI failed to read the timed message header from remote federate."); + // Extract the header information. of the sender + uint16_t reactor_port_id; + uint16_t federate_id; + size_t length; + tag_t intended_tag; + // Extract information from the header. + extract_timed_header(&(buffer[1]), &reactor_port_id, &federate_id, &length, &intended_tag); + + size_t total_bytes_to_read = length + header_size; + size_t bytes_to_read = length; + + if (FED_COM_BUFFER_SIZE < header_size + 1) { + lf_print_error_and_exit("Buffer size (%d) is not large enough to " + "read the header plus one byte.", + FED_COM_BUFFER_SIZE); + } + + // Cut up the payload in chunks. + if (bytes_to_read > FED_COM_BUFFER_SIZE - header_size) { + bytes_to_read = FED_COM_BUFFER_SIZE - header_size; + } + + LF_PRINT_LOG("RTI received message from federate %d for federate %u port %u. Forwarding.", + sending_federate->id, federate_id, reactor_port_id); + + read_from_socket_errexit(sending_federate->socket, bytes_to_read, &(buffer[header_size]), + "RTI failed to read timed message from federate %d.", federate_id); + size_t bytes_read = bytes_to_read + header_size; + // Following only works for string messages. + // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); + + // Need to acquire the mutex lock to ensure that the thread handling + // messages coming from the socket connected to the destination does not + // issue a TAG before this message has been forwarded. + pthread_mutex_lock(&_RTI.rti_mutex); + + // If the destination federate is no longer connected, issue a warning + // and return. + if (_RTI.federates[federate_id].state == NOT_CONNECTED) { + pthread_mutex_unlock(&_RTI.rti_mutex); + lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", + federate_id); + LF_PRINT_LOG("Fed status: next_event (%lld, %d), " + "completed (%lld, %d), " + "last_granted (%lld, %d), " + "last_provisionally_granted (%lld, %d).", + _RTI.federates[federate_id].next_event.time - start_time, + _RTI.federates[federate_id].next_event.microstep, + _RTI.federates[federate_id].completed.time - start_time, + _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].last_granted.time - start_time, + _RTI.federates[federate_id].last_granted.microstep, + _RTI.federates[federate_id].last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].last_provisionally_granted.microstep + ); + return; + } + + // Forward the message or message chunk. + int destination_socket = _RTI.federates[federate_id].socket; + + LF_PRINT_DEBUG( + "RTI forwarding message to port %d of federate %d of length %d.", + reactor_port_id, + federate_id, + length + ); + + // Need to make sure that the destination federate's thread has already + // sent the starting MSG_TYPE_TIMESTAMP message. + while (_RTI.federates[federate_id].state == PENDING) { + // Need to wait here. + pthread_cond_wait(&_RTI.sent_start_time, &_RTI.rti_mutex); + } + + write_to_socket_errexit(destination_socket, bytes_read, buffer, + "RTI failed to forward message to federate %d.", federate_id); + + // The message length may be longer than the buffer, + // in which case we have to handle it in chunks. + size_t total_bytes_read = bytes_read; + while (total_bytes_read < total_bytes_to_read) { + LF_PRINT_DEBUG("Forwarding message in chunks."); + bytes_to_read = total_bytes_to_read - total_bytes_read; + if (bytes_to_read > FED_COM_BUFFER_SIZE) { + bytes_to_read = FED_COM_BUFFER_SIZE; + } + read_from_socket_errexit(sending_federate->socket, bytes_to_read, buffer, + "RTI failed to read message chunks."); + total_bytes_read += bytes_to_read; + + // FIXME: a mutex needs to be held for this so that other threads + // do not write to destination_socket and cause interleaving. However, + // holding the _RTI.rti_mutex might be very expensive. Instead, each outgoing + // socket should probably have its own mutex. + write_to_socket_errexit(destination_socket, bytes_to_read, buffer, + "RTI failed to send message chunks."); + } + + + + // If the destination federate has previously sent a NET that is larger + // than the intended tag of this message, then reset that NET to be equal + // to the intended tag of this message. This is needed because the NET + // is a promise that is valid only in the absence of network inputs, + // and now there is a network input. Hence, the promise needs to be + // updated. + if (lf_tag_compare(_RTI.federates[federate_id].next_event, intended_tag) > 0) { + _RTI.federates[federate_id].next_event = intended_tag; + + // Check to see whether we can reply now with a time advance grant. + // If the federate has no upstream federates, then it does not wait for + // nor expect a reply. It just proceeds to advance time. + if (_RTI.federates[federate_id].num_upstream > 0) { + send_advance_grant_if_safe(&_RTI.federates[federate_id]); + } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(&_RTI.federates[federate_id], visited); + } + + pthread_mutex_unlock(&_RTI.rti_mutex); +} + /** * Handle a logical tag complete (LTC) message. @see * MSG_TYPE_LOGICAL_TAG_COMPLETE in rti.h. @@ -790,12 +817,32 @@ void handle_next_event_tag(federate_t* fed) { // it might be more efficient to use a // select() mechanism to read and process // federates' buffers in an orderly fashion - - fed->next_event = extract_tag(buffer); - LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%lld, %u).", - fed->id, fed->next_event.time - start_time, - fed->next_event.microstep); + // If the fed->next_event is smaller than this NET message, we need to + // verify that the federate has already completed the previously promised + // fed->next_event. Otherwise, we ignore this NET. This scenario can happen + // when there is a message in-transit to the federate, and we have recorded + // the tag of that in-transit message as the federate's next event, but the + // message hasn't yet been delivered. + tag_t intended_tag = extract_tag(buffer); + if (lf_tag_compare(intended_tag, fed->next_event) <= 0 || + lf_tag_compare(fed->completed, fed->next_event) >= 0) { + fed->next_event = intended_tag; + + LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%ld, %u).", + fed->id, fed->next_event.time - start_time, + fed->next_event.microstep); + } else { + lf_print_error("RTI received from federate %d the Next Event Tag (NET) (%ld, %u), " + "but it is ignoring it since there is a message still in transit to federate %d. " + "Current recorded NET: (%ld, %u), LTC: (%ld, %u).", + fed->id, intended_tag.time - lf_time_start(), intended_tag.microstep, + fed->id, + fed->next_event.time - lf_time_start(), + fed->next_event.microstep, + fed->completed.time - lf_time_start(), + fed->completed.microstep); + } // Check to see whether we can reply now with a time advance grant. // If the federate has no upstream federates, then it does not wait for @@ -838,7 +885,14 @@ void handle_time_advance_notice(federate_t* fed) { // network inputs, the federate will not produce an output with tag // less than the NET. tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; - if (lf_tag_compare(ta, fed->next_event) > 0) { + // If the fed->next_event is smaller than this TAN message, we need to + // verify that the federate has already completed the previously promised + // fed->next_event. Otherwise, we ignore this TAN. This scenario can happen + // when there is a message in-transit to the federate, and we have recorded + // the tag of that in-transit message as the federate's next event, but the + // message hasn't yet been delivered. + if (lf_tag_compare(ta, fed->next_event) <= 0 || + lf_tag_compare(fed->completed, fed->next_event) >= 0) { fed->next_event = ta; // We need to reply just as if this were a NET because it could unblock // network input port control reactions. @@ -847,6 +901,16 @@ void handle_time_advance_notice(federate_t* fed) { if (fed->num_upstream > 0) { send_advance_grant_if_safe(fed); } + } else { + lf_print_error("RTI received from federate %d a Time Advance Notice (TAN) (%ld, %u), " + "but it is ignoring it since there is a message still in transit to federate %d. " + "Current recorded NET: (%ld, %u), LTC: (%ld, %u).", + fed->id, ta.time - lf_time_start(), ta.microstep, + fed->id, + fed->next_event.time - lf_time_start(), + fed->next_event.microstep, + fed->completed.time - lf_time_start(), + fed->completed.microstep); } // Check downstream federates to see whether they should now be granted a TAG. diff --git a/core/threaded/scheduler_sync_tag_advance.c b/core/threaded/scheduler_sync_tag_advance.c index e8a60fcf9..d7d116b17 100644 --- a/core/threaded/scheduler_sync_tag_advance.c +++ b/core/threaded/scheduler_sync_tag_advance.c @@ -88,10 +88,7 @@ bool _lf_sched_should_stop_locked() { * @return should_exit True if the worker thread should exit. False otherwise. */ bool _lf_sched_advance_tag_locked() { - // If this is not the very first step, notify that the previous step is complete. - if (_lf_logical_tag_completed) { - logical_tag_complete(current_tag); - } + logical_tag_complete(current_tag); if (_lf_sched_should_stop_locked()) { return true; From 24c26914656fcfb26407d5ced920d4d230b2afc6 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 09:30:20 -0500 Subject: [PATCH 15/37] Moved related functions closer together and adjusted the logic The new logic does two things:\n 1- The RTI double checks that if we are replacing the NET of a federate with a larger value, it has finished the previously (already) promised NET, and,\n 2- the RTI now attempts to send TAG and PTAGs if it is updating the next event of a federate upon forwarding a message --- core/federated/RTI/rti.c | 211 +++++++++++++++++++-------------------- 1 file changed, 105 insertions(+), 106 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 629ccd792..bfe625b2a 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -228,67 +228,6 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty return socket_descriptor; } -/** - * Handle a port absent message being received rom a federate via the RIT. - * - * This function assumes the caller does not hold the mutex. - */ -void handle_port_absent_message(federate_t* sending_federate, unsigned char* buffer) { - size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); - - read_from_socket_errexit(sending_federate->socket, message_size, &(buffer[1]), - " RTI failed to read port absent message from federate %u.", - sending_federate->id); - - // Need to acquire the mutex lock to ensure that the thread handling - // messages coming from the socket connected to the destination does not - // issue a TAG before this message has been forwarded. - pthread_mutex_lock(&_RTI.rti_mutex); - - uint16_t reactor_port_id = extract_uint16(&(buffer[1])); - uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); - - // If the destination federate is no longer connected, issue a warning - // and return. - if (_RTI.federates[federate_id].state == NOT_CONNECTED) { - pthread_mutex_unlock(&_RTI.rti_mutex); - lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", - federate_id); - LF_PRINT_LOG("Fed status: next_event (%lld, %d), " - "completed (%lld, %d), " - "last_granted (%lld, %d), " - "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].next_event.time - start_time, - _RTI.federates[federate_id].next_event.microstep, - _RTI.federates[federate_id].completed.time - start_time, - _RTI.federates[federate_id].completed.microstep, - _RTI.federates[federate_id].last_granted.time - start_time, - _RTI.federates[federate_id].last_granted.microstep, - _RTI.federates[federate_id].last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].last_provisionally_granted.microstep - ); - return; - } - - LF_PRINT_LOG("RTI forwarding port absent message for port %u to federate %u.", - reactor_port_id, - federate_id); - - // Need to make sure that the destination federate's thread has already - // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[federate_id].state == PENDING) { - // Need to wait here. - pthread_cond_wait(&_RTI.sent_start_time, &_RTI.rti_mutex); - } - - // Forward the message. - int destination_socket = _RTI.federates[federate_id].socket; - write_to_socket_errexit(destination_socket, message_size + 1, buffer, - "RTI failed to forward message to federate %d.", federate_id); - - pthread_mutex_unlock(&_RTI.rti_mutex); -} - /** * Send a tag advance grant (TAG) message to the specified federate. * Do not send it if a previously sent PTAG was greater or if a @@ -592,8 +531,16 @@ bool send_advance_grant_if_safe(federate_t* fed) { LF_PRINT_LOG("All upstream federates are finished. Sending TAG(FOREVER)."); send_tag_advance_grant(fed, FOREVER_TAG); return true; - } else if ( - lf_tag_compare(t_d, fed->next_event) >= 0 // The federate has something to do. + } else if (lf_tag_compare(t_d, fed->next_event) > 0) { + LF_PRINT_LOG("Earliest upstream message time for fed %d is (%ld, %u) " + "(adjusted by after delay). Granting tag advance for (%ld, %u).", + fed->id, + t_d.time - lf_time_start(), t_d.microstep, + fed->next_event.time - lf_time_start(), + fed->next_event.microstep); + send_tag_advance_grant(fed, fed->next_event); + } else if ( + lf_tag_compare(t_d, fed->next_event) >= 0 // The federate has something to do. && lf_tag_compare(t_d, fed->last_provisionally_granted) > 0 // The grant is not redundant. && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. ) { @@ -627,6 +574,93 @@ void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { } } +/** + * @brief Update the next event tag of federate `federate_id`. + * + * Will try to see if the RTI can grant new TAG or PTAG messages to any + * downstream federates based on this new next event tag. + * + * This function assumes that the caller is holding the _RTI.rti_mutex. + * + * @param federate_id The id of the federate that needs to be updated. + * @param next_event_tag The next event tag for `federate_id`. + */ +void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { + _RTI.federates[federate_id].next_event = next_event_tag; + + // Check to see whether we can reply now with a time advance grant. + // If the federate has no upstream federates, then it does not wait for + // nor expect a reply. It just proceeds to advance time. + if (_RTI.federates[federate_id].num_upstream > 0) { + send_advance_grant_if_safe(&_RTI.federates[federate_id]); + } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + send_downstream_advance_grants_if_safe(&_RTI.federates[federate_id], visited); +} + +/** + * Handle a port absent message being received rom a federate via the RIT. + * + * This function assumes the caller does not hold the mutex. + */ +void handle_port_absent_message(federate_t* sending_federate, unsigned char* buffer) { + size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); + + read_from_socket_errexit(sending_federate->socket, message_size, &(buffer[1]), + " RTI failed to read port absent message from federate %u.", + sending_federate->id); + + // Need to acquire the mutex lock to ensure that the thread handling + // messages coming from the socket connected to the destination does not + // issue a TAG before this message has been forwarded. + pthread_mutex_lock(&_RTI.rti_mutex); + + uint16_t reactor_port_id = extract_uint16(&(buffer[1])); + uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); + + // If the destination federate is no longer connected, issue a warning + // and return. + if (_RTI.federates[federate_id].state == NOT_CONNECTED) { + pthread_mutex_unlock(&_RTI.rti_mutex); + lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", + federate_id); + LF_PRINT_LOG("Fed status: next_event (%lld, %d), " + "completed (%lld, %d), " + "last_granted (%lld, %d), " + "last_provisionally_granted (%lld, %d).", + _RTI.federates[federate_id].next_event.time - start_time, + _RTI.federates[federate_id].next_event.microstep, + _RTI.federates[federate_id].completed.time - start_time, + _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].last_granted.time - start_time, + _RTI.federates[federate_id].last_granted.microstep, + _RTI.federates[federate_id].last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].last_provisionally_granted.microstep + ); + return; + } + + LF_PRINT_LOG("RTI forwarding port absent message for port %u to federate %u.", + reactor_port_id, + federate_id); + + // Need to make sure that the destination federate's thread has already + // sent the starting MSG_TYPE_TIMESTAMP message. + while (_RTI.federates[federate_id].state == PENDING) { + // Need to wait here. + pthread_cond_wait(&_RTI.sent_start_time, &_RTI.rti_mutex); + } + + // Forward the message. + int destination_socket = _RTI.federates[federate_id].socket; + write_to_socket_errexit(destination_socket, message_size + 1, buffer, + "RTI failed to forward message to federate %d.", federate_id); + + pthread_mutex_unlock(&_RTI.rti_mutex); +} /** * Handle a timed message being received from a federate by the RTI to relay to another federate. @@ -739,8 +773,6 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { "RTI failed to send message chunks."); } - - // If the destination federate has previously sent a NET that is larger // than the intended tag of this message, then reset that NET to be equal // to the intended tag of this message. This is needed because the NET @@ -748,21 +780,9 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // and now there is a network input. Hence, the promise needs to be // updated. if (lf_tag_compare(_RTI.federates[federate_id].next_event, intended_tag) > 0) { - _RTI.federates[federate_id].next_event = intended_tag; - - // Check to see whether we can reply now with a time advance grant. - // If the federate has no upstream federates, then it does not wait for - // nor expect a reply. It just proceeds to advance time. - if (_RTI.federates[federate_id].num_upstream > 0) { - send_advance_grant_if_safe(&_RTI.federates[federate_id]); - } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(&_RTI.federates[federate_id], visited); + update_federate_next_event_tag_locked(federate_id, intended_tag); } - + pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -827,11 +847,12 @@ void handle_next_event_tag(federate_t* fed) { tag_t intended_tag = extract_tag(buffer); if (lf_tag_compare(intended_tag, fed->next_event) <= 0 || lf_tag_compare(fed->completed, fed->next_event) >= 0) { - fed->next_event = intended_tag; - LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%ld, %u).", - fed->id, fed->next_event.time - start_time, - fed->next_event.microstep); + LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%ld, %u).", + fed->id, fed->next_event.time - start_time, + fed->next_event.microstep); + + update_federate_next_event_tag_locked(fed->id, intended_tag); } else { lf_print_error("RTI received from federate %d the Next Event Tag (NET) (%ld, %u), " "but it is ignoring it since there is a message still in transit to federate %d. " @@ -843,19 +864,6 @@ void handle_next_event_tag(federate_t* fed) { fed->completed.time - lf_time_start(), fed->completed.microstep); } - - // Check to see whether we can reply now with a time advance grant. - // If the federate has no upstream federates, then it does not wait for - // nor expect a reply. It just proceeds to advance time. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); - } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); - pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -893,14 +901,11 @@ void handle_time_advance_notice(federate_t* fed) { // message hasn't yet been delivered. if (lf_tag_compare(ta, fed->next_event) <= 0 || lf_tag_compare(fed->completed, fed->next_event) >= 0) { - fed->next_event = ta; // We need to reply just as if this were a NET because it could unblock // network input port control reactions. // This is a side-effect of the combination of distributed cycles and // physical actions in federates. FIXME: More explanation is needed. - if (fed->num_upstream > 0) { - send_advance_grant_if_safe(fed); - } + update_federate_next_event_tag_locked(fed->id, ta); } else { lf_print_error("RTI received from federate %d a Time Advance Notice (TAN) (%ld, %u), " "but it is ignoring it since there is a message still in transit to federate %d. " @@ -913,12 +918,6 @@ void handle_time_advance_notice(federate_t* fed) { fed->completed.microstep); } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(fed, visited); - pthread_mutex_unlock(&_RTI.rti_mutex); } From 76b1980646719f58d816330547d93af6ff6a5187 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 09:31:30 -0500 Subject: [PATCH 16/37] Updated LF ref --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 88ab221f4..fd15d4588 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -b50058ca475f1cd24b064ae2c9a278c9a958df51 +57b113ea286a5b7684615fd84e5dc939a3064b52 From 74d588ed962afb39aea1e49b310f119e52dbd342 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 09:34:33 -0500 Subject: [PATCH 17/37] Tabs only --- core/federated/RTI/rti.c | 44 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index bfe625b2a..4a24a2a20 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -521,35 +521,35 @@ bool send_advance_grant_if_safe(federate_t* fed) { t_d.time - start_time, t_d.microstep); if (lf_tag_compare(t_d, FOREVER_TAG) == 0) { - // Upstream federates are all done. + // Upstream federates are all done. LF_PRINT_LOG("Upstream federates are all done. Granting tag advance."); - send_tag_advance_grant(fed, FOREVER_TAG); - return true; + send_tag_advance_grant(fed, FOREVER_TAG); + return true; } - if (t_d.time == FOREVER) { - LF_PRINT_LOG("All upstream federates are finished. Sending TAG(FOREVER)."); - send_tag_advance_grant(fed, FOREVER_TAG); - return true; - } else if (lf_tag_compare(t_d, fed->next_event) > 0) { - LF_PRINT_LOG("Earliest upstream message time for fed %d is (%ld, %u) " - "(adjusted by after delay). Granting tag advance for (%ld, %u).", - fed->id, - t_d.time - lf_time_start(), t_d.microstep, + if (t_d.time == FOREVER) { + LF_PRINT_LOG("All upstream federates are finished. Sending TAG(FOREVER)."); + send_tag_advance_grant(fed, FOREVER_TAG); + return true; + } else if (lf_tag_compare(t_d, fed->next_event) > 0) { + LF_PRINT_LOG("Earliest upstream message time for fed %d is (%ld, %u) " + "(adjusted by after delay). Granting tag advance for (%ld, %u).", + fed->id, + t_d.time - lf_time_start(), t_d.microstep, fed->next_event.time - lf_time_start(), fed->next_event.microstep); - send_tag_advance_grant(fed, fed->next_event); + send_tag_advance_grant(fed, fed->next_event); } else if ( lf_tag_compare(t_d, fed->next_event) >= 0 // The federate has something to do. - && lf_tag_compare(t_d, fed->last_provisionally_granted) > 0 // The grant is not redundant. - && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. - ) { - LF_PRINT_LOG("Earliest upstream message time for fed %d is (%lld, %u) " - "(adjusted by after delay). Granting provisional tag advance.", - fed->id, - t_d.time - start_time, t_d.microstep); - - send_provisional_tag_advance_grant(fed, t_d); + && lf_tag_compare(t_d, fed->last_provisionally_granted) > 0 // The grant is not redundant. + && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. + ) { + LF_PRINT_LOG("Earliest upstream message time for fed %d is (%lld, %u) " + "(adjusted by after delay). Granting provisional tag advance.", + fed->id, + t_d.time - start_time, t_d.microstep); + + send_provisional_tag_advance_grant(fed, t_d); } return false; } From 48f85bf75da4cfc835ac6f9ae2a2eb478fadc313 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 09:36:39 -0500 Subject: [PATCH 18/37] Adjusted checks for TAG and PTAG --- core/federated/RTI/rti.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 4a24a2a20..5938a1b38 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -531,7 +531,10 @@ bool send_advance_grant_if_safe(federate_t* fed) { LF_PRINT_LOG("All upstream federates are finished. Sending TAG(FOREVER)."); send_tag_advance_grant(fed, FOREVER_TAG); return true; - } else if (lf_tag_compare(t_d, fed->next_event) > 0) { + } else if (lf_tag_compare(t_d, fed->next_event) > 0 // The federate has something to do. + && lf_tag_compare(t_d, fed->last_provisionally_granted) >= 0 // The grant is not redundant. + && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. + ) { LF_PRINT_LOG("Earliest upstream message time for fed %d is (%ld, %u) " "(adjusted by after delay). Granting tag advance for (%ld, %u).", fed->id, @@ -540,7 +543,7 @@ bool send_advance_grant_if_safe(federate_t* fed) { fed->next_event.microstep); send_tag_advance_grant(fed, fed->next_event); } else if ( - lf_tag_compare(t_d, fed->next_event) >= 0 // The federate has something to do. + lf_tag_compare(t_d, fed->next_event) == 0 // The federate has something to do. && lf_tag_compare(t_d, fed->last_provisionally_granted) > 0 // The grant is not redundant. && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. ) { From b0aace2e7f428f3e86182a5129847e23aec5ca9f Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 09:40:44 -0500 Subject: [PATCH 19/37] Comments --- core/federated/RTI/rti.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 5938a1b38..cae522af1 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -531,10 +531,14 @@ bool send_advance_grant_if_safe(federate_t* fed) { LF_PRINT_LOG("All upstream federates are finished. Sending TAG(FOREVER)."); send_tag_advance_grant(fed, FOREVER_TAG); return true; - } else if (lf_tag_compare(t_d, fed->next_event) > 0 // The federate has something to do. - && lf_tag_compare(t_d, fed->last_provisionally_granted) >= 0 // The grant is not redundant. + } else if ( + lf_tag_compare(t_d, fed->next_event) > 0 // The federate has something to do. + && lf_tag_compare(t_d, fed->last_provisionally_granted) >= 0 // The grant is not redundant + // (equal is important to override any previous + // PTAGs). && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. ) { + // All upstream federates have events with a larger tag than fed, so it is safe to send a TAG. LF_PRINT_LOG("Earliest upstream message time for fed %d is (%ld, %u) " "(adjusted by after delay). Granting tag advance for (%ld, %u).", fed->id, @@ -547,6 +551,8 @@ bool send_advance_grant_if_safe(federate_t* fed) { && lf_tag_compare(t_d, fed->last_provisionally_granted) > 0 // The grant is not redundant. && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. ) { + // Some upstream federate has an event that has the same tag as fed's next event, so we can only provisionally + // grant a TAG (via a PTAG). LF_PRINT_LOG("Earliest upstream message time for fed %d is (%lld, %u) " "(adjusted by after delay). Granting provisional tag advance.", fed->id, From bcd65af563e6bb15b8dd50b780b5f061b40dc2c5 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 13:52:34 -0500 Subject: [PATCH 20/37] Intsert dummy events for TAN messages --- core/federated/federate.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/federated/federate.c b/core/federated/federate.c index 42ea399a7..fbc03dc64 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2643,6 +2643,9 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { if (_lf_bounded_NET(&next_tag)) { _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, next_tag.time, wait_for_reply); _fed.last_sent_NET = next_tag; + event_t* dummy = _lf_create_dummy_events( + NULL, tag.time, NULL, 0); + pqueue_insert(event_q, dummy); LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", next_tag.time - lf_time_start()); } else { @@ -2661,6 +2664,9 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // It is sent to enable downstream federates to advance. _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); _fed.last_sent_NET = tag; + event_t* dummy = _lf_create_dummy_events( + NULL, tag.time, NULL, 0); + pqueue_insert(event_q, dummy); LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", tag.time - lf_time_start()); From 5aaafb00d4bb270b3e26babda5985f53fa1cd36f Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 13:57:10 -0500 Subject: [PATCH 21/37] Comments, FIXME, and a small fix --- core/federated/federate.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index fbc03dc64..86c5bff1a 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2643,8 +2643,9 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { if (_lf_bounded_NET(&next_tag)) { _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, next_tag.time, wait_for_reply); _fed.last_sent_NET = next_tag; - event_t* dummy = _lf_create_dummy_events( - NULL, tag.time, NULL, 0); + // Create a dummy event (FIXME: This should be sufficient. No need to also send a TAN because + // the dummy event will result in a NET). + event_t* dummy = _lf_create_dummy_events(NULL, next_tag.time, NULL, 0); pqueue_insert(event_q, dummy); LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", next_tag.time - lf_time_start()); @@ -2664,8 +2665,9 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // It is sent to enable downstream federates to advance. _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); _fed.last_sent_NET = tag; - event_t* dummy = _lf_create_dummy_events( - NULL, tag.time, NULL, 0); + // Create a dummy event (FIXME: This should be sufficient. No need to also send a TAN because + // the dummy event will result in a NET). + event_t* dummy = _lf_create_dummy_events(NULL, tag.time, NULL, 0); pqueue_insert(event_q, dummy); LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", tag.time - lf_time_start()); From 44dc5a1c341733575412f06628f4941da590795b Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 23:08:39 -0500 Subject: [PATCH 22/37] Purge TAN. Instead, create dummy events if needed --- core/federated/RTI/rti.c | 66 +----------------------------- core/federated/RTI/rti.h | 1 - core/federated/federate.c | 81 ++++++++++++++----------------------- core/federated/net_common.h | 36 ++++++----------- 4 files changed, 45 insertions(+), 139 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index cae522af1..b7e078abb 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -280,7 +280,6 @@ void send_tag_advance_grant(federate_t* fed, tag_t tag) { /** * Find the earliest tag at which the specified federate may * experience its next event. This is the least next event tag (NET) - * or time advance notice (TAN) * of the specified federate and (transitively) upstream federates * (with delays of the connections added). For upstream federates, * we assume (conservatively) that federate upstream of those @@ -308,7 +307,6 @@ tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]) { } visited[fed->id] = true; - // Note that NET already takes into account any TAN messages received. tag_t result = fed->next_event; // If the candidate is less than this federate's next_event, use the candidate. @@ -421,7 +419,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { /** * Determine whether the specified federate fed is eligible for a tag advance grant, - * (TAG) and, if so, send it one. This is called upon receiving a LTC, TAN, NET + * (TAG) and, if so, send it one. This is called upon receiving a LTC, NET * or resign from an upstream federate. * * This function calculates the minimum M over @@ -441,7 +439,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { * * This should be called whenever an immediately upstream federate sends to * the RTI an LTC (Logical Tag Complete), or when a transitive upstream - * federate sends a TAN (Time Advance Notice) or a NET (Next Event Tag) message. + * federate sends a NET (Next Event Tag) message. * It is also called when an upstream federate resigns from the federation. * * This function assumes that the caller holds the mutex lock. @@ -876,60 +874,6 @@ void handle_next_event_tag(federate_t* fed) { pthread_mutex_unlock(&_RTI.rti_mutex); } -/** - * Handle a time advance notice (TAN) message. @see MSG_TYPE_TIME_ADVANCE_NOTICE - * in rti.h. - * - * This function assumes the caller does not hold the mutex. - * - * @param fed The federate sending a TAN message. - */ -void handle_time_advance_notice(federate_t* fed) { - unsigned char buffer[sizeof(int64_t)]; - read_from_socket_errexit(fed->socket, sizeof(int64_t), buffer, - "RTI failed to read the content of the time advance notice from federate %d.", fed->id); - - // Acquire a mutex lock to ensure that this state does change while a - // message is in transport or being used to determine a TAG. - pthread_mutex_lock(&_RTI.rti_mutex); - - fed->time_advance = extract_int64(buffer); - LF_PRINT_LOG("RTI received from federate %d the Time Advance Notice (TAN) %lld.", - fed->id, fed->time_advance - start_time); - - // If the TAN is greater than the most recently received NET, then - // update the NET to match the TAN. The NET is a promise that, absent - // network inputs, the federate will not produce an output with tag - // less than the NET. - tag_t ta = (tag_t) {.time = fed->time_advance, .microstep = 0}; - // If the fed->next_event is smaller than this TAN message, we need to - // verify that the federate has already completed the previously promised - // fed->next_event. Otherwise, we ignore this TAN. This scenario can happen - // when there is a message in-transit to the federate, and we have recorded - // the tag of that in-transit message as the federate's next event, but the - // message hasn't yet been delivered. - if (lf_tag_compare(ta, fed->next_event) <= 0 || - lf_tag_compare(fed->completed, fed->next_event) >= 0) { - // We need to reply just as if this were a NET because it could unblock - // network input port control reactions. - // This is a side-effect of the combination of distributed cycles and - // physical actions in federates. FIXME: More explanation is needed. - update_federate_next_event_tag_locked(fed->id, ta); - } else { - lf_print_error("RTI received from federate %d a Time Advance Notice (TAN) (%ld, %u), " - "but it is ignoring it since there is a message still in transit to federate %d. " - "Current recorded NET: (%ld, %u), LTC: (%ld, %u).", - fed->id, ta.time - lf_time_start(), ta.microstep, - fed->id, - fed->next_event.time - lf_time_start(), - fed->next_event.microstep, - fed->completed.time - lf_time_start(), - fed->completed.microstep); - } - - pthread_mutex_unlock(&_RTI.rti_mutex); -} - /////////////////// STOP functions //////////////////// /** * Boolean used to prevent the RTI from sending the @@ -1459,8 +1403,6 @@ void handle_federate_resign(federate_t *my_fed) { // Indicate that there will no further events from this federate. my_fed->next_event = FOREVER_TAG; - - my_fed->time_advance = FOREVER; // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket, // the close should happen when receiving a 0 length message from the other end. @@ -1529,9 +1471,6 @@ void* federate_thread_TCP(void* fed) { case MSG_TYPE_NEXT_EVENT_TAG: handle_next_event_tag(my_fed); break; - case MSG_TYPE_TIME_ADVANCE_NOTICE: - handle_time_advance_notice(my_fed); - break; case MSG_TYPE_LOGICAL_TAG_COMPLETE: handle_logical_tag_complete(my_fed); break; @@ -1945,7 +1884,6 @@ void initialize_federate(uint16_t id) { _RTI.federates[id].last_granted = NEVER_TAG; _RTI.federates[id].last_provisionally_granted = NEVER_TAG; _RTI.federates[id].next_event = NEVER_TAG; - _RTI.federates[id].time_advance = NEVER; _RTI.federates[id].state = NOT_CONNECTED; _RTI.federates[id].upstream = NULL; _RTI.federates[id].upstream_delay = NULL; diff --git a/core/federated/RTI/rti.h b/core/federated/RTI/rti.h index 6b2f930ae..54a4f26ba 100644 --- a/core/federated/RTI/rti.h +++ b/core/federated/RTI/rti.h @@ -77,7 +77,6 @@ typedef struct federate_t { tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). bool in_transit_message;// Indicator that next_event was set by the RTI because of a message destined to the federate. - instant_t time_advance; // Most recent TAN received from the federate (or NEVER if none received). fed_state_t state; // State of the federate. int* upstream; // Array of upstream federate ids. interval_t* upstream_delay; // Minimum delay on connections from upstream federates. diff --git a/core/federated/federate.c b/core/federated/federate.c index 86c5bff1a..2e93a8ff4 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2500,14 +2500,16 @@ bool _lf_bounded_NET(tag_t* tag) { /** * If this federate depends on upstream federates or sends data to downstream - * federates, then send to the RTI either a NET or a TAN, depending on whether - * there are network outputs that depend on physical actions. If there are no - * such outputs, then send next event tag (NET), which will give the tag of - * the earliest event on the event queue, or, if the queue is empty, the timeout + * federates, then send to the RTI a NET, which will give the tag of the + * earliest event on the event queue, or, if the queue is empty, the timeout * time, or, if there is no timeout, FOREVER. + * + * If there are network outputs that + * depend on physical actions, then insert a dummy event to ensure this federate + * advances its tag so that downstream federates can make progress. * - * A NET or TAN is a promise saying that, absent network inputs, this federate will - * not produce an output message with tag earlier than the NET value or (TAN,0). + * A NET is a promise saying that, absent network inputs, this federate will + * not produce an output message with tag earlier than the NET value. * * If there are upstream federates, then after sending a NET, this will block * until either the RTI grants the advance to the requested time or the wait @@ -2519,17 +2521,17 @@ bool _lf_bounded_NET(tag_t* tag) { * If the federate has neither upstream nor downstream federates, then this * returns the specified tag immediately without sending anything to the RTI. * - * If there is at least one physical action somewhere in the federate that - * can trigger an output to a downstream federate, then the NET is required - * to be less than the current physical time. If physical time is less than - * the earliest event in the event queue (or the event queue is empty), then - * this function will send a Time Advance Notice (TAN) message instead of NET. - * That message does not require a response from the RTI. The TAN message will - * be sent repeatedly as physical time advances with the time interval between - * messages controlled by the target parameter - * coordination-options: {advance-message-interval timevalue}. - * It will switch back to sending a NET message if and when its event queue - * has an event with a timestamp less than physical time. + * If there is at least one physical action somewhere in the federate that can + * trigger an output to a downstream federate, then the NET is required to be + * less than the current physical time. If physical time is less than the + * earliest event in the event queue (or the event queue is empty), then this + * function will insert a dummy event with a tag equal to the current physical + * time (and a microstep of 0). This will enforce advancement of tag for this + * federate and causes a NET message to be sent repeatedly as physical time + * advances with the time interval between messages controlled by the target + * parameter coordination-options: {advance-message-interval timevalue}. It will + * stop creating dummy events if and when its event queue has an event with a + * timestamp less than physical time. * * If wait_for_reply is false, then this function will simply send the * specified tag and return that tag immediately. This is useful when a @@ -2639,37 +2641,20 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // Check whether the new event on the event queue requires sending a new NET. tag_t next_tag = get_next_event_tag(); if (lf_tag_compare(next_tag, tag) != 0) { - // The next tag may also have to be bounded by physical time. - if (_lf_bounded_NET(&next_tag)) { - _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, next_tag.time, wait_for_reply); - _fed.last_sent_NET = next_tag; - // Create a dummy event (FIXME: This should be sufficient. No need to also send a TAN because - // the dummy event will result in a NET). - event_t* dummy = _lf_create_dummy_events(NULL, next_tag.time, NULL, 0); - pqueue_insert(event_q, dummy); - LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", - next_tag.time - lf_time_start()); - } else { - _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); - _fed.last_sent_NET = next_tag; - LF_PRINT_LOG("Sent next event tag (NET) (%lld, %u) to RTI.", - next_tag.time - start_time, next_tag.microstep); - } + _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); + _fed.last_sent_NET = next_tag; + LF_PRINT_LOG("Sent next event tag (NET) (%lld, %u) to RTI.", + next_tag.time - start_time, next_tag.microstep); } } } - // Next tag is greater than physical time and this fed has downstream - // federates. Need to send TAN rather than NET. - // TAN does not include a microstep and expects no reply. - // It is sent to enable downstream federates to advance. - _lf_send_time(MSG_TYPE_TIME_ADVANCE_NOTICE, tag.time, wait_for_reply); - _fed.last_sent_NET = tag; - // Create a dummy event (FIXME: This should be sufficient. No need to also send a TAN because - // the dummy event will result in a NET). + // Create a dummy event that will force this federate to advance time and subsequently enable progress for + // downstream federates. event_t* dummy = _lf_create_dummy_events(NULL, tag.time, NULL, 0); pqueue_insert(event_q, dummy); - LF_PRINT_DEBUG("Sent Time Advance Notice (TAN) %lld to RTI.", + + LF_PRINT_DEBUG("Inserted a dummy event for tag (%ld, %u).", tag.time - lf_time_start()); if (!wait_for_reply) { @@ -2677,13 +2662,9 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { return tag; } - // This federate should repeatedly send TAN messages - // to the RTI so that downstream federates can advance time until - // it has a candidate event that it can process. - // Before sending the next message, we need to wait some time so - // that we don't overwhelm the network and the RTI. - // That amount of time will be no greater than ADVANCE_MESSAGE_INTERVAL - // in the future. + // This federate should repeatedly advance its tag to ensure downstream federates can make progress. + // Before advancing to the next tag, we need to wait some time so that we don't overwhelm the network and the + // RTI. That amount of time will be no greater than ADVANCE_MESSAGE_INTERVAL in the future. LF_PRINT_DEBUG("Waiting for physical time to elapse or an event on the event queue."); // The above call to _lf_bounded_NET called lf_time_physical() @@ -2707,7 +2688,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // Either the timeout expired or the wait was interrupted by an event being // put onto the event queue. In either case, we can just loop around. // The next iteration will determine whether another - // TAN should be sent or a NET. + // NET should be sent or not. tag = get_next_event_tag(); } } diff --git a/core/federated/net_common.h b/core/federated/net_common.h index f553efe53..0ba445115 100644 --- a/core/federated/net_common.h +++ b/core/federated/net_common.h @@ -368,33 +368,21 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define MSG_TYPE_TAGGED_MESSAGE 5 /** - * Byte identifying a next event tag (NET) message sent from a federate - * in centralized coordination. - * The next eight bytes will be the timestamp. - * The next four bytes will be the microstep. - * This message from a federate tells the RTI the tag of the earliest event - * on that federate's event queue. In other words, absent any further inputs - * from other federates, this will be the least tag of the next set of - * reactions on that federate. If the event queue is empty and a timeout - * time has been specified, then the timeout time will be sent. If there is - * no timeout time, then FOREVER will be sent. Note that this message should - * not be sent if there are physical actions and the earliest event on the event - * queue has a tag that is ahead of physical time (or the queue is empty). - * In that case, send TAN instead. + * Byte identifying a next event tag (NET) message sent from a federate in + * centralized coordination. The next eight bytes will be the timestamp. The + * next four bytes will be the microstep. This message from a federate tells the + * RTI the tag of the earliest event on that federate's event queue. In other + * words, absent any further inputs from other federates, this will be the least + * tag of the next set of reactions on that federate. If the event queue is + * empty and a timeout time has been specified, then the timeout time will be + * sent. If there is no timeout time, then FOREVER will be sent. Note that if + * there are physical actions and the earliest event on the event queue has a + * tag that is ahead of physical time (or the queue is empty), the federate + * should try to regularly advance its tag (and thus send NET messages) to make + * sure downstream federates can make progress. */ #define MSG_TYPE_NEXT_EVENT_TAG 6 -/** - * Byte identifying a time advance notice (TAN) message sent from - * a federate in centralized coordination. This message is used by - * a federate that has outputs that are directly or indirectly - * triggered by a physical action to notify the RTI that its physical - * time has advanced and that it will produce no outputs with - * timestamps less than the specified timestamp. - * The next eight bytes will be the timestamp. - */ -#define MSG_TYPE_TIME_ADVANCE_NOTICE 253 - /** * Byte identifying a time advance grant (TAG) sent by the RTI to a federate * in centralized coordination. This message is a promise by the RTI to the federate From 4082de990ca31f0ee8a47708a7708079af74b7cd Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 23:11:43 -0500 Subject: [PATCH 23/37] Updated LF ref --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index fd15d4588..4e92adc37 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -57b113ea286a5b7684615fd84e5dc939a3064b52 +9e12bc5a2dea2dac988aeb7a0fce3ec4e07c5e81 From 9801a4f9f10962b6317428a5a319f983de182a57 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Mon, 30 May 2022 23:14:40 -0500 Subject: [PATCH 24/37] Converted tabs to spaces --- core/federated/RTI/rti.c | 160 ++++++++-------- core/federated/federate.c | 260 +++++++++++++------------- core/federated/federate.h | 376 +++++++++++++++++++------------------- 3 files changed, 398 insertions(+), 398 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index b7e078abb..f75ecdddc 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -243,8 +243,8 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty */ void send_tag_advance_grant(federate_t* fed, tag_t tag) { if (fed->state == NOT_CONNECTED - || lf_tag_compare(tag, fed->last_granted) <= 0 - || lf_tag_compare(tag, fed->last_provisionally_granted) < 0 + || lf_tag_compare(tag, fed->last_granted) <= 0 + || lf_tag_compare(tag, fed->last_provisionally_granted) < 0 ) { return; } @@ -354,8 +354,8 @@ tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]) { */ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { if (fed->state == NOT_CONNECTED - || lf_tag_compare(tag, fed->last_granted) <= 0 - || lf_tag_compare(tag, fed->last_provisionally_granted) <= 0 + || lf_tag_compare(tag, fed->last_granted) <= 0 + || lf_tag_compare(tag, fed->last_provisionally_granted) <= 0 ) { return; } @@ -411,7 +411,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { // in which case, another will not be sent. But it // may not have been already granted. if (lf_tag_compare(upstream_next_event, tag) >= 0) { - send_provisional_tag_advance_grant(upstream, tag); + send_provisional_tag_advance_grant(upstream, tag); } } } @@ -448,7 +448,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { */ bool send_advance_grant_if_safe(federate_t* fed) { - // Find the earliest LTC of upstream federates. + // Find the earliest LTC of upstream federates. tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < fed->num_upstream; j++) { @@ -460,7 +460,7 @@ bool send_advance_grant_if_safe(federate_t* fed) { tag_t candidate = _lf_delay_tag(upstream->completed, fed->upstream_delay[j]); if (lf_tag_compare(candidate, min_upstream_completed) < 0) { - min_upstream_completed = candidate; + min_upstream_completed = candidate; } } LF_PRINT_LOG("Minimum upstream LTC for fed %d is (%lld, %u) " @@ -468,8 +468,8 @@ bool send_advance_grant_if_safe(federate_t* fed) { fed->id, min_upstream_completed.time - start_time, min_upstream_completed.microstep); if (lf_tag_compare(min_upstream_completed, fed->last_granted) > 0) { - send_tag_advance_grant(fed, min_upstream_completed); - return true; + send_tag_advance_grant(fed, min_upstream_completed); + return true; } // Can't make progress based only on upstream LTCs. @@ -558,7 +558,7 @@ bool send_advance_grant_if_safe(federate_t* fed) { send_provisional_tag_advance_grant(fed, t_d); } - return false; + return false; } /** @@ -572,13 +572,13 @@ bool send_advance_grant_if_safe(federate_t* fed) { * been visited (initially all false). */ void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { - visited[fed->id] = true; - for (int i = 0; i < fed->num_downstream; i++) { - federate_t* downstream = &_RTI.federates[fed->downstream[i]]; - if (visited[downstream->id]) continue; - send_advance_grant_if_safe(downstream); - send_downstream_advance_grants_if_safe(downstream, visited); - } + visited[fed->id] = true; + for (int i = 0; i < fed->num_downstream; i++) { + federate_t* downstream = &_RTI.federates[fed->downstream[i]]; + if (visited[downstream->id]) continue; + send_advance_grant_if_safe(downstream); + send_downstream_advance_grants_if_safe(downstream, visited); + } } /** @@ -635,18 +635,18 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); LF_PRINT_LOG("Fed status: next_event (%lld, %d), " - "completed (%lld, %d), " - "last_granted (%lld, %d), " - "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].next_event.time - start_time, - _RTI.federates[federate_id].next_event.microstep, - _RTI.federates[federate_id].completed.time - start_time, - _RTI.federates[federate_id].completed.microstep, - _RTI.federates[federate_id].last_granted.time - start_time, - _RTI.federates[federate_id].last_granted.microstep, - _RTI.federates[federate_id].last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].last_provisionally_granted.microstep - ); + "completed (%lld, %d), " + "last_granted (%lld, %d), " + "last_provisionally_granted (%lld, %d).", + _RTI.federates[federate_id].next_event.time - start_time, + _RTI.federates[federate_id].next_event.microstep, + _RTI.federates[federate_id].completed.time - start_time, + _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].last_granted.time - start_time, + _RTI.federates[federate_id].last_granted.microstep, + _RTI.federates[federate_id].last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].last_provisionally_granted.microstep + ); return; } @@ -724,18 +724,18 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); LF_PRINT_LOG("Fed status: next_event (%lld, %d), " - "completed (%lld, %d), " - "last_granted (%lld, %d), " - "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].next_event.time - start_time, - _RTI.federates[federate_id].next_event.microstep, - _RTI.federates[federate_id].completed.time - start_time, - _RTI.federates[federate_id].completed.microstep, - _RTI.federates[federate_id].last_granted.time - start_time, - _RTI.federates[federate_id].last_granted.microstep, - _RTI.federates[federate_id].last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].last_provisionally_granted.microstep - ); + "completed (%lld, %d), " + "last_granted (%lld, %d), " + "last_provisionally_granted (%lld, %d).", + _RTI.federates[federate_id].next_event.time - start_time, + _RTI.federates[federate_id].next_event.microstep, + _RTI.federates[federate_id].completed.time - start_time, + _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].last_granted.time - start_time, + _RTI.federates[federate_id].last_granted.microstep, + _RTI.federates[federate_id].last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].last_provisionally_granted.microstep + ); return; } @@ -903,9 +903,9 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { continue; } if (lf_tag_compare(_RTI.federates[i].next_event, _RTI.max_stop_tag) >= 0) { - // Need the next_event to be no greater than the stop tag. - _RTI.federates[i].next_event = _RTI.max_stop_tag; - _RTI.federates[i].in_transit_message = false; + // Need the next_event to be no greater than the stop tag. + _RTI.federates[i].next_event = _RTI.max_stop_tag; + _RTI.federates[i].in_transit_message = false; } write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", _RTI.federates[i].id); @@ -955,7 +955,7 @@ void handle_stop_request_message(federate_t* fed) { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; unsigned char buffer[bytes_to_read]; read_from_socket_errexit(fed->socket, bytes_to_read, buffer, - "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", fed->id); + "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", fed->id); // Acquire a mutex lock to ensure that this state does change while a // message is in transport or being used to determine a TAG. @@ -1025,7 +1025,7 @@ void handle_stop_request_reply(federate_t* fed) { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_REPLY_LENGTH - 1; unsigned char buffer_stop_time[bytes_to_read]; read_from_socket_errexit(fed->socket, bytes_to_read, buffer_stop_time, - "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", fed->id); + "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", fed->id); tag_t federate_stop_tag = extract_tag(buffer_stop_time); @@ -1537,16 +1537,16 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // First byte received is the message type. if (buffer[0] != MSG_TYPE_FED_IDS) { if(buffer[0] == MSG_TYPE_P2P_SENDING_FED_ID || buffer[0] == MSG_TYPE_P2P_TAGGED_MESSAGE) { - // The federate is trying to connect to a peer, not to the RTI. - // It has connected to the RTI instead. - // FIXME: This should not happen, but apparently has been observed. - // It should not happen because the peers get the port and IP address - // of the peer they want to connect to from the RTI. + // The federate is trying to connect to a peer, not to the RTI. + // It has connected to the RTI instead. + // FIXME: This should not happen, but apparently has been observed. + // It should not happen because the peers get the port and IP address + // of the peer they want to connect to from the RTI. // If the connection is a peer-to-peer connection between two // federates, reject the connection with the WRONG_SERVER error. - send_reject(socket_id, WRONG_SERVER); + send_reject(socket_id, WRONG_SERVER); } else { - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(socket_id, UNEXPECTED_MESSAGE); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); return -1; @@ -1593,38 +1593,38 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } } - // The MSG_TYPE_FED_IDS message has the right federation ID. - // Assign the address information for federate. - // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful - // to create sockets and can be efficiently sent over the network. - // First, convert the sockaddr structure into a sockaddr_in that contains an internet address. - struct sockaddr_in* pV4_addr = client_fd; - // Then extract the internet address (which is in IPv4 format) and assign it as the federate's socket server - _RTI.federates[fed_id].server_ip_addr = pV4_addr->sin_addr; + // The MSG_TYPE_FED_IDS message has the right federation ID. + // Assign the address information for federate. + // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful + // to create sockets and can be efficiently sent over the network. + // First, convert the sockaddr structure into a sockaddr_in that contains an internet address. + struct sockaddr_in* pV4_addr = client_fd; + // Then extract the internet address (which is in IPv4 format) and assign it as the federate's socket server + _RTI.federates[fed_id].server_ip_addr = pV4_addr->sin_addr; #if LOG_LEVEL >= LOG_LEVEL_DEBUG - // Create the human readable format and copy that into - // the .server_hostname field of the federate. - char str[INET_ADDRSTRLEN]; - inet_ntop( AF_INET, &_RTI.federates[fed_id].server_ip_addr, str, INET_ADDRSTRLEN ); - strncpy (_RTI.federates[fed_id].server_hostname, str, INET_ADDRSTRLEN); + // Create the human readable format and copy that into + // the .server_hostname field of the federate. + char str[INET_ADDRSTRLEN]; + inet_ntop( AF_INET, &_RTI.federates[fed_id].server_ip_addr, str, INET_ADDRSTRLEN ); + strncpy (_RTI.federates[fed_id].server_hostname, str, INET_ADDRSTRLEN); - LF_PRINT_DEBUG("RTI got address %s from federate %d.", _RTI.federates[fed_id].server_hostname, fed_id); + LF_PRINT_DEBUG("RTI got address %s from federate %d.", _RTI.federates[fed_id].server_hostname, fed_id); #endif - _RTI.federates[fed_id].socket = socket_id; + _RTI.federates[fed_id].socket = socket_id; - // Set the federate's state as pending - // because it is waiting for the start time to be - // sent by the RTI before beginning its execution. - _RTI.federates[fed_id].state = PENDING; + // Set the federate's state as pending + // because it is waiting for the start time to be + // sent by the RTI before beginning its execution. + _RTI.federates[fed_id].state = PENDING; - LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); - // Send an MSG_TYPE_ACK message. - unsigned char ack_message = MSG_TYPE_ACK; - write_to_socket_errexit(socket_id, 1, &ack_message, - "RTI failed to write MSG_TYPE_ACK message to federate %d.", fed_id); + LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); + // Send an MSG_TYPE_ACK message. + unsigned char ack_message = MSG_TYPE_ACK; + write_to_socket_errexit(socket_id, 1, &ack_message, + "RTI failed to write MSG_TYPE_ACK message to federate %d.", fed_id); - return (int32_t)fed_id; + return (int32_t)fed_id; } /** @@ -1808,7 +1808,7 @@ void connect_to_federates(int socket_descriptor) { int32_t fed_id = receive_and_check_fed_id_message(socket_id, (struct sockaddr_in*)&client_fd); if (fed_id >= 0 && receive_connection_information(socket_id, (uint16_t)fed_id) - && receive_udp_message_and_set_up_clock_sync(socket_id, (uint16_t)fed_id)) { + && receive_udp_message_and_set_up_clock_sync(socket_id, (uint16_t)fed_id)) { // Create a thread to communicate with the federate. // This has to be done after clock synchronization is finished @@ -1817,7 +1817,7 @@ void connect_to_federates(int socket_descriptor) { pthread_create(&(_RTI.federates[fed_id].thread_id), NULL, federate_thread_TCP, &(_RTI.federates[fed_id])); } else { - // Received message was rejected. Try again. + // Received message was rejected. Try again. i--; } } diff --git a/core/federated/federate.c b/core/federated/federate.c index 2e93a8ff4..2136a5436 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -64,7 +64,7 @@ lf_cond_t port_status_changed; federate_instance_t _fed = { .socket_TCP_RTI = -1, .number_of_inbound_p2p_connections = 0, - .inbound_socket_listeners = NULL, + .inbound_socket_listeners = NULL, .number_of_outbound_p2p_connections = 0, .sockets_for_inbound_p2p_connections = { -1 }, .sockets_for_outbound_p2p_connections = { -1 }, @@ -288,9 +288,9 @@ int send_message(int message_type, socket = _fed.socket_TCP_RTI; } if (socket < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); + lf_print_warning("Socket is no longer connected. Dropping message."); lf_mutex_unlock(&outbound_socket_mutex); - return 0; + return 0; } write_to_socket_errexit_with_mutex(socket, header_length, header_buffer, &outbound_socket_mutex, "Failed to send message header to to %s.", next_destination_str); @@ -404,9 +404,9 @@ int send_timed_message(interval_t additional_delay, socket = _fed.socket_TCP_RTI; } if (socket < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); + lf_print_warning("Socket is no longer connected. Dropping message."); lf_mutex_unlock(&outbound_socket_mutex); - return 0; + return 0; } write_to_socket_errexit_with_mutex(socket, header_length, header_buffer, &outbound_socket_mutex, "Failed to send timed message header to %s.", next_destination_str); @@ -433,9 +433,9 @@ void _lf_send_time(unsigned char type, instant_t time, bool exit_on_error) { encode_int64(time, &(buffer[1])); lf_mutex_lock(&outbound_socket_mutex); if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); + lf_print_warning("Socket is no longer connected. Dropping message."); lf_mutex_unlock(&outbound_socket_mutex); - return; + return; } ssize_t bytes_written = write_to_socket(_fed.socket_TCP_RTI, bytes_to_write, buffer); if (bytes_written < (ssize_t)bytes_to_write) { @@ -480,9 +480,9 @@ void _lf_send_tag(unsigned char type, tag_t tag, bool exit_on_error) { lf_mutex_lock(&outbound_socket_mutex); if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); + lf_print_warning("Socket is no longer connected. Dropping message."); lf_mutex_unlock(&outbound_socket_mutex); - return; + return; } ssize_t bytes_written = write_to_socket(_fed.socket_TCP_RTI, bytes_to_write, buffer); if (bytes_written < (ssize_t)bytes_to_write) { @@ -597,9 +597,9 @@ void* handle_p2p_connections_from_federates(void* ignored) { } *remote_fed_id_copy = remote_fed_id; int result = lf_thread_create( - &_fed.inbound_socket_listeners[received_federates], - listen_to_federates, - remote_fed_id_copy); + &_fed.inbound_socket_listeners[received_federates], + listen_to_federates, + remote_fed_id_copy); if (result != 0) { // Failed to create a listening thread. close(socket_id); @@ -625,8 +625,8 @@ void* handle_p2p_connections_from_federates(void* ignored) { * federate, or -1 if the RTI (centralized coordination). */ void _lf_close_outbound_socket(int fed_id) { - assert (fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES); - if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) { + assert (fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES); + if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) { shutdown(_fed.sockets_for_outbound_p2p_connections[fed_id], SHUT_RDWR); close(_fed.sockets_for_outbound_p2p_connections[fed_id]); _fed.sockets_for_outbound_p2p_connections[fed_id] = -1; @@ -648,22 +648,22 @@ void* listen_for_upstream_messages_from_downstream_federates(void* fed_id_ptr) { lf_mutex_lock(&outbound_socket_mutex); while(_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) { - // Unlock the mutex before performing a blocking read. - // Note that there is a race condition here, but the read will return - // a failure if the socket gets closed. + // Unlock the mutex before performing a blocking read. + // Note that there is a race condition here, but the read will return + // a failure if the socket gets closed. lf_mutex_unlock(&outbound_socket_mutex); LF_PRINT_DEBUG("Thread listening for MSG_TYPE_CLOSE_REQUEST from federate %d", fed_id); - ssize_t bytes_read = read_from_socket( - _fed.sockets_for_outbound_p2p_connections[fed_id], 1, &message); - // Reacquire the mutex lock before closing or reading the socket again. + ssize_t bytes_read = read_from_socket( + _fed.sockets_for_outbound_p2p_connections[fed_id], 1, &message); + // Reacquire the mutex lock before closing or reading the socket again. lf_mutex_lock(&outbound_socket_mutex); if (bytes_read == 1 && message == MSG_TYPE_CLOSE_REQUEST) { - // Received a request to close the socket. - LF_PRINT_DEBUG("Received MSG_TYPE_CLOSE_REQUEST from federate %d.", fed_id); - _lf_close_outbound_socket(fed_id); - } + // Received a request to close the socket. + LF_PRINT_DEBUG("Received MSG_TYPE_CLOSE_REQUEST from federate %d.", fed_id); + _lf_close_outbound_socket(fed_id); + } } lf_mutex_unlock(&outbound_socket_mutex); return NULL; @@ -844,9 +844,9 @@ void connect_to_federate(uint16_t remote_federate_id) { *remote_fed_id_copy = remote_federate_id; lf_thread_t thread_id; result = lf_thread_create( - &thread_id, - listen_for_upstream_messages_from_downstream_federates, - remote_fed_id_copy); + &thread_id, + listen_for_upstream_messages_from_downstream_federates, + remote_fed_id_copy); if (result != 0) { // Failed to create a listening thread. lf_print_error_and_exit( @@ -1131,7 +1131,7 @@ bool is_input_control_reaction_blocked() { * ports is known. */ void update_last_known_status_on_input_ports(tag_t tag) { - bool notify = false; + bool notify = false; for (int i = 0; i < _fed.triggers_for_network_input_control_reactions_size; i++) { trigger_t* input_port_action = _lf_action_for_port(i); // This is called when a TAG is received. @@ -1150,7 +1150,7 @@ void update_last_known_status_on_input_ports(tag_t tag) { ); input_port_action->last_known_status_tag = tag; if (input_port_action->is_a_control_reaction_waiting) { - notify = true; + notify = true; } } } @@ -1245,17 +1245,17 @@ port_status_t get_current_port_status(int portID) { // The status of the trigger is absent. return absent; } else if (network_input_port_action->status == unknown - && lf_tag_compare(network_input_port_action->last_known_status_tag, lf_tag()) >= 0) { + && lf_tag_compare(network_input_port_action->last_known_status_tag, lf_tag()) >= 0) { // We have a known status for this port in a future tag. Therefore, no event is going // to be present for this port at the current tag. set_network_port_status(portID, absent); return absent; } else if (_fed.is_last_TAG_provisional - && lf_tag_compare(_fed.last_TAG, lf_tag()) > 0) { - // In this case, a PTAG has been received with a larger tag than the current tag, - // which means that the input port is known to be absent. + && lf_tag_compare(_fed.last_TAG, lf_tag()) > 0) { + // In this case, a PTAG has been received with a larger tag than the current tag, + // which means that the input port is known to be absent. set_network_port_status(portID, absent); - return absent; + return absent; } return unknown; } @@ -1327,7 +1327,7 @@ void enqueue_network_control_reactions() { // If the granted tag is not provisional, there is no // need for network input control reactions if (lf_tag_compare(_fed.last_TAG, lf_tag()) != 0 - || _fed.is_last_TAG_provisional == false) { + || _fed.is_last_TAG_provisional == false) { return; } #endif @@ -1379,9 +1379,9 @@ void send_port_absent_to_federate(interval_t additional_delay, #endif // Do not write if the socket is closed. if (socket >= 0) { - write_to_socket_errexit_with_mutex(socket, message_length, buffer, &outbound_socket_mutex, - "Failed to send port absent message for port %hu to federate %hu.", - port_ID, fed_ID); + write_to_socket_errexit_with_mutex(socket, message_length, buffer, &outbound_socket_mutex, + "Failed to send port absent message for port %hu to federate %hu.", + port_ID, fed_ID); } lf_mutex_unlock(&outbound_socket_mutex); } @@ -1562,22 +1562,22 @@ trigger_handle_t schedule_message_received_from_network_already_locked( * @return 1 if the MSG_TYPE_CLOSE_REQUEST message is sent successfully, 0 otherwise. */ int _lf_request_close_inbound_socket(int fed_id) { - assert(fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES); + assert(fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES); - if (_fed.sockets_for_inbound_p2p_connections[fed_id] < 1) return 0; + if (_fed.sockets_for_inbound_p2p_connections[fed_id] < 1) return 0; - // Send a MSG_TYPE_CLOSE_REQUEST message. + // Send a MSG_TYPE_CLOSE_REQUEST message. unsigned char message_marker = MSG_TYPE_CLOSE_REQUEST; - LF_PRINT_LOG("Sending MSG_TYPE_CLOSE_REQUEST message to upstream federate."); + LF_PRINT_LOG("Sending MSG_TYPE_CLOSE_REQUEST message to upstream federate."); ssize_t written = write_to_socket( - _fed.sockets_for_inbound_p2p_connections[fed_id], - 1, &message_marker); + _fed.sockets_for_inbound_p2p_connections[fed_id], + 1, &message_marker); _fed.sockets_for_inbound_p2p_connections[fed_id] = -1; if (written == 1) { - LF_PRINT_LOG("Sent MSG_TYPE_CLOSE_REQUEST message to upstream federate."); - return 1; + LF_PRINT_LOG("Sent MSG_TYPE_CLOSE_REQUEST message to upstream federate."); + return 1; } else { - return 0; + return 0; } } @@ -1620,7 +1620,7 @@ void handle_port_absent_message(int socket, int fed_id) { size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; read_from_socket_errexit(socket, bytes_to_read, buffer, - "Failed to read port absent message."); + "Failed to read port absent message."); // Extract the header information. unsigned short port_id = extract_uint16(buffer); @@ -1671,7 +1671,7 @@ void handle_message(int socket, int fed_id) { size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t); unsigned char buffer[bytes_to_read]; read_from_socket_errexit(socket, bytes_to_read, buffer, - "Failed to read message header."); + "Failed to read message header."); // Extract the header information. unsigned short port_id; @@ -1689,7 +1689,7 @@ void handle_message(int socket, int fed_id) { // Allocate memory for the message contents. unsigned char* message_contents = (unsigned char*)malloc(length); read_from_socket_errexit(socket, length, message_contents, - "Failed to read message body."); + "Failed to read message body."); LF_PRINT_LOG("Message received by federate: %s. Length: %d.", message_contents, length); @@ -1720,7 +1720,7 @@ void handle_tagged_message(int socket, int fed_id) { + sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; read_from_socket_errexit(socket, bytes_to_read, buffer, - "Failed to read timed message header"); + "Failed to read timed message header"); // Extract the header information. unsigned short port_id; @@ -1756,13 +1756,13 @@ void handle_tagged_message(int socket, int fed_id) { #endif LF_PRINT_LOG("Received message with tag: (%lld, %u), Current tag: (%lld, %u).", intended_tag.time - start_time, intended_tag.microstep, - lf_time_logical_elapsed(), lf_tag().microstep); + lf_time_logical_elapsed(), lf_tag().microstep); // Read the payload. // Allocate memory for the message contents. unsigned char* message_contents = (unsigned char*)malloc(length); read_from_socket_errexit(socket, length, message_contents, - "Failed to read message body."); + "Failed to read message body."); // The following is only valid for string messages. // LF_PRINT_DEBUG("Message received: %s.", message_contents); @@ -1891,7 +1891,7 @@ void handle_tag_advance_grant() { size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; read_from_socket_errexit(_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read tag advance grant from RTI."); + "Failed to read tag advance grant from RTI."); tag_t TAG = extract_tag(buffer); lf_mutex_lock(&mutex); @@ -1910,11 +1910,11 @@ void handle_tag_advance_grant() { _fed.last_TAG.microstep = TAG.microstep; _fed.is_last_TAG_provisional = false; LF_PRINT_LOG("Received Time Advance Grant (TAG): (%lld, %u).", - _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); + _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); } else { lf_mutex_unlock(&mutex); lf_print_error("Received a TAG (%lld, %u) that wasn't larger " - "than the previous TAG or PTAG (%lld, %u). Ignoring the TAG.", + "than the previous TAG or PTAG (%lld, %u). Ignoring the TAG.", TAG.time - start_time, TAG.microstep, _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); } @@ -1967,7 +1967,7 @@ void handle_provisional_tag_advance_grant() { size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); unsigned char buffer[bytes_to_read]; read_from_socket_errexit(_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read provisional tag advance grant from RTI."); + "Failed to read provisional tag advance grant from RTI."); tag_t PTAG = extract_tag(buffer); // Note: it is important that last_known_status_tag of ports does not @@ -1978,31 +1978,31 @@ void handle_provisional_tag_advance_grant() { // Sanity check if (lf_tag_compare(PTAG, _fed.last_TAG) < 0 - || (lf_tag_compare(PTAG, _fed.last_TAG) == 0 && !_fed.is_last_TAG_provisional)) { + || (lf_tag_compare(PTAG, _fed.last_TAG) == 0 && !_fed.is_last_TAG_provisional)) { lf_mutex_unlock(&mutex); lf_print_error_and_exit("Received a PTAG (%lld, %d) that is equal or earlier " - "than an already received TAG (%lld, %d).", - PTAG.time, PTAG.microstep, - _fed.last_TAG.time, _fed.last_TAG.microstep); + "than an already received TAG (%lld, %d).", + PTAG.time, PTAG.microstep, + _fed.last_TAG.time, _fed.last_TAG.microstep); } _fed.last_TAG = PTAG; _fed.waiting_for_TAG = false; _fed.is_last_TAG_provisional = true; LF_PRINT_LOG("At tag (%lld, %d), received Provisional Tag Advance Grant (PTAG): (%lld, %u).", - current_tag.time - start_time, current_tag.microstep, - _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); + current_tag.time - start_time, current_tag.microstep, + _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); // Even if we don't modify the event queue, we need to broadcast a change // because we do not need to continue to wait for a TAG. - lf_cond_broadcast(&event_q_changed); - // Notify control reactions that are blocked. + lf_cond_broadcast(&event_q_changed); + // Notify control reactions that are blocked. // Check here whether there is any control reaction waiting // before broadcasting to avoid an unnecessary broadcast. // This also avoids problems waking up threads before execution // has started (while they are waiting for the start time). if (is_input_control_reaction_blocked()) { - lf_cond_broadcast(&port_status_changed); + lf_cond_broadcast(&port_status_changed); } // Possibly insert a dummy event into the event queue if current time is behind @@ -2013,44 +2013,44 @@ void handle_provisional_tag_advance_grant() { microstep_t dummy_event_relative_microstep = PTAG.microstep; if (lf_tag_compare(current_tag, PTAG) == 0) { - // The current tag can equal the PTAG if we are at the start time - // or if this federate has been able to advance time to the current - // tag (e.g., it has no upstream federates). In either case, either - // it is already treating the current tag as PTAG cycle (e.g. at the - // start time) or it will be completing the current cycle and sending - // a LTC message shortly. In either case, there is nothing more to do. - lf_mutex_unlock(&mutex); - return; + // The current tag can equal the PTAG if we are at the start time + // or if this federate has been able to advance time to the current + // tag (e.g., it has no upstream federates). In either case, either + // it is already treating the current tag as PTAG cycle (e.g. at the + // start time) or it will be completing the current cycle and sending + // a LTC message shortly. In either case, there is nothing more to do. + lf_mutex_unlock(&mutex); + return; } else if (lf_tag_compare(current_tag, PTAG) > 0) { - // Current tag is greater than the PTAG. - // It could be that we have sent an LTC that crossed with the incoming - // PTAG or that we have advanced to a tag greater than the PTAG. - // In the former case, there is nothing more to do. - // In the latter case, we may be blocked processing a PTAG cycle at - // a greater tag or we may be in the middle of processing a regular - // TAG. In either case, we know that at the PTAG tag, all outputs - // have either been sent or are absent, so we can send an LTC. - // Send an LTC to indicate absent outputs. - _lf_logical_tag_complete(PTAG); - // Nothing more to do. - lf_mutex_unlock(&mutex); - return; + // Current tag is greater than the PTAG. + // It could be that we have sent an LTC that crossed with the incoming + // PTAG or that we have advanced to a tag greater than the PTAG. + // In the former case, there is nothing more to do. + // In the latter case, we may be blocked processing a PTAG cycle at + // a greater tag or we may be in the middle of processing a regular + // TAG. In either case, we know that at the PTAG tag, all outputs + // have either been sent or are absent, so we can send an LTC. + // Send an LTC to indicate absent outputs. + _lf_logical_tag_complete(PTAG); + // Nothing more to do. + lf_mutex_unlock(&mutex); + return; } else if (PTAG.time == current_tag.time) { - // We now know current_tag < PTAG, but the times are equal. - // Adjust the microstep for scheduling the dummy event. - dummy_event_relative_microstep -= current_tag.microstep; + // We now know current_tag < PTAG, but the times are equal. + // Adjust the microstep for scheduling the dummy event. + dummy_event_relative_microstep -= current_tag.microstep; } - // We now know current_tag < PTAG. + // We now know current_tag < PTAG. // Schedule a dummy event at the specified time and (relative) microstep. - LF_PRINT_DEBUG("At tag (%lld, %d), inserting into the event queue a dummy event " - "with time %lld and (relative) microstep %d.", - current_tag.time - start_time, current_tag.microstep, - dummy_event_time - start_time, dummy_event_relative_microstep); + LF_PRINT_DEBUG("At tag (%lld, %d), inserting into the event queue a dummy event " + "with time %lld and (relative) microstep %d.", + current_tag.time - start_time, current_tag.microstep, + dummy_event_time - start_time, dummy_event_relative_microstep); - // Dummy event points to a NULL trigger and NULL real event. - event_t* dummy = _lf_create_dummy_events( - NULL, dummy_event_time, NULL, dummy_event_relative_microstep); - pqueue_insert(event_q, dummy); + // Dummy event points to a NULL trigger and NULL real event. + event_t* dummy = _lf_create_dummy_events( + NULL, dummy_event_time, NULL, dummy_event_relative_microstep); + pqueue_insert(event_q, dummy); lf_mutex_unlock(&mutex); } @@ -2065,7 +2065,7 @@ void handle_provisional_tag_advance_grant() { * This function assumes the caller holds the mutex lock. */ void _lf_fd_send_stop_request_to_rti() { - // Do not send a stop request twice. + // Do not send a stop request twice. if (_fed.sent_a_stop_request_to_rti == true) { return; } @@ -2080,12 +2080,12 @@ void _lf_fd_send_stop_request_to_rti() { lf_mutex_lock(&outbound_socket_mutex); if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); + lf_print_warning("Socket is no longer connected. Dropping message."); lf_mutex_unlock(&outbound_socket_mutex); - return; + return; } write_to_socket_errexit_with_mutex(_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, - buffer, &outbound_socket_mutex, + buffer, &outbound_socket_mutex, "Failed to send stop time %lld to the RTI.", current_tag.time - start_time); lf_mutex_unlock(&outbound_socket_mutex); _fed.sent_a_stop_request_to_rti = true; @@ -2105,7 +2105,7 @@ void handle_stop_granted_message() { size_t bytes_to_read = MSG_TYPE_STOP_GRANTED_LENGTH - 1; unsigned char buffer[bytes_to_read]; read_from_socket_errexit(_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read stop granted from RTI."); + "Failed to read stop granted from RTI."); // Acquire a mutex lock to ensure that this state does change while a // message is transport or being used to determine a TAG. @@ -2120,7 +2120,7 @@ void handle_stop_granted_message() { tag_t current_tag = lf_tag(); if (lf_tag_compare(received_stop_tag, current_tag) <= 0) { lf_print_error("RTI granted a MSG_TYPE_STOP_GRANTED tag that is equal to or less than this federate's current tag (%lld, %u). " - "Stopping at the next microstep instead.", + "Stopping at the next microstep instead.", current_tag.time - start_time, current_tag.microstep); received_stop_tag = current_tag; received_stop_tag.microstep++; @@ -2149,7 +2149,7 @@ void handle_stop_request_message() { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; unsigned char buffer[bytes_to_read]; read_from_socket_errexit(_fed.socket_TCP_RTI, bytes_to_read, buffer, - "Failed to read stop request from RTI."); + "Failed to read stop request from RTI."); // Acquire a mutex lock to ensure that this state does change while a // message is being used to determine a TAG. @@ -2170,7 +2170,7 @@ void handle_stop_request_message() { // Encode the current logical time plus one microstep // or the requested tag_to_stop, whichever is bigger. if (lf_tag_compare(tag_to_stop, current_tag) <= 0) { - // Can't stop at the requested tag. Make a counteroffer. + // Can't stop at the requested tag. Make a counteroffer. tag_to_stop = current_tag; tag_to_stop.microstep++; } @@ -2180,15 +2180,15 @@ void handle_stop_request_message() { lf_mutex_lock(&outbound_socket_mutex); if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); + lf_print_warning("Socket is no longer connected. Dropping message."); lf_mutex_unlock(&outbound_socket_mutex); lf_mutex_unlock(&mutex); - return; + return; } // Send the current logical time to the RTI. This message does not have an identifying byte since // since the RTI is waiting for a response from this federate. write_to_socket_errexit_with_mutex( - _fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer, &outbound_socket_mutex, + _fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer, &outbound_socket_mutex, "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI."); lf_mutex_unlock(&outbound_socket_mutex); @@ -2225,11 +2225,11 @@ void terminate_execution() { _lf_close_outbound_socket(i); } // Resign the federation, which will close the socket to the RTI. - if (_fed.socket_TCP_RTI >= 0) { + if (_fed.socket_TCP_RTI >= 0) { unsigned char message_marker = MSG_TYPE_RESIGN; ssize_t written = write_to_socket(_fed.socket_TCP_RTI, 1, &message_marker); if (written == 1) { - LF_PRINT_LOG("Resigned."); + LF_PRINT_LOG("Resigned."); } } lf_mutex_unlock(&outbound_socket_mutex); @@ -2237,19 +2237,19 @@ void terminate_execution() { // Request closing the incoming P2P sockets. for (int i=0; i < NUMBER_OF_FEDERATES; i++) { if (_lf_request_close_inbound_socket(i) == 0) { - // Sending the close request failed. Mark the socket closed. - _fed.sockets_for_inbound_p2p_connections[i] = -1; + // Sending the close request failed. Mark the socket closed. + _fed.sockets_for_inbound_p2p_connections[i] = -1; } } // Wait for each inbound socket listener thread to close. if (_fed.number_of_inbound_p2p_connections > 0) { - LF_PRINT_LOG("Waiting for %d threads listening for incoming messages to exit.", - _fed.number_of_inbound_p2p_connections); - for (int i=0; i < _fed.number_of_inbound_p2p_connections; i++) { - // Ignoring errors here. - lf_thread_join(_fed.inbound_socket_listeners[i], NULL); - } + LF_PRINT_LOG("Waiting for %d threads listening for incoming messages to exit.", + _fed.number_of_inbound_p2p_connections); + for (int i=0; i < _fed.number_of_inbound_p2p_connections; i++) { + // Ignoring errors here. + lf_thread_join(_fed.inbound_socket_listeners[i], NULL); + } } // Wait for the thread listening for messages from the RTI to close. @@ -2567,7 +2567,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // However, -fast is really incompatible with federated execution with // physical connections, so I don't think we need to worry about this. LF_PRINT_DEBUG("Granted tag (%lld, %u) because the federate has neither " - "upstream nor downstream federates.", + "upstream nor downstream federates.", tag.time - start_time, tag.microstep); return tag; } @@ -2576,7 +2576,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // or a larger tag, then return immediately. if (lf_tag_compare(_fed.last_TAG, tag) >= 0) { LF_PRINT_DEBUG("Granted tag (%lld, %u) because TAG or PTAG has been received.", - _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); + _fed.last_TAG.time - start_time, _fed.last_TAG.microstep); return _fed.last_TAG; } @@ -2597,7 +2597,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // What we do next depends on whether the NET has been bounded by // physical time or by an event on the event queue. if (!tag_bounded_by_physical_time) { - // This if statement does not fall through but rather returns. + // This if statement does not fall through but rather returns. // NET is not bounded by physical time or has no downstream federates. // Normal case. _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, tag, wait_for_reply); @@ -2617,7 +2617,7 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // throttle upstream federates. if (!_fed.has_upstream) { LF_PRINT_DEBUG("Not waiting for reply to NET (%lld, %u) because I " - "have no upstream federates.", + "have no upstream federates.", tag.time - start_time, tag.microstep); return tag; } @@ -2641,18 +2641,18 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { // Check whether the new event on the event queue requires sending a new NET. tag_t next_tag = get_next_event_tag(); if (lf_tag_compare(next_tag, tag) != 0) { - _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); - _fed.last_sent_NET = next_tag; - LF_PRINT_LOG("Sent next event tag (NET) (%lld, %u) to RTI.", - next_tag.time - start_time, next_tag.microstep); + _lf_send_tag(MSG_TYPE_NEXT_EVENT_TAG, next_tag, wait_for_reply); + _fed.last_sent_NET = next_tag; + LF_PRINT_LOG("Sent next event tag (NET) (%lld, %u) to RTI.", + next_tag.time - lf_time_start(), next_tag.microstep); } } } // Create a dummy event that will force this federate to advance time and subsequently enable progress for // downstream federates. - event_t* dummy = _lf_create_dummy_events(NULL, tag.time, NULL, 0); - pqueue_insert(event_q, dummy); + event_t* dummy = _lf_create_dummy_events(NULL, tag.time, NULL, 0); + pqueue_insert(event_q, dummy); LF_PRINT_DEBUG("Inserted a dummy event for tag (%ld, %u).", tag.time - lf_time_start()); diff --git a/core/federated/federate.h b/core/federated/federate.h index 496418adf..1a21d5d9c 100644 --- a/core/federated/federate.h +++ b/core/federated/federate.h @@ -40,194 +40,194 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * Structure that a federate instance uses to keep track of its own state. */ typedef struct federate_instance_t { - /** - * The TCP socket descriptor for this federate to communicate with the RTI. - * This is set by connect_to_rti(), which must be called before other - * functions that communicate with the rti are called. - */ - int socket_TCP_RTI; - - /** - * Thread listening for incoming TCP messages from the RTI. - */ - lf_thread_t RTI_socket_listener; - - /** - * Number of inbound physical connections to the federate. - * This can be either physical connections, or logical connections - * in the decentralized coordination, or both. - */ - size_t number_of_inbound_p2p_connections; - - /** - * Array of thread IDs for threads that listen for incoming messages. - * This is NULL if there are none and otherwise has size given by - * number_of_inbound_p2p_connections. - */ - lf_thread_t *inbound_socket_listeners; - - /** - * Number of outbound peer-to-peer connections from the federate. - * This can be either physical connections, or logical connections - * in the decentralized coordination, or both. - */ - size_t number_of_outbound_p2p_connections; - - /** - * An array that holds the socket descriptors for inbound - * connections from each federate. The index will be the federate - * ID of the remote sending federate. This is initialized at startup - * to -1 and is set to a socket ID by handle_p2p_connections_from_federates() - * when the socket is opened. - * - * @note There will not be an inbound socket unless a physical connection - * or a p2p logical connection (by setting the coordination target property - * to "distributed") is specified in the Lingua Franca program where this - * federate is the destination. Multiple incoming p2p connections from the - * same remote federate will use the same socket. - */ - int sockets_for_inbound_p2p_connections[NUMBER_OF_FEDERATES]; - - /** - * An array that holds the socket descriptors for outbound direct - * connections to each remote federate. The index will be the federate - * ID of the remote receiving federate. This is initialized at startup - * to -1 and is set to a socket ID by connect_to_federate() - * when the socket is opened. - * - * @note This federate will not open an outbound socket unless a physical - * connection or a p2p logical connection (by setting the coordination target - * property to "distributed") is specified in the Lingua Franca - * program where this federate acts as the source. Multiple outgoing p2p - * connections to the same remote federate will use the same socket. - */ - int sockets_for_outbound_p2p_connections[NUMBER_OF_FEDERATES]; - - /** - * Thread ID for a thread that accepts sockets and then supervises - * listening to those sockets for incoming P2P (physical) connections. - */ - lf_thread_t inbound_p2p_handling_thread_id; - - /** - * A socket descriptor for the socket server of the federate. - * This is assigned in create_server(). - * This socket is used to listen to incoming physical connections from - * remote federates. Once an incoming connection is accepted, the - * opened socket will be stored in - * federate_sockets_for_inbound_p2p_connections. - */ - int server_socket; - - /** - * The port used for the server socket - * to listen for messages from other federates. - * The federate informs the RTI of this port once - * it has created its socket server by sending - * an ADDRESS_AD message (@see rti.h). - */ - int server_port; - - /** - * Most recent TIME_ADVANCE_GRANT received from the RTI, or NEVER if none - * has been received. - * This is used to communicate between the listen_to_rti_TCP thread and the - * main federate thread. - * This variable should only be accessed while holding the mutex lock. - */ - tag_t last_TAG; - - /** - * Indicates whether the last TAG received is provisional or an ordinary - * TAG. - * If the last TAG has been provisional, network control reactions must be inserted. - * This variable should only be accessed while holding the mutex lock. - */ - bool is_last_TAG_provisional; - - /** - * Indicator of whether a NET has been sent to the RTI and no TAG - * yet received in reply. - * This variable should only be accessed while holding the mutex lock. - */ - bool waiting_for_TAG; - - /** - * Indicator of whether this federate has upstream federates. - * The default value of false may be overridden in _lf_initialize_trigger_objects. - */ - bool has_upstream; - - /** - * Indicator of whether this federate has downstream federates. - * The default value of false may be overridden in _lf_initialize_trigger_objects. - */ - bool has_downstream; - - /** - * Used to prevent the federate from sending a REQUEST_STOP - * message multiple times to the RTI. - * This variable should only be accessed while holding the mutex lock. - */ - bool sent_a_stop_request_to_rti; - - /** - * A record of the most recently sent LTC (logical tag complete) message. - * In some situations, federates can send logical_tag_complete for - * the same tag twice or more in-a-row to the RTI. For example, when - * _lf_next() returns without advancing tag. To prevent overwhelming - * the RTI with extra messages, record the last sent logical tag - * complete message and check against it in - * _lf_logical_tag_complete(). - * - * @note Here, the underlying assumption is that the TCP stack will - * deliver the Logical TAG Complete message to the RTI eventually - * if it is deliverable - */ - tag_t last_sent_LTC; - - /** - * A record of the most recently sent NET (next event tag) message. - */ - tag_t last_sent_NET; - - /** - * For use in federates with centralized coordination, the minimum - * time delay between a physical action within this federate and an - * output from this federate. This is NEVER if there is causal - * path from a physical action to any output. - */ - instant_t min_delay_from_physical_action_to_federate_output; - - /** - * This list is also used to determine the status of a given network - * input port at a given logical time. The status of the port (trigger->status) can be: - * present, absent, or unknown. To determine the status of that port, for a given trigger - * 't' in this list, a (number of) network input control reactions are inserted into the - * reaction queue, which is are special kind of reaction that wait long enough until the - * status of the port becomes known. In the centralized coordination, this wait is until - * the RTI informs the reaction of the status of the port. In the decentralized coordination, - * this wait is until the STP offset expires (or the status is somehow becomes known sooner). - */ - - /** - * List of triggers for network input control reactions, used - * to trigger these reaction at the beginning of every tag. - */ - trigger_t** triggers_for_network_input_control_reactions; - size_t triggers_for_network_input_control_reactions_size; - - - /** - * The triggers for all network output control reactions. - * - * This is used to trigger network output - * control reactions that will potentially send an ABSENT - * message to any downstream federate that might be blocking - * on the network port. The ABSENT message will only be sent if - * the output is not present. - */ - trigger_t* trigger_for_network_output_control_reactions; + /** + * The TCP socket descriptor for this federate to communicate with the RTI. + * This is set by connect_to_rti(), which must be called before other + * functions that communicate with the rti are called. + */ + int socket_TCP_RTI; + + /** + * Thread listening for incoming TCP messages from the RTI. + */ + lf_thread_t RTI_socket_listener; + + /** + * Number of inbound physical connections to the federate. + * This can be either physical connections, or logical connections + * in the decentralized coordination, or both. + */ + size_t number_of_inbound_p2p_connections; + + /** + * Array of thread IDs for threads that listen for incoming messages. + * This is NULL if there are none and otherwise has size given by + * number_of_inbound_p2p_connections. + */ + lf_thread_t *inbound_socket_listeners; + + /** + * Number of outbound peer-to-peer connections from the federate. + * This can be either physical connections, or logical connections + * in the decentralized coordination, or both. + */ + size_t number_of_outbound_p2p_connections; + + /** + * An array that holds the socket descriptors for inbound + * connections from each federate. The index will be the federate + * ID of the remote sending federate. This is initialized at startup + * to -1 and is set to a socket ID by handle_p2p_connections_from_federates() + * when the socket is opened. + * + * @note There will not be an inbound socket unless a physical connection + * or a p2p logical connection (by setting the coordination target property + * to "distributed") is specified in the Lingua Franca program where this + * federate is the destination. Multiple incoming p2p connections from the + * same remote federate will use the same socket. + */ + int sockets_for_inbound_p2p_connections[NUMBER_OF_FEDERATES]; + + /** + * An array that holds the socket descriptors for outbound direct + * connections to each remote federate. The index will be the federate + * ID of the remote receiving federate. This is initialized at startup + * to -1 and is set to a socket ID by connect_to_federate() + * when the socket is opened. + * + * @note This federate will not open an outbound socket unless a physical + * connection or a p2p logical connection (by setting the coordination target + * property to "distributed") is specified in the Lingua Franca + * program where this federate acts as the source. Multiple outgoing p2p + * connections to the same remote federate will use the same socket. + */ + int sockets_for_outbound_p2p_connections[NUMBER_OF_FEDERATES]; + + /** + * Thread ID for a thread that accepts sockets and then supervises + * listening to those sockets for incoming P2P (physical) connections. + */ + lf_thread_t inbound_p2p_handling_thread_id; + + /** + * A socket descriptor for the socket server of the federate. + * This is assigned in create_server(). + * This socket is used to listen to incoming physical connections from + * remote federates. Once an incoming connection is accepted, the + * opened socket will be stored in + * federate_sockets_for_inbound_p2p_connections. + */ + int server_socket; + + /** + * The port used for the server socket + * to listen for messages from other federates. + * The federate informs the RTI of this port once + * it has created its socket server by sending + * an ADDRESS_AD message (@see rti.h). + */ + int server_port; + + /** + * Most recent TIME_ADVANCE_GRANT received from the RTI, or NEVER if none + * has been received. + * This is used to communicate between the listen_to_rti_TCP thread and the + * main federate thread. + * This variable should only be accessed while holding the mutex lock. + */ + tag_t last_TAG; + + /** + * Indicates whether the last TAG received is provisional or an ordinary + * TAG. + * If the last TAG has been provisional, network control reactions must be inserted. + * This variable should only be accessed while holding the mutex lock. + */ + bool is_last_TAG_provisional; + + /** + * Indicator of whether a NET has been sent to the RTI and no TAG + * yet received in reply. + * This variable should only be accessed while holding the mutex lock. + */ + bool waiting_for_TAG; + + /** + * Indicator of whether this federate has upstream federates. + * The default value of false may be overridden in _lf_initialize_trigger_objects. + */ + bool has_upstream; + + /** + * Indicator of whether this federate has downstream federates. + * The default value of false may be overridden in _lf_initialize_trigger_objects. + */ + bool has_downstream; + + /** + * Used to prevent the federate from sending a REQUEST_STOP + * message multiple times to the RTI. + * This variable should only be accessed while holding the mutex lock. + */ + bool sent_a_stop_request_to_rti; + + /** + * A record of the most recently sent LTC (logical tag complete) message. + * In some situations, federates can send logical_tag_complete for + * the same tag twice or more in-a-row to the RTI. For example, when + * _lf_next() returns without advancing tag. To prevent overwhelming + * the RTI with extra messages, record the last sent logical tag + * complete message and check against it in + * _lf_logical_tag_complete(). + * + * @note Here, the underlying assumption is that the TCP stack will + * deliver the Logical TAG Complete message to the RTI eventually + * if it is deliverable + */ + tag_t last_sent_LTC; + + /** + * A record of the most recently sent NET (next event tag) message. + */ + tag_t last_sent_NET; + + /** + * For use in federates with centralized coordination, the minimum + * time delay between a physical action within this federate and an + * output from this federate. This is NEVER if there is causal + * path from a physical action to any output. + */ + instant_t min_delay_from_physical_action_to_federate_output; + + /** + * This list is also used to determine the status of a given network + * input port at a given logical time. The status of the port (trigger->status) can be: + * present, absent, or unknown. To determine the status of that port, for a given trigger + * 't' in this list, a (number of) network input control reactions are inserted into the + * reaction queue, which is are special kind of reaction that wait long enough until the + * status of the port becomes known. In the centralized coordination, this wait is until + * the RTI informs the reaction of the status of the port. In the decentralized coordination, + * this wait is until the STP offset expires (or the status is somehow becomes known sooner). + */ + + /** + * List of triggers for network input control reactions, used + * to trigger these reaction at the beginning of every tag. + */ + trigger_t** triggers_for_network_input_control_reactions; + size_t triggers_for_network_input_control_reactions_size; + + + /** + * The triggers for all network output control reactions. + * + * This is used to trigger network output + * control reactions that will potentially send an ABSENT + * message to any downstream federate that might be blocking + * on the network port. The ABSENT message will only be sent if + * the output is not present. + */ + trigger_t* trigger_for_network_output_control_reactions; } federate_instance_t; From 15b2f435e27cce41a05a2c65b42cfe622fdb6c86 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Tue, 31 May 2022 15:06:33 -0500 Subject: [PATCH 25/37] Removed in_transit_message --- core/federated/RTI/rti.c | 1 - core/federated/RTI/rti.h | 1 - 2 files changed, 2 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index f75ecdddc..9c92de62c 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -905,7 +905,6 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { if (lf_tag_compare(_RTI.federates[i].next_event, _RTI.max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. _RTI.federates[i].next_event = _RTI.max_stop_tag; - _RTI.federates[i].in_transit_message = false; } write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", _RTI.federates[i].id); diff --git a/core/federated/RTI/rti.h b/core/federated/RTI/rti.h index 54a4f26ba..454514808 100644 --- a/core/federated/RTI/rti.h +++ b/core/federated/RTI/rti.h @@ -76,7 +76,6 @@ typedef struct federate_t { tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). - bool in_transit_message;// Indicator that next_event was set by the RTI because of a message destined to the federate. fed_state_t state; // State of the federate. int* upstream; // Array of upstream federate ids. interval_t* upstream_delay; // Minimum delay on connections from upstream federates. From fe87760173020aad6b4e8a4eab51a1bedad5f333 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Wed, 1 Jun 2022 23:39:55 -0500 Subject: [PATCH 26/37] Adjusted log messages --- core/federated/RTI/rti.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 9c92de62c..c181a4168 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -523,12 +523,6 @@ bool send_advance_grant_if_safe(federate_t* fed) { LF_PRINT_LOG("Upstream federates are all done. Granting tag advance."); send_tag_advance_grant(fed, FOREVER_TAG); return true; - } - - if (t_d.time == FOREVER) { - LF_PRINT_LOG("All upstream federates are finished. Sending TAG(FOREVER)."); - send_tag_advance_grant(fed, FOREVER_TAG); - return true; } else if ( lf_tag_compare(t_d, fed->next_event) > 0 // The federate has something to do. && lf_tag_compare(t_d, fed->last_provisionally_granted) >= 0 // The grant is not redundant @@ -703,8 +697,9 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { bytes_to_read = FED_COM_BUFFER_SIZE - header_size; } - LF_PRINT_LOG("RTI received message from federate %d for federate %u port %u. Forwarding.", - sending_federate->id, federate_id, reactor_port_id); + LF_PRINT_LOG("RTI received message from federate %d for federate %u port %u with intended tag (%ld, %u). Forwarding.", + sending_federate->id, federate_id, reactor_port_id, + intended_tag.time - lf_time_start(), intended_tag.microstep); read_from_socket_errexit(sending_federate->socket, bytes_to_read, &(buffer[header_size]), "RTI failed to read timed message from federate %d.", federate_id); @@ -786,7 +781,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // is a promise that is valid only in the absence of network inputs, // and now there is a network input. Hence, the promise needs to be // updated. - if (lf_tag_compare(_RTI.federates[federate_id].next_event, intended_tag) > 0) { + if (lf_tag_compare(_RTI.federates[federate_id].next_event, intended_tag) >= 0) { update_federate_next_event_tag_locked(federate_id, intended_tag); } @@ -855,11 +850,12 @@ void handle_next_event_tag(federate_t* fed) { if (lf_tag_compare(intended_tag, fed->next_event) <= 0 || lf_tag_compare(fed->completed, fed->next_event) >= 0) { + + update_federate_next_event_tag_locked(fed->id, intended_tag); + LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%ld, %u).", fed->id, fed->next_event.time - start_time, fed->next_event.microstep); - - update_federate_next_event_tag_locked(fed->id, intended_tag); } else { lf_print_error("RTI received from federate %d the Next Event Tag (NET) (%ld, %u), " "but it is ignoring it since there is a message still in transit to federate %d. " From 98dd2c2539bebe1186c58d6ae303e72a225b0f1b Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Thu, 2 Jun 2022 16:13:45 -0500 Subject: [PATCH 27/37] Added a pqueue for in-transit messages --- core/federated/RTI/CMakeLists.txt | 8 +- .../RTI/message_record/message_record.c | 151 ++++++++++++++++++ .../RTI/message_record/message_record.h | 86 ++++++++++ .../RTI/message_record/rti_pqueue_support.h | 112 +++++++++++++ core/federated/RTI/rti.c | 109 ++++++++----- core/federated/RTI/rti.h | 6 + core/utils/pqueue_support.h | 2 +- 7 files changed, 428 insertions(+), 46 deletions(-) create mode 100644 core/federated/RTI/message_record/message_record.c create mode 100644 core/federated/RTI/message_record/message_record.h create mode 100644 core/federated/RTI/message_record/rti_pqueue_support.h diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt index c51591889..fbb30c4ae 100644 --- a/core/federated/RTI/CMakeLists.txt +++ b/core/federated/RTI/CMakeLists.txt @@ -43,7 +43,13 @@ include_directories(${CoreLib}/platform) include_directories(${CoreLib}/federated) # Declare a new executable target and list all its sources -add_executable(RTI rti.c ${LF_PLATFORM_FILE}) +add_executable( + RTI + rti.c + ${LF_PLATFORM_FILE} + ${CoreLib}/utils/pqueue.c + message_record/message_record.c +) IF(CMAKE_BUILD_TYPE MATCHES DEBUG) # Set the LOG_LEVEL to 4 to get DEBUG messages diff --git a/core/federated/RTI/message_record/message_record.c b/core/federated/RTI/message_record/message_record.c new file mode 100644 index 000000000..05e726eba --- /dev/null +++ b/core/federated/RTI/message_record/message_record.c @@ -0,0 +1,151 @@ +/** + * @file message_record.c + * @author your name (you@domain.com) + * @brief + * @version 0.1 + * @date 2022-06-02 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "message_record.h" +#include + +/** + * @brief Initialize the in-transit message record queue. + * + * @return in_transit_message_record_q + */ +in_transit_message_record_q_t initialize_in_transit_message_q() { + in_transit_message_record_q_t queue = + (in_transit_message_record_q_t)calloc( + 1, + sizeof(struct in_transit_message_record_q) + ); + queue->main_queue = pqueue_init( + 10, + in_reverse_order, + get_message_record_index, + get_message_record_position, + set_message_record_position, + tags_match, + print_message_record + ); + + queue->transfer_queue = pqueue_init( + 10, + in_reverse_order, + get_message_record_index, + get_message_record_position, + set_message_record_position, + tags_match, + print_message_record + ); + + return queue; +} + +/** + * @brief Free the memory occupied by the `queue`. + * + * @param queue The queue to free. + */ +void free_in_transit_message_q(in_transit_message_record_q_t queue) { + pqueue_free(queue->main_queue); + pqueue_free(queue->transfer_queue); + free(queue); +} + +/** + * @brief Add a record of the in-transit message. + * + * @param queue The queue to add to (of type `in_transit_message_record_q_t`. + * @param tag The tag of the in-transit message. + * @return 0 on success. + */ +int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag) { + in_transit_message_record_t* in_transit_record = malloc(sizeof(in_transit_message_record_t)); + in_transit_record->tag = tag; + return pqueue_insert( + queue->main_queue, + (void*)in_transit_record + ); +} + +/** + * @brief Clean the record of in-transit messages up to and including `tag`. + * + * @param queue The queue to clean (of type `in_transit_message_record_q_t`. + * @param tag Will clean all messages with tags <= tag. + */ +void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t queue, tag_t tag) { + in_transit_message_record_t* head_of_in_transit_messages = (in_transit_message_record_t*)pqueue_peek(queue->main_queue); + while ( + head_of_in_transit_messages != NULL && // Queue is not empty + head_of_in_transit_messages->tag.time <= tag.time // The head message record has a time less than or equal to + // `tag.time`. + ) { + // Now compare the tags. The message record queue is ordered according to the `time` field, so we need to check + // all records with that `time` and find those that have smaller or equal full tags. + if (lf_tag_compare( + head_of_in_transit_messages->tag, + tag + ) <= 0 + ) { + LF_PRINT_DEBUG( + "RTI: Removed a message with tag (%ld, %u) from the list of in-transit messages.", + head_of_in_transit_messages->tag.time - lf_time_start(), + head_of_in_transit_messages->tag.microstep + ); + + free(pqueue_pop(queue->main_queue)); + } else { + // Add it to the transfer queue + pqueue_insert(queue->transfer_queue, pqueue_pop(queue->main_queue)); + } + head_of_in_transit_messages = (in_transit_message_record_t*)pqueue_peek(queue->main_queue); + } + // Empty the transfer queue (which holds messages with equal time but larger microstep) into the main queue. + pqueue_empty_into(&queue->main_queue, &queue->transfer_queue); +} + +/** + * @brief Get the minimum tag of all currently recorded in-transit messages. + * + * @param queue The queue to search in (of type `in_transit_message_record_q`). + * @return tag_t The minimum tag of all currently recorded in-transit messages. Return `FOREVER_TAG` if the queue is empty. + */ +tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t queue) { + tag_t minimum_tag = FOREVER_TAG; + + in_transit_message_record_t* head_of_in_transit_messages = (in_transit_message_record_t*)pqueue_peek(queue->main_queue); + while (head_of_in_transit_messages != NULL) { // Queue is not empty + // The message record queue is ordered according to the `time` field, so we need to check + // all records with the minimum `time` and find those that have the smallest tag. + if (lf_tag_compare( + head_of_in_transit_messages->tag, + minimum_tag + ) <= 0 + ) { + minimum_tag = head_of_in_transit_messages->tag; + + LF_PRINT_DEBUG( + "RTI: Removed a message with tag (%ld, %u) from the list of in-transit messages.", + head_of_in_transit_messages->tag.time - lf_time_start(), + head_of_in_transit_messages->tag.microstep + ); + } else if (head_of_in_transit_messages->tag.time > minimum_tag.time) { + break; + } + + // Add the head to the transfer queue. + pqueue_insert(queue->transfer_queue, pqueue_pop(queue->main_queue)); + + head_of_in_transit_messages = (in_transit_message_record_t*)pqueue_peek(queue->main_queue); + } + // Empty the transfer queue (which holds messages with equal time but larger microstep) into the main queue. + pqueue_empty_into(&queue->main_queue, &queue->transfer_queue); + + return minimum_tag; +} diff --git a/core/federated/RTI/message_record/message_record.h b/core/federated/RTI/message_record/message_record.h new file mode 100644 index 000000000..22276a43a --- /dev/null +++ b/core/federated/RTI/message_record/message_record.h @@ -0,0 +1,86 @@ +/** + * @file message_record.h + * @author Soroush Bateni (soroush@berkeley.edu) + * @brief Record-keeping for in-transit messages. + * @version 0.1 + * @date 2022-06-02 + * + * @copyright Copyright (c) 2022, The University of California at Berkeley. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +***************/ + +#ifndef RTI_MESSAGE_RECORD_H +#define RTI_MESSAGE_RECORD_H + +#include "rti_pqueue_support.h" + +/** + * @brief Queue to keep a record of in-transit messages. + * + */ +struct in_transit_message_record_q { + pqueue_t* main_queue; // The primary queue. + pqueue_t* transfer_queue; // Queue used for housekeeping. +}; + +typedef struct in_transit_message_record_q* in_transit_message_record_q_t; + +/** + * @brief Initialize the in-transit message record queue. + * + * @return in_transit_message_record_q + */ +in_transit_message_record_q_t initialize_in_transit_message_q(); + +/** + * @brief Free the memory occupied by the `queue`. + * + * @param queue The queue to free. + */ +void free_in_transit_message_q(in_transit_message_record_q_t queue); + +/** + * @brief Add a record of the in-transit message. + * + * @param queue The queue to add to (of type `in_transit_message_record_q`. + * @param tag The tag of the in-transit message. + * @return 0 on success. + */ +int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag); + +/** + * @brief Clean the record of in-transit messages up to and including `tag`. + * + * @param queue The queue to clean (of type `in_transit_message_record_q`. + * @param tag Will clean all messages with tags <= tag. + */ +void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t queue, tag_t tag); + +/** + * @brief Get the minimum tag of all currently recorded in-transit messages. + * + * @param queue The queue to search in (of type `in_transit_message_record_q`). + * @return tag_t The minimum tag of all currently recorded in-transit messages. Return `FOREVER_TAG` if the queue is empty. + */ +tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t queue); + +#endif // RTI_MESSAGE_RECORD_H diff --git a/core/federated/RTI/message_record/rti_pqueue_support.h b/core/federated/RTI/message_record/rti_pqueue_support.h new file mode 100644 index 000000000..a851135d1 --- /dev/null +++ b/core/federated/RTI/message_record/rti_pqueue_support.h @@ -0,0 +1,112 @@ +/** + * @file rti_pqueue_support.h + * @author Soroush Bateni (soroush@berkeley.edu) + * @brief Header-only support functions for pqueue (in the RTI). + * @version 0.1 + * @date 2022-06-02 + * + * @copyright Copyright (c) 2022, The University of California at Berkeley. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +***************/ + +#ifndef RTI_PQUEUE_SUPPORT_H +#define RTI_PQUEUE_SUPPORT_H + +#include "tag.h" +#include "utils/pqueue.h" +#include "utils/util.h" + +// ********** Priority Queue Support Start +/** + * @brief Represent an in-transit message. + * + */ +typedef struct in_transit_message_record { + tag_t tag; // Tag of the in-transit message. + size_t pos; // Position in the priority queue. +} in_transit_message_record_t; + + +/** + * Return whether the first and second argument are given in reverse order. + */ +static int in_reverse_order(pqueue_pri_t thiz, pqueue_pri_t that) { + return (thiz > that); +} + +/** + * Return false (0) regardless of tag order. + */ +static int in_no_particular_order(pqueue_pri_t thiz, pqueue_pri_t that) { + return false; +} + +/** + * Return whether or not the given `in_transit_message_record_t` types have the same tag. + */ +static int tags_match(void* next, void* curr) { + return (lf_tag_compare( + ((in_transit_message_record_t*)next)->tag, + ((in_transit_message_record_t*)curr)->tag + ) == 0); +} + +/** + * Report a priority equal to the time of the given in-transit message. + * Used for sorting pointers to in_transit_message_record_t structs. + */ +static pqueue_pri_t get_message_record_index(void *a) { + return (pqueue_pri_t)(((in_transit_message_record_t*) a)->tag.time); +} + +/** + * Return the given in_transit_message_record_t's position in the queue. + */ +static size_t get_message_record_position(void *a) { + return ((in_transit_message_record_t*) a)->pos; +} + +/** + * Set the given in_transit_message_record_t's position in the queue. + */ +static void set_message_record_position(void *a, size_t pos) { + ((in_transit_message_record_t*) a)->pos = pos; +} + +/** + * Print some information about the given in-transit message. + * + * DEBUG function only. + */ +static void print_message_record(void *message) { + in_transit_message_record_t *r = (in_transit_message_record_t*)message; + LF_PRINT_DEBUG( + "Tag of the in_transit_message_record_t: (%ld, %u). " + "Its position in the priority queue: %u", + r->tag.time - lf_time_start(), + r->tag.microstep, + r->pos + ); +} + +// ********** Priority Queue Support End +#endif diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index c181a4168..8397160c5 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -518,12 +518,7 @@ bool send_advance_grant_if_safe(federate_t* fed) { LF_PRINT_LOG("Earliest next event upstream has tag (%lld, %u).", t_d.time - start_time, t_d.microstep); - if (lf_tag_compare(t_d, FOREVER_TAG) == 0) { - // Upstream federates are all done. - LF_PRINT_LOG("Upstream federates are all done. Granting tag advance."); - send_tag_advance_grant(fed, FOREVER_TAG); - return true; - } else if ( + if ( lf_tag_compare(t_d, fed->next_event) > 0 // The federate has something to do. && lf_tag_compare(t_d, fed->last_provisionally_granted) >= 0 // The grant is not redundant // (equal is important to override any previous @@ -578,6 +573,9 @@ void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { /** * @brief Update the next event tag of federate `federate_id`. * + * It will update the recorded next event tag of federate `federate_id` to the minimum of `next_event_tag` and the + * minimum tag of in-transit messages (if any) to the federate. + * * Will try to see if the RTI can grant new TAG or PTAG messages to any * downstream federates based on this new next event tag. * @@ -587,7 +585,23 @@ void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { * @param next_event_tag The next event tag for `federate_id`. */ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { - _RTI.federates[federate_id].next_event = next_event_tag; + tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(_RTI.federates[federate_id].in_transit_message_tags); + if (lf_tag_compare( + min_in_transit_tag, + next_event_tag + ) < 0 + ) { + next_event_tag = min_in_transit_tag; + } + + _RTI.federates[federate_id].next_event = next_event_tag; + + LF_PRINT_DEBUG( + "RTI: Updated the recorded next event tag for federate %d to (%ld, %u).", + federate_id, + next_event_tag.time - lf_time_start(), + next_event_tag.microstep + ); // Check to see whether we can reply now with a time advance grant. // If the federate has no upstream federates, then it does not wait for @@ -744,6 +758,34 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { length ); + // Record this in-transit message in federate's in-transit message queue. + if (lf_tag_compare(_RTI.federates[federate_id].completed, intended_tag) < 0) { + // Add a record of this message to the list of in-transit messages to this federate. + add_in_transit_message_record( + _RTI.federates[federate_id].in_transit_message_tags, + intended_tag + ); + LF_PRINT_DEBUG( + "RTI: Adding a message with tag (%ld, %u) to the list of in-transit messages for federate %d.", + intended_tag.time - lf_time_start(), + intended_tag.microstep, + federate_id + ); + } else { + lf_print_error( + "RTI: Federate %d has already completed tag (%ld, %u) " + "but there is an in-transit message with tag (%ld, %u) from federate %d. " + "This is going to cause an STP violation under centralized coordination.", + federate_id, + _RTI.federates[federate_id].completed.time - lf_time_start(), + _RTI.federates[federate_id].completed.microstep, + intended_tag.time - lf_time_start(), + intended_tag.microstep, + sending_federate->id + ); + // FIXME: Drop the federate? + } + // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. while (_RTI.federates[federate_id].state == PENDING) { @@ -774,16 +816,8 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { write_to_socket_errexit(destination_socket, bytes_to_read, buffer, "RTI failed to send message chunks."); } - - // If the destination federate has previously sent a NET that is larger - // than the intended tag of this message, then reset that NET to be equal - // to the intended tag of this message. This is needed because the NET - // is a promise that is valid only in the absence of network inputs, - // and now there is a network input. Hence, the promise needs to be - // updated. - if (lf_tag_compare(_RTI.federates[federate_id].next_event, intended_tag) >= 0) { - update_federate_next_event_tag_locked(federate_id, intended_tag); - } + + update_federate_next_event_tag_locked(federate_id, intended_tag); pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -810,6 +844,9 @@ void handle_logical_tag_complete(federate_t* fed) { LF_PRINT_LOG("RTI received from federate %d the Logical Tag Complete (LTC) (%lld, %u).", fed->id, fed->completed.time - start_time, fed->completed.microstep); + // See if we can remove any of the recorded in-transit messages for this. + clean_in_transit_message_record_up_to_tag(fed->in_transit_message_tags, fed->completed); + // Check downstream federates to see whether they should now be granted a TAG. for (int i = 0; i < fed->num_downstream; i++) { federate_t* downstream = &_RTI.federates[fed->downstream[i]]; @@ -838,35 +875,17 @@ void handle_next_event_tag(federate_t* fed) { pthread_mutex_lock(&_RTI.rti_mutex); // FIXME: Instead of using a mutex, // it might be more efficient to use a // select() mechanism to read and process - // federates' buffers in an orderly fashion - - // If the fed->next_event is smaller than this NET message, we need to - // verify that the federate has already completed the previously promised - // fed->next_event. Otherwise, we ignore this NET. This scenario can happen - // when there is a message in-transit to the federate, and we have recorded - // the tag of that in-transit message as the federate's next event, but the - // message hasn't yet been delivered. - tag_t intended_tag = extract_tag(buffer); - if (lf_tag_compare(intended_tag, fed->next_event) <= 0 || - lf_tag_compare(fed->completed, fed->next_event) >= 0) { - + // federates' buffers in an orderly fashion. - update_federate_next_event_tag_locked(fed->id, intended_tag); - LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%ld, %u).", - fed->id, fed->next_event.time - start_time, - fed->next_event.microstep); - } else { - lf_print_error("RTI received from federate %d the Next Event Tag (NET) (%ld, %u), " - "but it is ignoring it since there is a message still in transit to federate %d. " - "Current recorded NET: (%ld, %u), LTC: (%ld, %u).", - fed->id, intended_tag.time - lf_time_start(), intended_tag.microstep, - fed->id, - fed->next_event.time - lf_time_start(), - fed->next_event.microstep, - fed->completed.time - lf_time_start(), - fed->completed.microstep); - } + tag_t intended_tag = extract_tag(buffer); + LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) (%ld, %u).", + fed->id, fed->next_event.time - start_time, + fed->next_event.microstep); + update_federate_next_event_tag_locked( + fed->id, + intended_tag + ); pthread_mutex_unlock(&_RTI.rti_mutex); } @@ -1879,6 +1898,7 @@ void initialize_federate(uint16_t id) { _RTI.federates[id].last_granted = NEVER_TAG; _RTI.federates[id].last_provisionally_granted = NEVER_TAG; _RTI.federates[id].next_event = NEVER_TAG; + _RTI.federates[id].in_transit_message_tags = initialize_in_transit_message_q(); _RTI.federates[id].state = NOT_CONNECTED; _RTI.federates[id].upstream = NULL; _RTI.federates[id].upstream_delay = NULL; @@ -1941,6 +1961,7 @@ void wait_for_federates(int socket_descriptor) { for (int i = 0; i < _RTI.number_of_federates; i++) { lf_print("RTI: Waiting for thread handling federate %d.", _RTI.federates[i].id); pthread_join(_RTI.federates[i].thread_id, &thread_exit_status); + free_in_transit_message_q(_RTI.federates[i].in_transit_message_tags); lf_print("RTI: Federate %d thread exited.", _RTI.federates[i].id); } diff --git a/core/federated/RTI/rti.h b/core/federated/RTI/rti.h index 454514808..f62162603 100644 --- a/core/federated/RTI/rti.h +++ b/core/federated/RTI/rti.h @@ -35,6 +35,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define RTI_H #include "reactor.h" +#include "message_record/message_record.h" ///////////////////////////////////////////// //// Data structures @@ -76,6 +77,11 @@ typedef struct federate_t { tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). + in_transit_message_record_q_t in_transit_message_tags; // Record of in-transit messages to this federate that are not + // yet processed. Messages are assumed to be processed once an + // LTC arrives that has the same or larger tag. Since messages + // are assumed to be delivered in-order, there is no need to + // sort this. fed_state_t state; // State of the federate. int* upstream; // Array of upstream federate ids. interval_t* upstream_delay; // Minimum delay on connections from upstream federates. diff --git a/core/utils/pqueue_support.h b/core/utils/pqueue_support.h index cc903adb7..7703a8ede 100644 --- a/core/utils/pqueue_support.h +++ b/core/utils/pqueue_support.h @@ -140,4 +140,4 @@ static void print_event(void *event) { } // ********** Priority Queue Support End -#endif \ No newline at end of file +#endif From f26d3422fd863fab4c5222f16cee9564567c2f43 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Thu, 2 Jun 2022 16:15:53 -0500 Subject: [PATCH 28/37] Comments only --- .../RTI/message_record/message_record.c | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/core/federated/RTI/message_record/message_record.c b/core/federated/RTI/message_record/message_record.c index 05e726eba..252c0eee2 100644 --- a/core/federated/RTI/message_record/message_record.c +++ b/core/federated/RTI/message_record/message_record.c @@ -1,13 +1,32 @@ /** * @file message_record.c - * @author your name (you@domain.com) - * @brief + * @author Soroush Bateni (soroush@berkeley.edu) + * @brief Record-keeping for in-transit messages. * @version 0.1 * @date 2022-06-02 * - * @copyright Copyright (c) 2022 - * - */ + * @copyright Copyright (c) 2022, The University of California at Berkeley. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +***************/ #include "message_record.h" #include From f46abe879c882dc8cb04553c1fb9d5c661c62c8f Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Thu, 2 Jun 2022 16:20:52 -0500 Subject: [PATCH 29/37] Comments only --- .../RTI/message_record/message_record.c | 16 ++++++++-------- .../RTI/message_record/message_record.h | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/federated/RTI/message_record/message_record.c b/core/federated/RTI/message_record/message_record.c index 252c0eee2..0557bda84 100644 --- a/core/federated/RTI/message_record/message_record.c +++ b/core/federated/RTI/message_record/message_record.c @@ -79,7 +79,7 @@ void free_in_transit_message_q(in_transit_message_record_q_t queue) { /** * @brief Add a record of the in-transit message. * - * @param queue The queue to add to (of type `in_transit_message_record_q_t`. + * @param queue The queue to add to (of type `in_transit_message_record_q_t`). * @param tag The tag of the in-transit message. * @return 0 on success. */ @@ -95,7 +95,7 @@ int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag /** * @brief Clean the record of in-transit messages up to and including `tag`. * - * @param queue The queue to clean (of type `in_transit_message_record_q_t`. + * @param queue The queue to clean (of type `in_transit_message_record_q_t`). * @param tag Will clean all messages with tags <= tag. */ void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t queue, tag_t tag) { @@ -148,12 +148,6 @@ tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t queue) { ) <= 0 ) { minimum_tag = head_of_in_transit_messages->tag; - - LF_PRINT_DEBUG( - "RTI: Removed a message with tag (%ld, %u) from the list of in-transit messages.", - head_of_in_transit_messages->tag.time - lf_time_start(), - head_of_in_transit_messages->tag.microstep - ); } else if (head_of_in_transit_messages->tag.time > minimum_tag.time) { break; } @@ -166,5 +160,11 @@ tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t queue) { // Empty the transfer queue (which holds messages with equal time but larger microstep) into the main queue. pqueue_empty_into(&queue->main_queue, &queue->transfer_queue); + LF_PRINT_DEBUG( + "RTI: Minimum tag of all in-transit messages: (%ld, %u).", + head_of_in_transit_messages->tag.time - lf_time_start(), + head_of_in_transit_messages->tag.microstep + ); + return minimum_tag; } diff --git a/core/federated/RTI/message_record/message_record.h b/core/federated/RTI/message_record/message_record.h index 22276a43a..afb996841 100644 --- a/core/federated/RTI/message_record/message_record.h +++ b/core/federated/RTI/message_record/message_record.h @@ -61,7 +61,7 @@ void free_in_transit_message_q(in_transit_message_record_q_t queue); /** * @brief Add a record of the in-transit message. * - * @param queue The queue to add to (of type `in_transit_message_record_q`. + * @param queue The queue to add to (of type `in_transit_message_record_q`). * @param tag The tag of the in-transit message. * @return 0 on success. */ @@ -70,7 +70,7 @@ int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag /** * @brief Clean the record of in-transit messages up to and including `tag`. * - * @param queue The queue to clean (of type `in_transit_message_record_q`. + * @param queue The queue to clean (of type `in_transit_message_record_q`). * @param tag Will clean all messages with tags <= tag. */ void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t queue, tag_t tag); From f257a3bfd534a0e5037d8748da7d751f9ab77320 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 3 Jun 2022 15:38:15 -0500 Subject: [PATCH 30/37] Updated LF ref --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 4e92adc37..612d1f506 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -9e12bc5a2dea2dac988aeb7a0fce3ec4e07c5e81 +b7bccaf76f4e6f6af1d214ddf1211368a8806384 From a228fb3507c51066c2a3de9991d0f869cb6dd075 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 3 Jun 2022 15:41:08 -0500 Subject: [PATCH 31/37] Updated LF ref --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 612d1f506..7f9108cd1 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -b7bccaf76f4e6f6af1d214ddf1211368a8806384 +a6109d42118c6da35d8d485a766fc3e3e3521f8a From eaec0e6e1417531bb575f8aaf61aa6eba2909d01 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 3 Jun 2022 16:41:09 -0500 Subject: [PATCH 32/37] Fixed issue where a dummy event could be created at FOREVER --- core/federated/federate.c | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/core/federated/federate.c b/core/federated/federate.c index 2136a5436..6fc9ed3b4 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -2041,16 +2041,18 @@ void handle_provisional_tag_advance_grant() { dummy_event_relative_microstep -= current_tag.microstep; } // We now know current_tag < PTAG. - // Schedule a dummy event at the specified time and (relative) microstep. - LF_PRINT_DEBUG("At tag (%lld, %d), inserting into the event queue a dummy event " - "with time %lld and (relative) microstep %d.", - current_tag.time - start_time, current_tag.microstep, - dummy_event_time - start_time, dummy_event_relative_microstep); - // Dummy event points to a NULL trigger and NULL real event. - event_t* dummy = _lf_create_dummy_events( - NULL, dummy_event_time, NULL, dummy_event_relative_microstep); - pqueue_insert(event_q, dummy); + if (dummy_event_time != FOREVER) { + // Schedule a dummy event at the specified time and (relative) microstep. + LF_PRINT_DEBUG("At tag (%lld, %d), inserting into the event queue a dummy event " + "with time %lld and (relative) microstep %d.", + current_tag.time - start_time, current_tag.microstep, + dummy_event_time - start_time, dummy_event_relative_microstep); + // Dummy event points to a NULL trigger and NULL real event. + event_t* dummy = _lf_create_dummy_events( + NULL, dummy_event_time, NULL, dummy_event_relative_microstep); + pqueue_insert(event_q, dummy); + } lf_mutex_unlock(&mutex); } @@ -2234,6 +2236,7 @@ void terminate_execution() { } lf_mutex_unlock(&outbound_socket_mutex); + LF_PRINT_DEBUG("Requesting closing of incoming P2P sockets."); // Request closing the incoming P2P sockets. for (int i=0; i < NUMBER_OF_FEDERATES; i++) { if (_lf_request_close_inbound_socket(i) == 0) { @@ -2242,6 +2245,7 @@ void terminate_execution() { } } + LF_PRINT_DEBUG("Waiting for inbound p2p socket listener threads."); // Wait for each inbound socket listener thread to close. if (_fed.number_of_inbound_p2p_connections > 0) { LF_PRINT_LOG("Waiting for %d threads listening for incoming messages to exit.", @@ -2252,9 +2256,11 @@ void terminate_execution() { } } + LF_PRINT_DEBUG("Waiting for RTI's socket listener threads."); // Wait for the thread listening for messages from the RTI to close. lf_thread_join(_fed.RTI_socket_listener, NULL); + LF_PRINT_DEBUG("Freeing memory occupied by the federate."); free(_fed.inbound_socket_listeners); free(federation_metadata.rti_host); free(federation_metadata.rti_user); @@ -2649,10 +2655,12 @@ tag_t _lf_send_next_event_tag(tag_t tag, bool wait_for_reply) { } } - // Create a dummy event that will force this federate to advance time and subsequently enable progress for - // downstream federates. - event_t* dummy = _lf_create_dummy_events(NULL, tag.time, NULL, 0); - pqueue_insert(event_q, dummy); + if (tag.time != FOREVER) { + // Create a dummy event that will force this federate to advance time and subsequently enable progress for + // downstream federates. + event_t* dummy = _lf_create_dummy_events(NULL, tag.time, NULL, 0); + pqueue_insert(event_q, dummy); + } LF_PRINT_DEBUG("Inserted a dummy event for tag (%ld, %u).", tag.time - lf_time_start()); From b2baf961209db8deeeca96d9eca240450f777e37 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 10 Jun 2022 09:42:27 -0500 Subject: [PATCH 33/37] Comments only --- core/federated/RTI/rti.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/federated/RTI/rti.h b/core/federated/RTI/rti.h index f62162603..be708c61e 100644 --- a/core/federated/RTI/rti.h +++ b/core/federated/RTI/rti.h @@ -78,10 +78,8 @@ typedef struct federate_t { tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). in_transit_message_record_q_t in_transit_message_tags; // Record of in-transit messages to this federate that are not - // yet processed. Messages are assumed to be processed once an - // LTC arrives that has the same or larger tag. Since messages - // are assumed to be delivered in-order, there is no need to - // sort this. + // yet processed. This record is ordered based on the time + // value of each message for a more efficient access. fed_state_t state; // State of the federate. int* upstream; // Array of upstream federate ids. interval_t* upstream_delay; // Minimum delay on connections from upstream federates. From f05be58b838141ab2398b1f855fd8dd165a42e89 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 10 Jun 2022 12:36:18 -0500 Subject: [PATCH 34/37] Address comment from @edwardalee --- .../RTI/message_record/message_record.c | 20 +++++++++---------- .../RTI/message_record/message_record.h | 16 +++++++-------- core/federated/RTI/rti.h | 2 +- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/core/federated/RTI/message_record/message_record.c b/core/federated/RTI/message_record/message_record.c index 0557bda84..6d4c278d4 100644 --- a/core/federated/RTI/message_record/message_record.c +++ b/core/federated/RTI/message_record/message_record.c @@ -36,11 +36,11 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * @return in_transit_message_record_q */ -in_transit_message_record_q_t initialize_in_transit_message_q() { - in_transit_message_record_q_t queue = - (in_transit_message_record_q_t)calloc( +in_transit_message_record_q_t* initialize_in_transit_message_q() { + in_transit_message_record_q_t* queue = + (in_transit_message_record_q_t*)calloc( 1, - sizeof(struct in_transit_message_record_q) + sizeof(in_transit_message_record_q_t) ); queue->main_queue = pqueue_init( 10, @@ -70,7 +70,7 @@ in_transit_message_record_q_t initialize_in_transit_message_q() { * * @param queue The queue to free. */ -void free_in_transit_message_q(in_transit_message_record_q_t queue) { +void free_in_transit_message_q(in_transit_message_record_q_t* queue) { pqueue_free(queue->main_queue); pqueue_free(queue->transfer_queue); free(queue); @@ -79,11 +79,11 @@ void free_in_transit_message_q(in_transit_message_record_q_t queue) { /** * @brief Add a record of the in-transit message. * - * @param queue The queue to add to (of type `in_transit_message_record_q_t`). + * @param queue The queue to add to. * @param tag The tag of the in-transit message. * @return 0 on success. */ -int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag) { +int add_in_transit_message_record(in_transit_message_record_q_t* queue, tag_t tag) { in_transit_message_record_t* in_transit_record = malloc(sizeof(in_transit_message_record_t)); in_transit_record->tag = tag; return pqueue_insert( @@ -95,10 +95,10 @@ int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag /** * @brief Clean the record of in-transit messages up to and including `tag`. * - * @param queue The queue to clean (of type `in_transit_message_record_q_t`). + * @param queue The queue to clean. * @param tag Will clean all messages with tags <= tag. */ -void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t queue, tag_t tag) { +void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t* queue, tag_t tag) { in_transit_message_record_t* head_of_in_transit_messages = (in_transit_message_record_t*)pqueue_peek(queue->main_queue); while ( head_of_in_transit_messages != NULL && // Queue is not empty @@ -135,7 +135,7 @@ void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t que * @param queue The queue to search in (of type `in_transit_message_record_q`). * @return tag_t The minimum tag of all currently recorded in-transit messages. Return `FOREVER_TAG` if the queue is empty. */ -tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t queue) { +tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t* queue) { tag_t minimum_tag = FOREVER_TAG; in_transit_message_record_t* head_of_in_transit_messages = (in_transit_message_record_t*)pqueue_peek(queue->main_queue); diff --git a/core/federated/RTI/message_record/message_record.h b/core/federated/RTI/message_record/message_record.h index afb996841..b70e4ac1f 100644 --- a/core/federated/RTI/message_record/message_record.h +++ b/core/federated/RTI/message_record/message_record.h @@ -37,26 +37,24 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * @brief Queue to keep a record of in-transit messages. * */ -struct in_transit_message_record_q { +typedef struct { pqueue_t* main_queue; // The primary queue. pqueue_t* transfer_queue; // Queue used for housekeeping. -}; - -typedef struct in_transit_message_record_q* in_transit_message_record_q_t; +} in_transit_message_record_q_t; /** * @brief Initialize the in-transit message record queue. * * @return in_transit_message_record_q */ -in_transit_message_record_q_t initialize_in_transit_message_q(); +in_transit_message_record_q_t* initialize_in_transit_message_q(); /** * @brief Free the memory occupied by the `queue`. * * @param queue The queue to free. */ -void free_in_transit_message_q(in_transit_message_record_q_t queue); +void free_in_transit_message_q(in_transit_message_record_q_t* queue); /** * @brief Add a record of the in-transit message. @@ -65,7 +63,7 @@ void free_in_transit_message_q(in_transit_message_record_q_t queue); * @param tag The tag of the in-transit message. * @return 0 on success. */ -int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag); +int add_in_transit_message_record(in_transit_message_record_q_t* queue, tag_t tag); /** * @brief Clean the record of in-transit messages up to and including `tag`. @@ -73,7 +71,7 @@ int add_in_transit_message_record(in_transit_message_record_q_t queue, tag_t tag * @param queue The queue to clean (of type `in_transit_message_record_q`). * @param tag Will clean all messages with tags <= tag. */ -void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t queue, tag_t tag); +void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t* queue, tag_t tag); /** * @brief Get the minimum tag of all currently recorded in-transit messages. @@ -81,6 +79,6 @@ void clean_in_transit_message_record_up_to_tag(in_transit_message_record_q_t que * @param queue The queue to search in (of type `in_transit_message_record_q`). * @return tag_t The minimum tag of all currently recorded in-transit messages. Return `FOREVER_TAG` if the queue is empty. */ -tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t queue); +tag_t get_minimum_in_transit_message_tag(in_transit_message_record_q_t* queue); #endif // RTI_MESSAGE_RECORD_H diff --git a/core/federated/RTI/rti.h b/core/federated/RTI/rti.h index be708c61e..ca609c5d5 100644 --- a/core/federated/RTI/rti.h +++ b/core/federated/RTI/rti.h @@ -77,7 +77,7 @@ typedef struct federate_t { tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). - in_transit_message_record_q_t in_transit_message_tags; // Record of in-transit messages to this federate that are not + in_transit_message_record_q_t* in_transit_message_tags; // Record of in-transit messages to this federate that are not // yet processed. This record is ordered based on the time // value of each message for a more efficient access. fed_state_t state; // State of the federate. From 5ea530346b7a7209e8a5f760e1189d54f7758683 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 10 Jun 2022 12:38:25 -0500 Subject: [PATCH 35/37] Updated LF ref --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 7f9108cd1..841306532 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -a6109d42118c6da35d8d485a766fc3e3e3521f8a +d70bc48df07bc533dd7d10210e77a2c116c056a0 From 445b2c8195543e1126c2a40bc2cfdccbc9a86138 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 10 Jun 2022 16:22:05 -0500 Subject: [PATCH 36/37] Updated LF ref --- lingua-franca-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lingua-franca-ref.txt b/lingua-franca-ref.txt index 841306532..fc1a5ef62 100644 --- a/lingua-franca-ref.txt +++ b/lingua-franca-ref.txt @@ -1 +1 @@ -d70bc48df07bc533dd7d10210e77a2c116c056a0 +544168274369b9d3893695846171bd2ed43d4682 From 47c70ed3e835a8363a1b97d05fc1ea66c9e66b43 Mon Sep 17 00:00:00 2001 From: Soroush Bateni Date: Fri, 10 Jun 2022 19:08:37 -0500 Subject: [PATCH 37/37] Fixed bizarre merge artifact --- core/federated/federate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/core/federated/federate.c b/core/federated/federate.c index f4479bfba..b226a28df 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -1828,6 +1828,7 @@ void handle_tagged_message(int socket, int fed_id) { intended_tag.time - lf_time_start(), intended_tag.microstep ); + action->intended_tag = intended_tag; _lf_insert_reactions_for_trigger(action, message_token); // Set the status of the port as present here to inform the network input