From 85c5f1ce07955878f12a8c214efb821b806ad21d Mon Sep 17 00:00:00 2001
From: Nikhil Jain <nikhil.jain@acm.org>
Date: Fri, 28 Jul 2017 22:20:29 -0700
Subject: [PATCH] Prevent forward events from overtaking cancel events.

Add compile time option to stop sending of events from a PE if a cancel event is
being held for its corresponding forward event from being sent.

cmake option: PRIORITIZE_CANCEL_EVENTS
default value: OFF

This is still no fool-proof because on the receive side, MPI_Testsome may
succeed for events sent later. However, for small messages that ROSS usually
sends (<1 KB), this is unlikely.

Change-Id: I2800a2a6d3537317c6182047ff0beeb3ba238037
---
 CMakeLists.txt     |  6 ++++++
 core/network-mpi.c | 11 +++++++++++
 2 files changed, 17 insertions(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e292b5292..4aff7446a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -99,6 +99,12 @@ IF(NOT VALID_ARCH)
     "Please set ARCH to be one of (i386 | bgl | bgp | x86_64 | ppc64)")
 ENDIF(NOT VALID_ARCH)
 
+# Prevent forward events from overtaking cancel events
+OPTION(PRIORITIZE_CANCEL_EVENTS "Disable second forward event from overtaking the cancel event?" OFF)
+IF(PRIORITIZE_CANCEL_EVENTS)
+  ADD_DEFINITIONS(-DPRIORITIZE_CANCEL_EVENTS=1)
+ENDIF(PRIORITIZE_CANCEL_EVENTS)
+
 # ROSS Core code
 ADD_SUBDIRECTORY(core)
 
diff --git a/core/network-mpi.c b/core/network-mpi.c
index 73beb4590..1d5f195c4 100644
--- a/core/network-mpi.c
+++ b/core/network-mpi.c
@@ -515,6 +515,12 @@ send_begin(tw_pe *me)
       if (!e)
 	break;
 
+#if PRIORITIZE_CANCEL_EVENTS
+      if (e->state.cancel_asend) {
+        break;
+      }
+#endif
+
       if(e == me->abort_event)
 	tw_error(TW_LOC, "Sending abort event!");
 
@@ -628,7 +634,9 @@ send_finish(tw_pe *me, tw_event *e, char * buffer)
        */
       e->state.cancel_asend = 0;
       e->state.cancel_q = 1;
+#if !PRIORITIZE_CANCEL_EVENTS
       tw_eventq_push(&outq, e);
+#endif
     } else {
       /* Event finished transmission and was not cancelled.
        * Add to our sent event queue so we can retain the
@@ -735,6 +743,9 @@ tw_net_cancel(tw_event *e)
      * this message is completed.
      */
     e->state.cancel_asend = 1;
+#if PRIORITIZE_CANCEL_EVENTS
+    tw_eventq_unshift(&outq, e);
+#endif
     break;
 
   case TW_pe_sevent_q: