Index: exec/cpg.c
===================================================================
--- exec/cpg.c	(revision 2043)
+++ exec/cpg.c	(revision 2044)
@@ -128,6 +128,11 @@
 	CPD_STATE_JOIN_COMPLETED
 };
 
+enum cpg_sync_state {
+	CPGSYNC_DOWNLIST,
+	CPGSYNC_JOINLIST
+};
+
 struct cpg_pd {
 	void *conn;
 	mar_cpg_name_t group_name;
@@ -135,8 +140,19 @@
 	enum cpd_state cpd_state;
 	struct list_head list;
 };
+
 DECLARE_LIST_INIT(cpg_pd_list_head);
 
+static unsigned int my_member_list[PROCESSOR_COUNT_MAX];
+
+static unsigned int my_member_list_entries;
+
+static unsigned int my_old_member_list[PROCESSOR_COUNT_MAX];
+
+static unsigned int my_old_member_list_entries = 0;
+
+static enum cpg_sync_state my_sync_state = CPGSYNC_DOWNLIST;
+
 struct process_info {
 	unsigned int nodeid;
 	uint32_t pid;
@@ -206,6 +222,8 @@
 
 static int cpg_node_joinleave_send (unsigned int pid, mar_cpg_name_t *group_name, int fn, int reason);
 
+static int cpg_exec_send_downlist(void);
+
 static int cpg_exec_send_joinlist(void);
 
 static void cpg_sync_init (void);
@@ -363,12 +381,26 @@
 
 static int cpg_sync_process (void)
 {
-	return cpg_exec_send_joinlist();
+	int res = -1;
+
+	if (my_sync_state == CPGSYNC_DOWNLIST) {
+		res = cpg_exec_send_downlist();
+		if (res == -1) {
+			return (-1);
+		}
+		my_sync_state = CPGSYNC_JOINLIST;
+	}
+	if (my_sync_state == CPGSYNC_JOINLIST) {
+		res = cpg_exec_send_joinlist();
+	}
+	return (res);
 }
 
 static void cpg_sync_activate (void)
 {
-
+	memcpy (my_old_member_list, my_member_list,
+		my_member_list_entries * sizeof (unsigned int));
+	my_old_member_list_entries = my_member_list_entries;
 }
 static void cpg_sync_abort (void)
 {
@@ -537,45 +569,46 @@
 	unsigned int *joined_list, int joined_list_entries,
 	struct memb_ring_id *ring_id)
 {
-	int i;
-	uint32_t lowest_nodeid = 0xffffffff;
-	struct iovec req_exec_cpg_iovec;
+	unsigned int lowest_nodeid = 0xffffffff;
+	int entries;
+	int i, j;
+	int found;
 
-	/* We don't send the library joinlist in here because it can end up
-	   out of order with the rest of the messages (which are totem ordered).
-	   So we get the lowest nodeid to send out a list of left nodes instead.
-	   On receipt of that message, all nodes will then notify their local clients
-	   of the new joinlist */
+	/* Don't send this message until we get the final configuration message */
+	if (configuration_type == TOTEM_CONFIGURATION_REGULAR) {
+		my_sync_state = CPGSYNC_DOWNLIST;
 
-	if (left_list_entries) {
-		for (i = 0; i < member_list_entries; i++) {
-			if (member_list[i] < lowest_nodeid)
-				lowest_nodeid = member_list[i];
+		memcpy (my_member_list, member_list, member_list_entries *
+			sizeof (unsigned int));
+		my_member_list_entries = member_list_entries;
+
+		for (i = 0; i < my_member_list_entries; i++) {
+			if (my_member_list[i] < lowest_nodeid) {
+				lowest_nodeid = my_member_list[i];
+			}
 		}
 
-		log_printf(LOG_LEVEL_DEBUG, "confchg, low nodeid=%d, us = %d\n", lowest_nodeid, totempg_my_nodeid_get());
+		entries = 0;
 		if (lowest_nodeid == totempg_my_nodeid_get()) {
-
-			req_exec_cpg_downlist.header.id = SERVICE_ID_MAKE(CPG_SERVICE, MESSAGE_REQ_EXEC_CPG_DOWNLIST);
-			req_exec_cpg_downlist.header.size = sizeof(struct req_exec_cpg_downlist);
-
-			req_exec_cpg_downlist.left_nodes = left_list_entries;
-			for (i = 0; i < left_list_entries; i++) {
-				req_exec_cpg_downlist.nodeids[i] = left_list[i];
+			/*
+			 * Determine list of nodeids for downlist message
+			 */
+			for (i = 0; i < my_old_member_list_entries; i++) {
+				found = 0;
+				for (j = 0; j < my_member_list_entries; j++) {
+					if (my_old_member_list[i] == my_member_list[j]) {
+						found = 1;
+						break;
+					}
+				}
+				if (found == 0) {
+					req_exec_cpg_downlist.nodeids[entries++] =
+						my_old_member_list[i];
+				}
 			}
-			log_printf(LOG_LEVEL_DEBUG, "confchg, build downlist: %d nodes\n", left_list_entries);
 		}
+		req_exec_cpg_downlist.left_nodes = entries;
 	}
-
-	/* Don't send this message until we get the final configuration message */
-	if (configuration_type == TOTEM_CONFIGURATION_REGULAR && req_exec_cpg_downlist.left_nodes) {
-		req_exec_cpg_iovec.iov_base = (char *)&req_exec_cpg_downlist;
-		req_exec_cpg_iovec.iov_len = req_exec_cpg_downlist.header.size;
-
-		totempg_groups_mcast_joined (openais_group_handle, &req_exec_cpg_iovec, 1, TOTEMPG_AGREED);
-		req_exec_cpg_downlist.left_nodes = 0;
-		log_printf(LOG_LEVEL_DEBUG, "confchg, sent downlist\n");
-	}
 }
 
 /* Can byteswap join & leave messages */
@@ -837,6 +870,19 @@
 }
 
 
+static int cpg_exec_send_downlist(void)
+{
+	struct iovec iov;
+
+	req_exec_cpg_downlist.header.id = SERVICE_ID_MAKE(CPG_SERVICE, MESSAGE_REQ_EXEC_CPG_DOWNLIST);
+	req_exec_cpg_downlist.header.size = sizeof(struct req_exec_cpg_downlist);
+
+	iov.iov_base = (void *)&req_exec_cpg_downlist;
+	iov.iov_len = req_exec_cpg_downlist.header.size;
+
+	return totempg_groups_mcast_joined (openais_group_handle, &iov, 1, TOTEMPG_AGREED);
+}
+
 static int cpg_exec_send_joinlist(void)
 {
 	int count = 0;