From: jbeulich@novell.com
Subject: netback: use multicall for send multiple notifications
Patch-mainline: Never, SUSE-Xen specific

This also yields a small fairness improvement since now notifications
get sent in the order requests came in rather than in the inverse one.

--- a/drivers/xen/core/evtchn.c
+++ b/drivers/xen/core/evtchn.c
@@ -1535,6 +1535,27 @@ void notify_remote_via_irq(int irq)
 }
 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
 
+#if defined(CONFIG_XEN_BACKEND) || defined(CONFIG_XEN_BACKEND_MODULE)
+int multi_notify_remote_via_irq(multicall_entry_t *mcl, int irq)
+{
+	const struct irq_cfg *cfg = irq_cfg(irq);
+	unsigned int evtchn;
+
+	if (WARN_ON_ONCE(!cfg))
+		return -EINVAL;
+	BUG_ON(type_from_irq_cfg(cfg) == IRQT_VIRQ);
+	BUG_IF_IPI(cfg);
+
+	evtchn = evtchn_from_irq_cfg(cfg);
+	if (!VALID_EVTCHN(evtchn))
+		return -EINVAL;
+
+	multi_notify_remote_via_evtchn(mcl, evtchn);
+	return 0;
+}
+EXPORT_SYMBOL_GPL(multi_notify_remote_via_irq);
+#endif
+
 int irq_to_evtchn_port(int irq)
 {
 	const struct irq_cfg *cfg = irq_cfg(irq);
--- a/drivers/xen/netback/common.h
+++ b/drivers/xen/netback/common.h
@@ -55,6 +55,12 @@ typedef struct netif_st {
 
 	unsigned int     irq;
 
+	struct netif_st *notify_link[2];
+#define RX_IDX 0
+#define TX_IDX 1
+#define rx_notify_link notify_link[RX_IDX]
+#define tx_notify_link notify_link[TX_IDX]
+
 	/* The shared rings and indexes. */
 	netif_tx_back_ring_t tx;
 	netif_rx_back_ring_t rx;
--- a/drivers/xen/netback/netback.c
+++ b/drivers/xen/netback/netback.c
@@ -71,9 +71,8 @@ struct netbk_tx_pending_inuse {
 };
 
 static void netif_idx_release(u16 pending_idx);
-static void make_tx_response(netif_t *netif, 
-			     netif_tx_request_t *txp,
-			     s8       st);
+static bool make_tx_response(netif_t *, const netif_tx_request_t *, s8 st,
+			     netif_t **);
 static netif_rx_response_t *make_rx_response(netif_t *netif, 
 					     u16      id, 
 					     s8       st,
@@ -167,9 +166,12 @@ static LIST_HEAD(pending_inuse_head);
 static struct sk_buff_head tx_queue;
 
 static grant_handle_t grant_tx_handle[MAX_PENDING_REQS];
-static gnttab_unmap_grant_ref_t tx_unmap_ops[MAX_PENDING_REQS];
-static gnttab_map_grant_ref_t tx_map_ops[MAX_PENDING_REQS];
-static gnttab_copy_t tx_copy_ops[2 * MAX_PENDING_REQS];
+static union {
+	gnttab_map_grant_ref_t map_ops[MAX_PENDING_REQS];
+	gnttab_unmap_grant_ref_t unmap_ops[MAX_PENDING_REQS];
+	gnttab_copy_t copy_ops[2 * MAX_PENDING_REQS];
+	multicall_entry_t mcl[0];
+} tx;
 static netif_tx_request_t tx_slots[XEN_NETIF_NR_SLOTS_MIN];
 
 static struct list_head net_schedule_list;
@@ -223,6 +225,32 @@ static int check_mfn(int nr)
 	return alloc_index >= nr ? 0 : -ENOMEM;
 }
 
+static void flush_notify_list(netif_t *list, unsigned int idx,
+			      multicall_entry_t mcl[],
+			      unsigned int limit)
+{
+	unsigned int used = 0;
+
+	do {
+		netif_t *cur = list;
+
+		list = cur->notify_link[idx];
+		cur->notify_link[idx] = NULL;
+
+		if ((!used && !list) ||
+		    multi_notify_remote_via_irq(mcl + used, cur->irq))
+			notify_remote_via_irq(cur->irq);
+		else if (++used == limit) {
+			if (HYPERVISOR_multicall(mcl, used))
+				BUG();
+			used = 0;
+		}
+		netif_put(cur);
+	} while (list);
+	if (used && HYPERVISOR_multicall(mcl, used))
+		BUG();
+}
+
 static inline void maybe_schedule_tx_action(void)
 {
 	smp_mb();
@@ -684,14 +712,13 @@ static unsigned int netbk_add_frag_respo
 
 static void net_rx_action(unsigned long unused)
 {
-	netif_t *netif = NULL;
+	netif_t *netif, *notify_head = NULL, *notify_tail = NULL;
 	s8 status;
-	u16 id, irq, flags;
+	u16 id, flags;
 	netif_rx_response_t *resp;
 	multicall_entry_t *mcl;
 	struct sk_buff_head rxq;
 	struct sk_buff *skb;
-	int notify_nr = 0;
 	int ret;
 	int nr_frags;
 	int count;
@@ -705,8 +732,6 @@ static void net_rx_action(unsigned long
 	static mmu_update_t rx_mmu[NET_RX_RING_SIZE];
 	static gnttab_transfer_t grant_trans_op[NET_RX_RING_SIZE];
 	static gnttab_copy_t grant_copy_op[2 * NET_RX_RING_SIZE];
-	static unsigned char rx_notify[NR_IRQS];
-	static u16 notify_list[NET_RX_RING_SIZE];
 	static struct netbk_rx_meta meta[NET_RX_RING_SIZE];
 
 	struct netrx_pending_operations npo = {
@@ -854,28 +879,29 @@ static void net_rx_action(unsigned long
 						    nr_frags);
 
 		RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret);
-		irq = netif->irq;
-		if (ret && !rx_notify[irq]) {
-			rx_notify[irq] = 1;
-			notify_list[notify_nr++] = irq;
-		}
 
 		if (netif_queue_stopped(netif->dev) &&
 		    netif_schedulable(netif) &&
 		    !netbk_queue_full(netif))
 			netif_wake_queue(netif->dev);
 
-		netif_put(netif);
+		if (ret && netif != notify_tail && !netif->rx_notify_link) {
+			if (notify_tail)
+				notify_tail->rx_notify_link = netif;
+			else
+				notify_head = netif;
+			notify_tail = netif;
+		} else
+			netif_put(netif);
+
 		dev_kfree_skb(skb);
 
 		npo.meta_cons += nr_frags + 1;
 	}
 
-	while (notify_nr != 0) {
-		irq = notify_list[--notify_nr];
-		rx_notify[irq] = 0;
-		notify_remote_via_irq(irq);
-	}
+	if (notify_head)
+		flush_notify_list(notify_head, RX_IDX, rx_mcl,
+				  ARRAY_SIZE(rx_mcl));
 
 	/* More work to do? */
 	if (!skb_queue_empty(&rx_queue) && !timer_pending(&net_timer))
@@ -1029,11 +1055,11 @@ inline static void net_tx_action_dealloc
 	gnttab_unmap_grant_ref_t *gop;
 	u16 pending_idx;
 	PEND_RING_IDX dc, dp;
-	netif_t *netif;
+	netif_t *netif, *notify_head = NULL, *notify_tail = NULL;
 	LIST_HEAD(list);
 
 	dc = dealloc_cons;
-	gop = tx_unmap_ops;
+	gop = tx.unmap_ops;
 
 	/*
 	 * Free up any grants we have finished using
@@ -1069,7 +1095,7 @@ inline static void net_tx_action_dealloc
 	dealloc_cons = dc;
 
 	if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
-				      tx_unmap_ops, gop - tx_unmap_ops))
+				      tx.unmap_ops, gop - tx.unmap_ops))
 		BUG();
 
 	/* Copy any entries that have been pending for too long. */
@@ -1103,18 +1129,22 @@ inline static void net_tx_action_dealloc
 
 		netif = pending_tx_info[pending_idx].netif;
 
-		make_tx_response(netif, &pending_tx_info[pending_idx].req, 
-				 XEN_NETIF_RSP_OKAY);
+		if (!make_tx_response(netif, &pending_tx_info[pending_idx].req,
+				      XEN_NETIF_RSP_OKAY, &notify_tail))
+			netif_put(netif);
+		else if (!notify_head)
+			notify_head = netif;
 
 		/* Ready for next use. */
 		gnttab_reset_grant_page(mmap_pages[pending_idx]);
 
 		pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
 
-		netif_put(netif);
-
 		list_del_init(&inuse->list);
 	}
+	if (notify_head)
+		flush_notify_list(notify_head, TX_IDX, tx.mcl,
+				  sizeof(tx) / sizeof(tx.mcl[0]));
 }
 
 static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end)
@@ -1122,7 +1152,7 @@ static void netbk_tx_err(netif_t *netif,
 	RING_IDX cons = netif->tx.req_cons;
 
 	do {
-		make_tx_response(netif, txp, XEN_NETIF_RSP_ERROR);
+		make_tx_response(netif, txp, XEN_NETIF_RSP_ERROR, NULL);
 		if (cons == end)
 			break;
 		txp = RING_GET_REQUEST(&netif->tx, cons++);
@@ -1228,7 +1258,12 @@ static int netbk_count_requests(netif_t
 struct netbk_tx_gop {
 	gnttab_map_grant_ref_t *map;
 	gnttab_copy_t *copy;
-	void *ptr;
+	union {
+		void *ptr;
+		struct {
+			netif_t *head, *tail;
+		} notify;
+	};
 };
 
 static void netbk_fill_tx_copy(const netif_tx_request_t *txreq,
@@ -1322,15 +1357,18 @@ static int netbk_tx_check_gop(struct sk_
 		txp = &pending_tx_info[pending_idx].req;
 		if (txp->size > cop->len)
 			cmpxchg_local(&err, GNTST_okay, (--cop)->status);
-		make_tx_response(netif, txp,
-				 err == GNTST_okay ? XEN_NETIF_RSP_OKAY
-						   : XEN_NETIF_RSP_ERROR);
+		if (!make_tx_response(netif, txp,
+				      err == GNTST_okay ? XEN_NETIF_RSP_OKAY
+							: XEN_NETIF_RSP_ERROR,
+				      &gop->notify.tail))
+			netif_put(netif);
+		else if (!gop->notify.head)
+			gop->notify.head = netif;
 		pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
-		netif_put(netif);
 	} else if (unlikely((err = mop->status) != GNTST_okay)) {
 		++mop;
 		txp = &pending_tx_info[pending_idx].req;
-		make_tx_response(netif, txp, XEN_NETIF_RSP_ERROR);
+		make_tx_response(netif, txp, XEN_NETIF_RSP_ERROR, NULL);
 		pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
 		netif_put(netif);
 	} else {
@@ -1349,12 +1387,15 @@ static int netbk_tx_check_gop(struct sk_
 		txp = &pending_tx_info[pending_idx].req;
 		if (txp->size > cop->len)
 			cmpxchg_local(&newerr, GNTST_okay, (--cop)->status);
-		make_tx_response(netif, txp,
-				 newerr == GNTST_okay ? XEN_NETIF_RSP_OKAY
-						      : XEN_NETIF_RSP_ERROR);
+		if (!make_tx_response(netif, txp,
+				      newerr == GNTST_okay ? XEN_NETIF_RSP_OKAY
+							   : XEN_NETIF_RSP_ERROR,
+				      &gop->notify.tail))
+			netif_put(netif);
+		else if (!gop->notify.head)
+			gop->notify.head = netif;
 		cmpxchg_local(&err, GNTST_okay, newerr);
 		pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
-		netif_put(netif);
 	}
 
 	for (i = start; i < nr_frags; i++, mop++) {
@@ -1376,7 +1417,7 @@ static int netbk_tx_check_gop(struct sk_
 
 		/* Error on this fragment: respond to client with an error. */
 		txp = &pending_tx_info[pending_idx].req;
-		make_tx_response(netif, txp, XEN_NETIF_RSP_ERROR);
+		make_tx_response(netif, txp, XEN_NETIF_RSP_ERROR, NULL);
 		pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx;
 		netif_put(netif);
 
@@ -1506,8 +1547,8 @@ static void net_tx_action(unsigned long
 
 	net_tx_action_dealloc();
 
-	gop.map = tx_map_ops;
-	gop.copy = tx_copy_ops + ARRAY_SIZEOF(tx_copy_ops);
+	gop.map = tx.map_ops;
+	gop.copy = tx.copy_ops + ARRAY_SIZEOF(tx.copy_ops);
 	while (NR_PENDING_REQS + XEN_NETIF_NR_SLOTS_MIN < MAX_PENDING_REQS
 	       && !list_empty(&net_schedule_list)) {
 		/* Get a netif from the list with work to do. */
@@ -1692,14 +1733,16 @@ static void net_tx_action(unsigned long
      * because it is unlikely that a network buffer will be paged out or shared,
      * and therefore it is unlikely to fail with GNTST_eagain. */
 	MULTI_grant_table_op(&mcl[0], GNTTABOP_copy, gop.copy,
-			     tx_copy_ops + ARRAY_SIZE(tx_copy_ops) - gop.copy);
+			     tx.copy_ops + ARRAY_SIZE(tx.copy_ops) - gop.copy);
 	MULTI_grant_table_op(&mcl[1], GNTTABOP_map_grant_ref,
-			     tx_map_ops, gop.map - tx_map_ops);
+			     tx.map_ops, gop.map - tx.map_ops);
 	if (HYPERVISOR_multicall_check(mcl, 2, NULL))
 		BUG();
 
-	gop.map = tx_map_ops;
-	gop.copy = tx_copy_ops + ARRAY_SIZE(tx_copy_ops);
+	gop.map = tx.map_ops;
+	gop.copy = tx.copy_ops + ARRAY_SIZE(tx.copy_ops);
+	gop.notify.head = NULL;
+	gop.notify.tail = NULL;
 	while ((skb = __skb_dequeue(&tx_queue)) != NULL) {
 		struct net_device *dev;
 		netif_tx_request_t *txp;
@@ -1764,6 +1807,10 @@ static void net_tx_action(unsigned long
 		netif_rx(skb);
 	}
 
+	if (gop.notify.head)
+		flush_notify_list(gop.notify.head, TX_IDX, tx.mcl,
+				  sizeof(tx) / sizeof(tx.mcl[0]));
+
  out:
 	if (netbk_copy_skb_mode == NETBK_DELAYED_COPY_SKB &&
 	    !list_empty(&pending_inuse_head)) {
@@ -1811,9 +1858,8 @@ irqreturn_t netif_be_int(int irq, void *
 	return IRQ_HANDLED;
 }
 
-static void make_tx_response(netif_t *netif, 
-			     netif_tx_request_t *txp,
-			     s8       st)
+static bool make_tx_response(netif_t *netif, const netif_tx_request_t *txp,
+			     s8 st, netif_t **tailp)
 {
 	RING_IDX i = netif->tx.rsp_prod_pvt;
 	netif_tx_response_t *resp;
@@ -1828,8 +1874,16 @@ static void make_tx_response(netif_t *ne
 
 	netif->tx.rsp_prod_pvt = ++i;
 	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->tx, notify);
-	if (notify)
-		notify_remote_via_irq(netif->irq);
+	if (notify) {
+		if (!tailp)
+			notify_remote_via_irq(netif->irq);
+		else if (netif != *tailp && !netif->tx_notify_link) {
+			if (*tailp)
+				(*tailp)->tx_notify_link = netif;
+			*tailp = netif;
+		} else
+			notify = false;
+	}
 
 #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER
 	if (i == netif->tx.req_cons) {
@@ -1839,6 +1893,8 @@ static void make_tx_response(netif_t *ne
 			add_to_net_schedule_list_tail(netif);
 	}
 #endif
+
+	return notify;
 }
 
 static netif_rx_response_t *make_rx_response(netif_t *netif, 
--- a/include/xen/evtchn.h
+++ b/include/xen/evtchn.h
@@ -193,6 +193,18 @@ static inline void notify_remote_via_evt
 	VOID(HYPERVISOR_event_channel_op(EVTCHNOP_send, &send));
 }
 
+static inline void
+multi_notify_remote_via_evtchn(multicall_entry_t *mcl, int port)
+{
+	struct evtchn_send *send = (void *)(mcl->args + 2);
+
+	BUILD_BUG_ON(sizeof(*send) > sizeof(mcl->args) - 2 * sizeof(*mcl->args));
+	send->port = port;
+	mcl->op = __HYPERVISOR_event_channel_op;
+	mcl->args[0] = EVTCHNOP_send;
+	mcl->args[1] = (unsigned long)send;
+}
+
 static inline int close_evtchn(int port)
 {
 	struct evtchn_close close = { .port = port };
@@ -207,6 +219,7 @@ int xen_test_irq_pending(int irq);
  * by bind_*_to_irqhandler().
  */
 void notify_remote_via_irq(int irq);
+int multi_notify_remote_via_irq(multicall_entry_t *, int irq);
 int irq_to_evtchn_port(int irq);
 
 #if defined(CONFIG_SMP) && !defined(MODULE) && defined(CONFIG_X86)
