From 5b3d71363518f42ffbfe3790fc9ebbab38662c54 Mon Sep 17 00:00:00 2001
From: Mark McLoughlin <markmc@redhat.com>
Date: Tue, 21 Jul 2009 17:25:13 +0100
Subject: [PATCH] virtio_net: remove the tx mitigation timer

https://bugzilla.redhat.com/504647

The tx mitigation timer is designed to reduce the rate of guest exits
thereby improving throughput. However, we have identified a number of
cases where mitigation hurts throughput and benchmarking has show that
it only helps throughput in a very limited number of cases.

This may be due to the fact that in RHEL5, the timer is effectively
1ms rather than 150us, since we don't have high resolution timers.

This patch firstly removes the timer and replaces it with a bottom half.
When the host is notified of a packet on the tx queue, it schedules a
bottom half in the I/O thread in order to flush the queue.

Next, it disables enabling notifications until it has flushed the queue.
In order to avoid a race condition with this, it flushes the queue a
second time when notifications are re-enabled.

Finally, if it successfully flushed a number of packets in the bottom
half, it disables notifications re-schedules the bottom half to quickly
flush the queue again.

Migration is handled by equating the tx_timer_active savevm field to
tx_bh_scheduled.

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Message-Id: <1248193513.7126.12.camel@blaa>
Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
Bugzilla: 504647
Acked-by: Mark Wagner <mwagner@redhat.com>
Acked-by: Dor Laor <dlaor@redhat.com>
Acked-by: Juan Quintela <quintela@redhat.com>
---
 qemu/hw/virtio-net.c |   72 +++++++++++++++++++++++++-------------------------
 qemu/hw/virtio-net.h |    2 -
 2 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/qemu/hw/virtio-net.c b/qemu/hw/virtio-net.c
index 58e59f5..be8ba6d 100644
--- a/qemu/hw/virtio-net.c
+++ b/qemu/hw/virtio-net.c
@@ -29,8 +29,8 @@ typedef struct VirtIONet
     VirtQueue *rx_vq;
     VirtQueue *tx_vq;
     VLANClientState *vc;
-    QEMUTimer *tx_timer;
-    int tx_timer_active;
+    QEMUBH *tx_bh;
+    int tx_bh_scheduled;
     struct {
         VirtQueueElement elem;
         ssize_t len;
@@ -325,9 +325,10 @@ static void virtio_net_receive_raw(void *opaque, const uint8_t *buf, int size)
 }
 
 /* TX */
-static void virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
+static int virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq, int enable_notify)
 {
     VirtQueueElement elem;
+    int num_packets = 0;
 #ifdef TAP_VNET_HDR
     int has_vnet_hdr = tap_has_vnet_hdr(n->vc->vlan->first_client);
 #else
@@ -335,11 +336,11 @@ static void virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 #endif
 
     if (!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK))
-        return;
+        return num_packets;
 
     if (n->tx_queue.out_num) {
         virtio_queue_set_notification(n->tx_vq, 0);
-        return;
+        return num_packets;
     }
 
     while (virtqueue_pop(vq, &elem)) {
@@ -378,43 +379,46 @@ static void virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
             n->tx_queue.len = len;
             n->tx_queue.out_sg = out_sg;
             n->tx_queue.out_num = out_num;
-            return;
+            return num_packets;
         }
 
         virtqueue_push(vq, &elem, len + ret);
         virtio_notify(&n->vdev, vq);
+
+        num_packets++;
+    }
+
+    if (enable_notify) {
+        virtio_queue_set_notification(vq, 1);
+        num_packets += virtio_net_flush_tx(n, vq, 0);
     }
+
+    return num_packets;
 }
 
 static void virtio_net_handle_tx(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIONet *n = to_virtio_net(vdev);
 
-    if (n->tx_timer_active) {
-        virtio_queue_set_notification(vq, 1);
-        qemu_del_timer(n->tx_timer);
-        n->tx_timer_active = 0;
-        virtio_net_flush_tx(n, vq);
-    } else {
-        qemu_mod_timer(n->tx_timer,
-                       qemu_get_clock(vm_clock) + TX_TIMER_INTERVAL);
-        n->tx_timer_active = 1;
-        virtio_queue_set_notification(vq, 0);
-    }
+    if (n->tx_bh_scheduled)
+        return;
+
+    virtio_queue_set_notification(n->tx_vq, 0);
+    qemu_bh_schedule(n->tx_bh);
+    n->tx_bh_scheduled = 1;
 }
 
-static void virtio_net_tx_timer(void *opaque)
+static void virtio_net_tx_bh(void *opaque)
 {
     VirtIONet *n = opaque;
 
-    n->tx_timer_active = 0;
-
-    /* Just in case the driver is not ready on more */
-    if (!(n->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK))
-        return;
+    n->tx_bh_scheduled = 0;
 
-    virtio_queue_set_notification(n->tx_vq, 1);
-    virtio_net_flush_tx(n, n->tx_vq);
+    if (virtio_net_flush_tx(n, n->tx_vq, 1)) {
+        virtio_queue_set_notification(n->tx_vq, 0);
+        qemu_bh_schedule(n->tx_bh);
+        n->tx_bh_scheduled = 1;
+    }
 }
 
 static void virtio_net_retry_tx(void *opaque)
@@ -436,7 +440,7 @@ static void virtio_net_retry_tx(void *opaque)
     }
 
     virtio_queue_set_notification(n->tx_vq, 1);
-    virtio_net_flush_tx(n, n->tx_vq);
+    virtio_net_flush_tx(n, n->tx_vq, 0);
 }
 
 static void virtio_net_save(QEMUFile *f, void *opaque)
@@ -446,7 +450,7 @@ static void virtio_net_save(QEMUFile *f, void *opaque)
     virtio_save(&n->vdev, f);
 
     qemu_put_buffer(f, n->mac, 6);
-    qemu_put_be32(f, n->tx_timer_active);
+    qemu_put_be32(f, n->tx_bh_scheduled);
     qemu_put_be32(f, n->mergeable_rx_bufs);
 
 #ifdef TAP_VNET_HDR
@@ -464,7 +468,7 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
     virtio_load(&n->vdev, f);
 
     qemu_get_buffer(f, n->mac, 6);
-    n->tx_timer_active = qemu_get_be32(f);
+    n->tx_bh_scheduled = qemu_get_be32(f);
     n->mergeable_rx_bufs = qemu_get_be32(f);
 
 #ifdef TAP_VNET_HDR
@@ -472,9 +476,8 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         tap_using_vnet_hdr(n->vc->vlan->first_client, 1);
 #endif
 
-    if (n->tx_timer_active) {
-        qemu_mod_timer(n->tx_timer,
-                       qemu_get_clock(vm_clock) + TX_TIMER_INTERVAL);
+    if (n->tx_bh_scheduled) {
+        qemu_bh_schedule(n->tx_bh);
     }
 
     return 0;
@@ -486,9 +489,6 @@ static void virtio_net_cleanup(VLANClientState *vc)
 
     unregister_savevm("virtio-net", n);
 
-    qemu_del_timer(n->tx_timer);
-    qemu_free_timer(n->tx_timer);
-
     virtio_cleanup(&n->vdev);
 }
 
@@ -526,8 +526,8 @@ PCIDevice *virtio_net_init(PCIBus *bus, NICInfo *nd, int devfn)
 
     qemu_format_nic_info_str(n->vc, n->mac);
 
-    n->tx_timer = qemu_new_timer(vm_clock, virtio_net_tx_timer, n);
-    n->tx_timer_active = 0;
+    n->tx_bh = qemu_bh_new(virtio_net_tx_bh, n);
+    n->tx_bh_scheduled = 0;
     n->mergeable_rx_bufs = 0;
 
     register_savevm("virtio-net", virtio_net_id++, 2,
diff --git a/qemu/hw/virtio-net.h b/qemu/hw/virtio-net.h
index 9ac9e34..5d6d814 100644
--- a/qemu/hw/virtio-net.h
+++ b/qemu/hw/virtio-net.h
@@ -41,8 +41,6 @@
 
 #define VIRTIO_NET_S_LINK_UP    1       /* Link is up */
 
-#define TX_TIMER_INTERVAL 150000 /* 150 us */
-
 /* Maximum packet size we can receive from tap device: header + 64k */
 #define VIRTIO_NET_MAX_BUFSIZE (sizeof(struct virtio_net_hdr) + (64 << 10))
 
-- 
1.6.3.rc4.29.g8146