From: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
To: virtualization@lists.linux.dev
Cc: "Michael S. Tsirkin" <mst@redhat.com>,
Jason Wang <jasowang@redhat.com>,
Xuan Zhuo <xuanzhuo@linux.alibaba.com>,
"David S. Miller" <davem@davemloft.net>,
Eric Dumazet <edumazet@google.com>,
Jakub Kicinski <kuba@kernel.org>, Paolo Abeni <pabeni@redhat.com>,
Alexei Starovoitov <ast@kernel.org>,
Daniel Borkmann <daniel@iogearbox.net>,
Jesper Dangaard Brouer <hawk@kernel.org>,
John Fastabend <john.fastabend@gmail.com>,
netdev@vger.kernel.org, bpf@vger.kernel.org
Subject: [PATCH vhost 5/5] virtio_net: sq support premapped mode
Date: Wed, 8 May 2024 10:23:31 +0800 [thread overview]
Message-ID: <20240508022331.63751-6-xuanzhuo@linux.alibaba.com> (raw)
In-Reply-To: <20240508022331.63751-1-xuanzhuo@linux.alibaba.com>
If the xsk is enabling, the xsk tx will share the send queue.
But the xsk requires that the send queue use the premapped mode.
So the send queue must support premapped mode when it is bound to
af-xdp.
* virtnet_sq_set_premapped(sq, true) is used to enable premapped mode.
In this mode, the driver will record the dma info when skb or xdp
frame is sent.
Currently, the SQ premapped mode is operational only with af-xdp. In
this mode, af-xdp, the kernel stack, and xdp tx/redirect will share
the same SQ. Af-xdp independently manages its DMA. The kernel stack
and xdp tx/redirect utilize this DMA metadata to manage the DMA
info.
If the indirect descriptor feature be supported, the volume of DMA
details we need to maintain becomes quite substantial. Here, we have
a cap on the amount of DMA info we manage.
If the kernel stack and xdp tx/redirect attempt to use more
descriptors, virtnet_add_outbuf() will return an -ENOMEM error. But
the af-xdp can work continually.
* virtnet_sq_set_premapped(sq, false) is used to disable premapped mode.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio_net.c | 210 ++++++++++++++++++++++++++++++++++++++-
1 file changed, 207 insertions(+), 3 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index a28a84101d5b..7274daed4993 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -24,6 +24,7 @@
#include <net/xdp.h>
#include <net/net_failover.h>
#include <net/netdev_rx_queue.h>
+#include <uapi/linux/virtio_ring.h>
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -47,6 +48,7 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_REDIR BIT(1)
#define VIRTIO_XDP_FLAG BIT(0)
+#define VIRTIO_DMA_FLAG BIT(1)
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
@@ -146,6 +148,25 @@ struct virtnet_rq_dma {
u16 need_sync;
};
+struct virtnet_sq_dma {
+ union {
+ struct virtnet_sq_dma *next;
+ void *data;
+ };
+ dma_addr_t addr;
+ u32 len;
+ u8 num;
+};
+
+struct virtnet_sq_dma_info {
+ /* record for kfree */
+ void *p;
+
+ u32 free_num;
+
+ struct virtnet_sq_dma *free;
+};
+
/* Internal representation of a send virtqueue */
struct send_queue {
/* Virtqueue associated with this send _queue */
@@ -165,6 +186,11 @@ struct send_queue {
/* Record whether sq is in reset state. */
bool reset;
+
+ /* SQ is premapped mode or not. */
+ bool premapped;
+
+ struct virtnet_sq_dma_info dmainfo;
};
/* Internal representation of a receive virtqueue */
@@ -374,6 +400,173 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static void virtnet_sq_unmap(struct send_queue *sq, void **data)
+{
+ struct virtnet_sq_dma *head, *tail, *p;
+ unsigned long ptr;
+ int i;
+
+ ptr = (unsigned long)*data;
+
+ if (!(ptr & VIRTIO_DMA_FLAG))
+ return;
+
+ head = (void *)(ptr & ~VIRTIO_DMA_FLAG);
+
+ p = head;
+
+ for (i = 0; i < head->num; ++i) {
+ virtqueue_dma_unmap_page_attrs(sq->vq, p->addr, p->len,
+ DMA_TO_DEVICE, 0);
+ tail = p;
+ p = p->next;
+ }
+
+ *data = tail->data;
+
+ tail->next = sq->dmainfo.free;
+ sq->dmainfo.free = head;
+ sq->dmainfo.free_num += head->num;
+}
+
+static void *virtnet_dma_chain_update(struct send_queue *sq,
+ struct virtnet_sq_dma *head,
+ struct virtnet_sq_dma *tail,
+ u8 num, void *data)
+{
+ sq->dmainfo.free = tail->next;
+ sq->dmainfo.free_num -= num;
+ head->num = num;
+
+ tail->data = data;
+
+ return (void *)((unsigned long)head | VIRTIO_DMA_FLAG);
+}
+
+static struct virtnet_sq_dma *virtnet_sq_map_sg(struct send_queue *sq, int num, void *data)
+{
+ struct virtnet_sq_dma *head, *tail, *p;
+ struct scatterlist *sg;
+ int i;
+
+ if (num > sq->dmainfo.free_num)
+ return NULL;
+
+ head = sq->dmainfo.free;
+ p = head;
+
+ tail = NULL;
+
+ for (i = 0; i < num; ++i) {
+ if (virtqueue_dma_map_sg_attrs(sq->vq, &sq->sg[i], DMA_TO_DEVICE, 0))
+ goto err;
+
+ tail = p;
+ tail->addr = sg->dma_address;
+ tail->len = sg->length;
+
+ p = p->next;
+ }
+
+ return virtnet_dma_chain_update(sq, head, tail, num, data);
+
+err:
+ if (tail) {
+ data = virtnet_dma_chain_update(sq, head, tail, i, data);
+ virtnet_sq_unmap(sq, &data);
+ }
+
+ return NULL;
+}
+
+static int virtnet_add_outbuf(struct send_queue *sq, u8 num, void *data)
+{
+ int ret;
+
+ if (sq->premapped) {
+ data = virtnet_sq_map_sg(sq, num, data);
+ if (!data)
+ return -ENOMEM;
+ }
+
+ ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
+ if (ret && sq->premapped)
+ virtnet_sq_unmap(sq, &data);
+
+ return ret;
+}
+
+static int virtnet_sq_alloc_dma_meta(struct send_queue *sq)
+{
+ struct virtnet_sq_dma *d;
+ int num, i;
+
+ num = virtqueue_get_vring_size(sq->vq);
+
+ /* Currently, the SQ premapped mode is operational only with af-xdp. In
+ * this mode, af-xdp, the kernel stack, and xdp tx/redirect will share
+ * the same SQ. Af-xdp independently manages its DMA. The kernel stack
+ * and xdp tx/redirect utilize this DMA metadata to manage the DMA info.
+ *
+ * If the indirect descriptor feature be supported, the volume of DMA
+ * details we need to maintain becomes quite substantial. Here, we have
+ * a cap on the amount of DMA info we manage, effectively limiting it to
+ * twice the size of the ring buffer.
+ *
+ * If the kernel stack and xdp tx/redirect attempt to use more
+ * descriptors than allowed by this double ring buffer size,
+ * virtnet_add_outbuf() will return an -ENOMEM error. But the af-xdp can
+ * work continually.
+ */
+ if (virtio_has_feature(sq->vq->vdev, VIRTIO_RING_F_INDIRECT_DESC))
+ num = num * 2;
+
+ sq->dmainfo.free = kcalloc(num, sizeof(*sq->dmainfo.free), GFP_KERNEL);
+ if (!sq->dmainfo.free)
+ return -ENOMEM;
+
+ sq->dmainfo.p = sq->dmainfo.free;
+ sq->dmainfo.free_num = num;
+
+ for (i = 0; i < num; ++i) {
+ d = &sq->dmainfo.free[i];
+ d->next = d + 1;
+ }
+
+ d->next = NULL;
+
+ return 0;
+}
+
+static void virtnet_sq_free_dma_meta(struct send_queue *sq)
+{
+ kfree(sq->dmainfo.p);
+
+ sq->dmainfo.p = NULL;
+ sq->dmainfo.free = NULL;
+ sq->dmainfo.free_num = 0;
+}
+
+int virtnet_sq_set_premapped(struct send_queue *sq, bool premapped);
+
+/* This function must be called immediately after creating the vq, or after vq
+ * reset, and before adding any buffers to it.
+ */
+int virtnet_sq_set_premapped(struct send_queue *sq, bool premapped)
+{
+ if (premapped) {
+ if (virtnet_sq_alloc_dma_meta(sq))
+ return -ENOMEM;
+ } else {
+ virtnet_sq_free_dma_meta(sq);
+ }
+
+ BUG_ON(virtqueue_set_dma_premapped(sq->vq, premapped));
+
+ sq->premapped = premapped;
+ return 0;
+}
+
static void __free_old_xmit(struct send_queue *sq, bool in_napi,
struct virtnet_sq_free_stats *stats)
{
@@ -383,6 +576,8 @@ static void __free_old_xmit(struct send_queue *sq, bool in_napi,
while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
++stats->packets;
+ virtnet_sq_unmap(sq, &ptr);
+
if (!is_xdp_frame(ptr)) {
struct sk_buff *skb = ptr;
@@ -915,8 +1110,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
skb_frag_size(frag), skb_frag_off(frag));
}
- err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
- xdp_to_ptr(xdpf), GFP_ATOMIC);
+ err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
if (unlikely(err))
return -ENOSPC; /* Caller handle free/refcnt */
@@ -2380,7 +2574,7 @@ static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
return num_sg;
num_sg++;
}
- return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
+ return virtnet_add_outbuf(sq, num_sg, skb);
}
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
@@ -4215,6 +4409,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
for (i = 0; i < vi->max_queue_pairs; i++) {
__netif_napi_del(&vi->rq[i].napi);
__netif_napi_del(&vi->sq[i].napi);
+
+ virtnet_sq_free_dma_meta(&vi->sq[i]);
}
/* We called __netif_napi_del(),
@@ -4263,6 +4459,14 @@ static void free_receive_page_frags(struct virtnet_info *vi)
static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
{
+ struct virtnet_info *vi = vq->vdev->priv;
+ struct send_queue *sq;
+ int i = vq2rxq(vq);
+
+ sq = &vi->sq[i];
+
+ virtnet_sq_unmap(sq, &buf);
+
if (!is_xdp_frame(buf))
dev_kfree_skb(buf);
else
--
2.32.0.3.g01195cf9f
prev parent reply other threads:[~2024-05-08 2:23 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-05-08 2:23 [PATCH vhost 0/5] virtio_net: introduce api to enable/disable premapped mode for sq Xuan Zhuo
2024-05-08 2:23 ` [PATCH vhost 1/5] virtio_ring: introduce vring_need_unmap_buffer Xuan Zhuo
2024-05-08 2:23 ` [PATCH vhost 2/5] virtio_ring: introduce dma map api for page Xuan Zhuo
2024-05-08 2:23 ` [PATCH vhost 3/5] virtio_ring: introduce virtqueue_dma_map_sg_attrs Xuan Zhuo
2024-05-08 2:23 ` [PATCH vhost 4/5] virtio_ring: virtqueue_set_dma_premapped() support to disable Xuan Zhuo
2024-05-08 2:23 ` Xuan Zhuo [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240508022331.63751-6-xuanzhuo@linux.alibaba.com \
--to=xuanzhuo@linux.alibaba.com \
--cc=ast@kernel.org \
--cc=bpf@vger.kernel.org \
--cc=daniel@iogearbox.net \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=hawk@kernel.org \
--cc=jasowang@redhat.com \
--cc=john.fastabend@gmail.com \
--cc=kuba@kernel.org \
--cc=mst@redhat.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=virtualization@lists.linux.dev \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).