From: Herbert Xu <herbert@gondor.apana.org.au>
Date: Wed, 27 May 2009 15:31:21 +1000
Subject: [net] ipv6: add GRO support
Message-id: E1M9BjV-0002CY-Lz@gondolin.me.apana.org.au
O-Subject: [PATCH 16/17] ipv6: Add GRO support
Bugzilla: 499347
RH-Acked-by: David Miller <davem@redhat.com>
RH-Acked-by: Neil Horman <nhorman@redhat.com>
RH-Acked-by: Thomas Graf <tgraf@redhat.com>

RHEL5 bugzilla #499347

ipv6: Add GRO support

This patch adds GRO support for IPv6.  IPv6 GRO supports extension
headers in the same way as GSO (by using the same infrastructure).
It's also simpler compared to IPv4 since we no longer have to worry
about fragmentation attributes or header checksums.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/net/protocol.h b/include/net/protocol.h
index 38d4bf6..5dba7b8 100644
--- a/include/net/protocol.h
+++ b/include/net/protocol.h
@@ -65,6 +65,12 @@ struct inet6_protocol
 	unsigned int	flags;	/* INET6_PROTO_xxx */
 };
 
+struct inet6_gro_protocol {
+	struct sk_buff	      **(*gro_receive)(struct sk_buff **head,
+					       struct sk_buff *skb);
+	int			(*gro_complete)(struct sk_buff *skb);
+};
+
 #define INET6_PROTO_NOPOLICY	0x1
 #define INET6_PROTO_FINAL	0x2
 /* This should be set for any extension header which is compatible with GSO. */
@@ -99,6 +105,7 @@ extern struct net_gro_protocol *inet_gro_protos[MAX_INET_PROTOS];
 
 #if defined(CONFIG_IPV6) || defined (CONFIG_IPV6_MODULE)
 extern struct inet6_protocol *inet6_protos[MAX_INET_PROTOS];
+extern struct inet6_gro_protocol *inet6_gro_protos[MAX_INET_PROTOS];
 #endif
 
 extern int	inet_add_protocol(struct net_protocol *prot, unsigned char num);
@@ -110,6 +117,8 @@ extern void	inet_unregister_protosw(struct inet_protosw *p);
 
 #if defined(CONFIG_IPV6) || defined (CONFIG_IPV6_MODULE)
 extern int	inet6_add_protocol(struct inet6_protocol *prot, unsigned char num);
+extern int	inet6_add_gro_protocol(struct inet6_gro_protocol *prot,
+				       unsigned char num);
 extern int	inet6_del_protocol(struct inet6_protocol *prot, unsigned char num);
 extern void	inet6_register_protosw(struct inet_protosw *p);
 extern void	inet6_unregister_protosw(struct inet_protosw *p);
diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c
index f689c24..c68aa3b 100644
--- a/net/ipv6/ipv6_sockglue.c
+++ b/net/ipv6/ipv6_sockglue.c
@@ -60,8 +60,7 @@
 
 DEFINE_SNMP_STAT(struct ipstats_mib, ipv6_statistics) __read_mostly;
 
-static struct inet6_protocol *ipv6_gso_pull_exthdrs(struct sk_buff *skb,
-						    int proto)
+static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
 {
 	struct inet6_protocol *ops = NULL;
 
@@ -92,7 +91,7 @@ static struct inet6_protocol *ipv6_gso_pull_exthdrs(struct sk_buff *skb,
 		__skb_pull(skb, len);
 	}
 
-	return ops;
+	return proto;
 }
 
 static int ipv6_gso_send_check(struct sk_buff *skb)
@@ -109,7 +108,9 @@ static int ipv6_gso_send_check(struct sk_buff *skb)
 	err = -EPROTONOSUPPORT;
 
 	rcu_read_lock();
-	ops = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
+	ops = rcu_dereference(inet6_protos[
+		ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr)]);
+
 	if (likely(ops && ops->gso_send_check)) {
 		skb->h.raw = skb->data;
 		err = ops->gso_send_check(skb);
@@ -145,7 +146,9 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb, int features)
 	segs = ERR_PTR(-EPROTONOSUPPORT);
 
 	rcu_read_lock();
-	ops = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
+	ops = rcu_dereference(inet6_protos[
+		ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr)]);
+
 	if (likely(ops && ops->gso_segment)) {
 		skb->h.raw = skb->data;
 		segs = ops->gso_segment(skb, features);
@@ -165,6 +168,124 @@ out:
 	return segs;
 }
 
+struct ipv6_gro_cb {
+	struct napi_gro_cb napi;
+	int proto;
+};
+
+#define IPV6_GRO_CB(skb) ((struct ipv6_gro_cb *)(skb)->cb)
+
+static struct sk_buff **ipv6_gro_receive(struct sk_buff **head,
+					 struct sk_buff *skb)
+{
+	struct inet6_gro_protocol *ops;
+	struct sk_buff **pp = NULL;
+	struct sk_buff *p;
+	struct ipv6hdr *iph;
+	unsigned int nlen;
+	unsigned int hlen;
+	unsigned int off;
+	int flush = 1;
+	int proto;
+	unsigned int csum;
+
+	off = skb_gro_offset(skb);
+	hlen = off + sizeof(*iph);
+	iph = skb_gro_header_fast(skb, off);
+	if (skb_gro_header_hard(skb, hlen)) {
+		iph = skb_gro_header_slow(skb, hlen, off);
+		if (unlikely(!iph))
+			goto out;
+	}
+
+	skb_gro_pull(skb, sizeof(*iph));
+	skb_set_transport_header(skb, skb_gro_offset(skb));
+
+	flush += ntohs(iph->payload_len) != skb_gro_len(skb);
+
+	rcu_read_lock();
+	proto = iph->nexthdr;
+	ops = rcu_dereference(inet6_gro_protos[proto]);
+	if (!ops || !ops->gro_receive) {
+		__pskb_pull(skb, skb_gro_offset(skb));
+		proto = ipv6_gso_pull_exthdrs(skb, proto);
+		skb_gro_pull(skb, -skb_transport_offset(skb));
+		skb_reset_transport_header(skb);
+		__skb_push(skb, skb_gro_offset(skb));
+
+		if (!ops || !ops->gro_receive)
+			goto out_unlock;
+
+		iph = ipv6_hdr(skb);
+	}
+
+	IPV6_GRO_CB(skb)->proto = proto;
+
+	flush--;
+	nlen = skb_network_header_len(skb);
+
+	for (p = *head; p; p = p->next) {
+		struct ipv6hdr *iph2;
+
+		if (!NAPI_GRO_CB(p)->same_flow)
+			continue;
+
+		iph2 = ipv6_hdr(p);
+
+		/* All fields must match except length. */
+		if (nlen != skb_network_header_len(p) ||
+		    memcmp(iph, iph2, offsetof(struct ipv6hdr, payload_len)) ||
+		    memcmp(&iph->nexthdr, &iph2->nexthdr,
+			   nlen - offsetof(struct ipv6hdr, nexthdr))) {
+			NAPI_GRO_CB(p)->same_flow = 0;
+			continue;
+		}
+
+		NAPI_GRO_CB(p)->flush |= flush;
+	}
+
+	NAPI_GRO_CB(skb)->flush |= flush;
+
+	csum = skb->csum;
+	skb_postpull_rcsum(skb, iph, skb_network_header_len(skb));
+
+	pp = ops->gro_receive(head, skb);
+
+	skb->csum = csum;
+
+out_unlock:
+	rcu_read_unlock();
+
+out:
+	NAPI_GRO_CB(skb)->flush |= flush;
+
+	return pp;
+}
+
+static int ipv6_gro_complete(struct sk_buff *skb)
+{
+	struct inet6_gro_protocol *ops;
+	struct ipv6hdr *iph = ipv6_hdr(skb);
+	int err = -ENOSYS;
+
+	iph->payload_len = htons(skb->len - skb_network_offset(skb) -
+				 sizeof(*iph));
+
+	rcu_read_lock();
+	ops = rcu_dereference(inet6_gro_protos[IPV6_GRO_CB(skb)->proto]);
+	if (unlikely(!ops || !ops->gro_complete)) {
+		WARN_ON(1);
+		goto out_unlock;
+	}
+
+	err = ops->gro_complete(skb);
+
+out_unlock:
+	rcu_read_unlock();
+
+	return err;
+}
+
 static struct packet_type ipv6_packet_type = {
 	.type = __constant_htons(ETH_P_IPV6), 
 	.func = ipv6_rcv,
@@ -172,6 +293,12 @@ static struct packet_type ipv6_packet_type = {
 	.gso_segment = ipv6_gso_segment,
 };
 
+static struct gro_packet_type ipv6_gro_packet_type = {
+	.type = __constant_htons(ETH_P_IPV6),
+	.gro_receive = ipv6_gro_receive,
+	.gro_complete = ipv6_gro_complete,
+};
+
 struct ip6_ra_chain *ip6_ra_chain;
 DEFINE_RWLOCK(ip6_ra_lock);
 
@@ -1129,9 +1256,11 @@ EXPORT_SYMBOL(compat_ipv6_getsockopt);
 void __init ipv6_packet_init(void)
 {
 	dev_add_pack(&ipv6_packet_type);
+	dev_add_pack_gro(&ipv6_gro_packet_type);
 }
 
 void ipv6_packet_cleanup(void)
 {
+	dev_remove_pack_gro(&ipv6_gro_packet_type);
 	dev_remove_pack(&ipv6_packet_type);
 }
diff --git a/net/ipv6/protocol.c b/net/ipv6/protocol.c
index 52c1d58..a2f45b9 100644
--- a/net/ipv6/protocol.c
+++ b/net/ipv6/protocol.c
@@ -40,6 +40,7 @@
 #include <net/protocol.h>
 
 struct inet6_protocol *inet6_protos[MAX_INET_PROTOS];
+struct inet6_gro_protocol *inet6_gro_protos[MAX_INET_PROTOS];
 static DEFINE_SPINLOCK(inet6_proto_lock);
 
 
@@ -61,6 +62,25 @@ int inet6_add_protocol(struct inet6_protocol *prot, unsigned char protocol)
 	return ret;
 }
 
+int inet6_add_gro_protocol(struct inet6_gro_protocol *prot,
+			   unsigned char protocol)
+{
+	int ret, hash = protocol & (MAX_INET_PROTOS - 1);
+
+	spin_lock_bh(&inet6_proto_lock);
+
+	if (inet6_gro_protos[hash]) {
+		ret = -1;
+	} else {
+		inet6_gro_protos[hash] = prot;
+		ret = 0;
+	}
+
+	spin_unlock_bh(&inet6_proto_lock);
+
+	return ret;
+}
+
 /*
  *	Remove a protocol from the hash tables.
  */