filter: optimize accesses to ancillary data

We can translate pseudo load instructions at filter check time to
dedicated instructions to speed up filtering and avoid one switch().
libpcap currently uses SKF_AD_PROTOCOL, but custom filters probably use
other ancillary accesses.

Note : I made the assertion that ancillary data was always accessed with
BPF_LD|BPF_?|BPF_ABS instructions, not with BPF_LD|BPF_?|BPF_IND ones
(offset given by K constant, not by K + X register)

On x86_64, this saves a few bytes of text :

# size net/core/filter.o.*
   text	   data	    bss	    dec	    hex	filename
   4864	      0	      0	   4864	   1300	net/core/filter.o.new
   4944	      0	      0	   4944	   1350	net/core/filter.o.old

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 44 insertions, 28 deletions

diff --git a/net/core/filter.c b/net/core/filter.c
index e8a6ac411ffb..2b27d4efdd48 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -85,6 +85,17 @@ enum {
 	BPF_S_JMP_JGT_X,
 	BPF_S_JMP_JSET_K,
 	BPF_S_JMP_JSET_X,
+	/* Ancillary data */
+	BPF_S_ANC_PROTOCOL,
+	BPF_S_ANC_PKTTYPE,
+	BPF_S_ANC_IFINDEX,
+	BPF_S_ANC_NLATTR,
+	BPF_S_ANC_NLATTR_NEST,
+	BPF_S_ANC_MARK,
+	BPF_S_ANC_QUEUE,
+	BPF_S_ANC_HATYPE,
+	BPF_S_ANC_RXHASH,
+	BPF_S_ANC_CPU,
 };
 
 /* No hurry in this branch */
@@ -107,11 +118,7 @@ static inline void *load_pointer(const struct sk_buff *skb, int k,
 {
 	if (k >= 0)
 		return skb_header_pointer(skb, k, size, buffer);
-	else {
-		if (k >= SKF_AD_OFF)
-			return NULL;
-		return __load_pointer(skb, k, size);
-	}
+	return __load_pointer(skb, k, size);
 }
 
 /**
@@ -269,7 +276,7 @@ load_w:
 				A = get_unaligned_be32(ptr);
 				continue;
 			}
-			break;
+			return 0;
 		case BPF_S_LD_H_ABS:
 			k = K;
 load_h:
@@ -278,7 +285,7 @@ load_h:
 				A = get_unaligned_be16(ptr);
 				continue;
 			}
-			break;
+			return 0;
 		case BPF_S_LD_B_ABS:
 			k = K;
 load_b:
@@ -287,7 +294,7 @@ load_b:
 				A = *(u8 *)ptr;
 				continue;
 			}
-			break;
+			return 0;
 		case BPF_S_LD_W_LEN:
 			A = skb->len;
 			continue;
@@ -338,45 +345,35 @@ load_b:
 		case BPF_S_STX:
 			mem[K] = X;
 			continue;
-		default:
-			WARN_ON(1);
-			return 0;
-		}
-
-		/*
-		 * Handle ancillary data, which are impossible
-		 * (or very difficult) to get parsing packet contents.
-		 */
-		switch (k-SKF_AD_OFF) {
-		case SKF_AD_PROTOCOL:
+		case BPF_S_ANC_PROTOCOL:
 			A = ntohs(skb->protocol);
 			continue;
-		case SKF_AD_PKTTYPE:
+		case BPF_S_ANC_PKTTYPE:
 			A = skb->pkt_type;
 			continue;
-		case SKF_AD_IFINDEX:
+		case BPF_S_ANC_IFINDEX:
 			if (!skb->dev)
 				return 0;
 			A = skb->dev->ifindex;
 			continue;
-		case SKF_AD_MARK:
+		case BPF_S_ANC_MARK:
 			A = skb->mark;
 			continue;
-		case SKF_AD_QUEUE:
+		case BPF_S_ANC_QUEUE:
 			A = skb->queue_mapping;
 			continue;
-		case SKF_AD_HATYPE:
+		case BPF_S_ANC_HATYPE:
 			if (!skb->dev)
 				return 0;
 			A = skb->dev->type;
 			continue;
-		case SKF_AD_RXHASH:
+		case BPF_S_ANC_RXHASH:
 			A = skb->rxhash;
 			continue;
-		case SKF_AD_CPU:
+		case BPF_S_ANC_CPU:
 			A = raw_smp_processor_id();
 			continue;
-		case SKF_AD_NLATTR: {
+		case BPF_S_ANC_NLATTR: {
 			struct nlattr *nla;
 
 			if (skb_is_nonlinear(skb))
@@ -392,7 +389,7 @@ load_b:
 				A = 0;
 			continue;
 		}
-		case SKF_AD_NLATTR_NEST: {
+		case BPF_S_ANC_NLATTR_NEST: {
 			struct nlattr *nla;
 
 			if (skb_is_nonlinear(skb))
@@ -412,6 +409,7 @@ load_b:
 			continue;
 		}
 		default:
+			WARN_ON(1);
 			return 0;
 		}
 	}
@@ -600,6 +598,24 @@ int sk_chk_filter(struct sock_filter *filter, int flen)
 			    pc + ftest->jf + 1 >= flen)
 				return -EINVAL;
 			break;
+		case BPF_S_LD_W_ABS:
+		case BPF_S_LD_H_ABS:
+		case BPF_S_LD_B_ABS:
+#define ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE:	\
+				code = BPF_S_ANC_##CODE;	\
+				break
+			switch (ftest->k) {
+			ANCILLARY(PROTOCOL);
+			ANCILLARY(PKTTYPE);
+			ANCILLARY(IFINDEX);
+			ANCILLARY(NLATTR);
+			ANCILLARY(NLATTR_NEST);
+			ANCILLARY(MARK);
+			ANCILLARY(QUEUE);
+			ANCILLARY(HATYPE);
+			ANCILLARY(RXHASH);
+			ANCILLARY(CPU);
+			}
 		}
 		ftest->code = code;
 	}