bpf_jit_comp.rar_legacy

/* bpf_jit_comp.c: BPF JIT compiler * * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation * * Based on the x86 BPF compiler, by Eric Dumazet (eric.dumazet@gmail.com) * Ported to ppc32 by Denis Kirjanov <kda@linux-powerpc.org> * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; version 2 * of the License. */ #include <linux/moduleloader.h> #include <asm/cacheflush.h> #include <linux/netdevice.h> #include <linux/filter.h> #include <linux/if_vlan.h> #include "bpf_jit.h" int bpf_jit_enable __read_mostly; static inline void bpf_flush_icache(void *start, void *end) { smp_wmb(); flush_icache_range((unsigned long)start, (unsigned long)end); } static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx) { int i; const struct sock_filter *filter = fp->insns; if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) { /* Make stackframe */ if (ctx->seen & SEEN_DATAREF) { /* If we call any helpers (for loads), save LR */ EMIT(PPC_INST_MFLR | __PPC_RT(R0)); PPC_BPF_STL(0, 1, PPC_LR_STKOFF); /* Back up non-volatile regs. */ PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D))); PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL))); } if (ctx->seen & SEEN_MEM) { /* * Conditionally save regs r15-r31 as some will be used * for M[] data. */ for (i = r_M; i < (r_M+16); i++) { if (ctx->seen & (1 << (i-r_M))) PPC_BPF_STL(i, 1, -(REG_SZ*(32-i))); } } PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME); } if (ctx->seen & SEEN_DATAREF) { /* * If this filter needs to access skb data, * prepare r_D and r_HL: * r_HL = skb->len - skb->data_len * r_D = skb->data */ PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff, data_len)); PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len)); PPC_SUB(r_HL, r_HL, r_scratch1); PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data)); } if (ctx->seen & SEEN_XREG) { /* * TODO: Could also detect whether first instr. sets X and * avoid this (as below, with A). */ PPC_LI(r_X, 0); } switch (filter[0].code) { case BPF_RET | BPF_K: case BPF_LD | BPF_W | BPF_LEN: case BPF_LD | BPF_W | BPF_ABS: case BPF_LD | BPF_H | BPF_ABS: case BPF_LD | BPF_B | BPF_ABS: /* first instruction sets A register (or is RET 'constant') */ break; default: /* make sure we dont leak kernel information to user */ PPC_LI(r_A, 0); } } static void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) { int i; if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) { PPC_ADDI(1, 1, BPF_PPC_STACKFRAME); if (ctx->seen & SEEN_DATAREF) { PPC_BPF_LL(0, 1, PPC_LR_STKOFF); PPC_MTLR(0); PPC_BPF_LL(r_D, 1, -(REG_SZ*(32-r_D))); PPC_BPF_LL(r_HL, 1, -(REG_SZ*(32-r_HL))); } if (ctx->seen & SEEN_MEM) { /* Restore any saved non-vol registers */ for (i = r_M; i < (r_M+16); i++) { if (ctx->seen & (1 << (i-r_M))) PPC_BPF_LL(i, 1, -(REG_SZ*(32-i))); } } } /* The RETs have left a return value in R3. */ PPC_BLR(); } #define CHOOSE_LOAD_FUNC(K, func) \ ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) /* Assemble the body code between the prologue & epilogue. */ static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, unsigned int *addrs) { const struct sock_filter *filter = fp->insns; int flen = fp->len; u8 *func; unsigned int true_cond; int i; /* Start of epilogue code */ unsigned int exit_addr = addrs[flen]; for (i = 0; i < flen; i++) { unsigned int K = filter[i].k; u16 code = bpf_anc_helper(&filter[i]); /* * addrs[] maps a BPF bytecode address into a real offset from * the start of the body code. */ addrs[i] = ctx->idx * 4; switch (code) { /*** ALU ops ***/ case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */ ctx->seen |= SEEN_XREG; PPC_ADD(r_A, r_A, r_X); break; case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */ if (!K) break; PPC_ADDI(r_A, r_A, IMM_L(K)); if (K >= 32768) PPC_ADDIS(r_A, r_A, IMM_HA(K)); break; case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */ ctx->seen |= SEEN_XREG; PPC_SUB(r_A, r_A, r_X); break; case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ if (!K) break; PPC_ADDI(r_A, r_A, IMM_L(-K)); if (K >= 32768) PPC_ADDIS(r_A, r_A, IMM_HA(-K)); break; case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */ ctx->seen |= SEEN_XREG; PPC_MUL(r_A, r_A, r_X); break; case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ if (K < 32768) PPC_MULI(r_A, r_A, K); else { PPC_LI32(r_scratch1, K); PPC_MUL(r_A, r_A, r_scratch1); } break; case BPF_ALU | BPF_MOD | BPF_X: /* A %= X; */ case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ ctx->seen |= SEEN_XREG; PPC_CMPWI(r_X, 0); if (ctx->pc_ret0 != -1) { PPC_BCC(COND_EQ, addrs[ctx->pc_ret0]); } else { PPC_BCC_SHORT(COND_NE, (ctx->idx*4)+12); PPC_LI(r_ret, 0); PPC_JMP(exit_addr); } if (code == (BPF_ALU | BPF_MOD | BPF_X)) { PPC_DIVWU(r_scratch1, r_A, r_X); PPC_MUL(r_scratch1, r_X, r_scratch1); PPC_SUB(r_A, r_A, r_scratch1); } else { PPC_DIVWU(r_A, r_A, r_X); } break; case BPF_ALU | BPF_MOD | BPF_K: /* A %= K; */ PPC_LI32(r_scratch2, K); PPC_DIVWU(r_scratch1, r_A, r_scratch2); PPC_MUL(r_scratch1, r_scratch2, r_scratch1); PPC_SUB(r_A, r_A, r_scratch1); break; case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */ if (K == 1) break; PPC_LI32(r_scratch1, K); PPC_DIVWU(r_A, r_A, r_scratch1); break; case BPF_ALU | BPF_AND | BPF_X: ctx->seen |= SEEN_XREG; PPC_AND(r_A, r_A, r_X); break; case BPF_ALU | BPF_AND | BPF_K: if (!IMM_H(K)) PPC_ANDI(r_A, r_A, K); else { PPC_LI32(r_scratch1, K); PPC_AND(r_A, r_A, r_scratch1); } break; case BPF_ALU | BPF_OR | BPF_X: ctx->seen |= SEEN_XREG; PPC_OR(r_A, r_A, r_X); break; case BPF_ALU | BPF_OR | BPF_K: if (IMM_L(K)) PPC_ORI(r_A, r_A, IMM_L(K)); if (K >= 65536) PPC_ORIS(r_A, r_A, IMM_H(K)); break; case BPF_ANC | SKF_AD_ALU_XOR_X: case BPF_ALU | BPF_XOR | BPF_X: /* A ^= X */ ctx->seen |= SEEN_XREG; PPC_XOR(r_A, r_A, r_X); break; case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ if (IMM_L(K)) PPC_XORI(r_A, r_A, IMM_L(K)); if (K >= 65536) PPC_XORIS(r_A, r_A, IMM_H(K)); break; case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */ ctx->seen |= SEEN_XREG; PPC_SLW(r_A, r_A, r_X); break; case BPF_ALU | BPF_LSH | BPF_K: if (K == 0) break; else PPC_SLWI(r_A, r_A, K); break; case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */ ctx->seen |= SEEN_XREG; PPC_SRW(r_A, r_A, r_X); break; case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */ if (K == 0) break; else PPC_SRWI(r_A, r_A, K); break; case BPF_ALU | BPF_NEG: PPC_NEG(r_A, r_A); break; case BPF_RET | BPF_K: PPC_LI32(r_ret, K); if (!K) { if (ctx->pc_ret0 == -1) ctx->pc_ret0 = i; } /* * If this isn't the very last instruction, branch to * the epilogue if we've stuff to clean up. Otherwise, * if there's nothing to tidy, just return. If we /are/ * the last instruction, we're about to fall through to * the epilogue to return. */ if (i != flen - 1) { /* * Note: 'seen' is properly valid only on pass * #2. Both parts of this conditional are the * same instruction size though, meaning the * first pass will still correctly determine the * code size/addresses. */ if (ctx->seen) PPC_JMP(exit_addr); else PPC_BLR(); } break; case BPF_RET | BPF_A: PPC_MR(r_ret, r_A); if (i != flen - 1) { if (ctx->seen) PPC_JMP(exit_addr); else PPC_BLR(); } break; case BPF_MISC | BPF_TAX: /* X = A */ PPC_MR(r_X, r_A); break; case BP