Open BTS 开源代码

/* * SSE Convolution * Copyright (C) 2012, 2013 Thomas Tsou <tom@tsou.cc> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with this library; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include <malloc.h> #include <string.h> #include <stdio.h> #ifdef HAVE_CONFIG_H #include "config.h" #endif #ifdef HAVE_SSE3 #include <xmmintrin.h> #include <pmmintrin.h> /* 4-tap SSE complex-real convolution */ static void sse_conv_real4(float *restrict x, float *restrict h, float *restrict y, int len) { __m128 m0, m1, m2, m3, m4, m5, m6, m7; /* Load (aligned) filter taps */ m0 = _mm_load_ps(&h[0]); m1 = _mm_load_ps(&h[4]); m7 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); for (int i = 0; i < len; i++) { /* Load (unaligned) input data */ m0 = _mm_loadu_ps(&x[2 * i + 0]); m1 = _mm_loadu_ps(&x[2 * i + 4]); m2 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m3 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3)); /* Quad multiply */ m4 = _mm_mul_ps(m2, m7); m5 = _mm_mul_ps(m3, m7); /* Sum and store */ m6 = _mm_hadd_ps(m4, m5); m0 = _mm_hadd_ps(m6, m6); _mm_store_ss(&y[2 * i + 0], m0); m0 = _mm_shuffle_ps(m0, m0, _MM_SHUFFLE(0, 3, 2, 1)); _mm_store_ss(&y[2 * i + 1], m0); } } /* 8-tap SSE complex-real convolution */ static void sse_conv_real8(float *restrict x, float *restrict h, float *restrict y, int len) { __m128 m0, m1, m2, m3, m4, m5, m6, m7, m8, m9; /* Load (aligned) filter taps */ m0 = _mm_load_ps(&h[0]); m1 = _mm_load_ps(&h[4]); m2 = _mm_load_ps(&h[8]); m3 = _mm_load_ps(&h[12]); m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m5 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); for (int i = 0; i < len; i++) { /* Load (unaligned) input data */ m0 = _mm_loadu_ps(&x[2 * i + 0]); m1 = _mm_loadu_ps(&x[2 * i + 4]); m2 = _mm_loadu_ps(&x[2 * i + 8]); m3 = _mm_loadu_ps(&x[2 * i + 12]); m6 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m7 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3)); m8 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m9 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3)); /* Quad multiply */ m6 = _mm_mul_ps(m6, m4); m7 = _mm_mul_ps(m7, m4); m8 = _mm_mul_ps(m8, m5); m9 = _mm_mul_ps(m9, m5); /* Sum and store */ m6 = _mm_add_ps(m6, m8); m7 = _mm_add_ps(m7, m9); m6 = _mm_hadd_ps(m6, m7); m6 = _mm_hadd_ps(m6, m6); _mm_store_ss(&y[2 * i + 0], m6); m6 = _mm_shuffle_ps(m6, m6, _MM_SHUFFLE(0, 3, 2, 1)); _mm_store_ss(&y[2 * i + 1], m6); } } /* 12-tap SSE complex-real convolution */ static void sse_conv_real12(float *restrict x, float *restrict h, float *restrict y, int len) { __m128 m0, m1, m2, m3, m4, m5, m6, m7; __m128 m8, m9, m10, m11, m12, m13, m14; /* Load (aligned) filter taps */ m0 = _mm_load_ps(&h[0]); m1 = _mm_load_ps(&h[4]); m2 = _mm_load_ps(&h[8]); m3 = _mm_load_ps(&h[12]); m4 = _mm_load_ps(&h[16]); m5 = _mm_load_ps(&h[20]); m12 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m13 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m14 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2)); for (int i = 0; i < len; i++) { /* Load (unaligned) input data */ m0 = _mm_loadu_ps(&x[2 * i + 0]); m1 = _mm_loadu_ps(&x[2 * i + 4]); m2 = _mm_loadu_ps(&x[2 * i + 8]); m3 = _mm_loadu_ps(&x[2 * i + 12]); m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3)); m6 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m7 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3)); m0 = _mm_loadu_ps(&x[2 * i + 16]); m1 = _mm_loadu_ps(&x[2 * i + 20]); m8 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m9 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3)); /* Quad multiply */ m0 = _mm_mul_ps(m4, m12); m1 = _mm_mul_ps(m5, m12); m2 = _mm_mul_ps(m6, m13); m3 = _mm_mul_ps(m7, m13); m4 = _mm_mul_ps(m8, m14); m5 = _mm_mul_ps(m9, m14); /* Sum and store */ m8 = _mm_add_ps(m0, m2); m9 = _mm_add_ps(m1, m3); m10 = _mm_add_ps(m8, m4); m11 = _mm_add_ps(m9, m5); m2 = _mm_hadd_ps(m10, m11); m3 = _mm_hadd_ps(m2, m2); _mm_store_ss(&y[2 * i + 0], m3); m3 = _mm_shuffle_ps(m3, m3, _MM_SHUFFLE(0, 3, 2, 1)); _mm_store_ss(&y[2 * i + 1], m3); } } /* 16-tap SSE complex-real convolution */ static void sse_conv_real16(float *restrict x, float *restrict h, float *restrict y, int len) { __m128 m0, m1, m2, m3, m4, m5, m6, m7; __m128 m8, m9, m10, m11, m12, m13, m14, m15; /* Load (aligned) filter taps */ m0 = _mm_load_ps(&h[0]); m1 = _mm_load_ps(&h[4]); m2 = _mm_load_ps(&h[8]); m3 = _mm_load_ps(&h[12]); m4 = _mm_load_ps(&h[16]); m5 = _mm_load_ps(&h[20]); m6 = _mm_load_ps(&h[24]); m7 = _mm_load_ps(&h[28]); m12 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m13 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m14 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2)); m15 = _mm_shuffle_ps(m6, m7, _MM_SHUFFLE(0, 2, 0, 2)); for (int i = 0; i < len; i++) { /* Load (unaligned) input data */ m0 = _mm_loadu_ps(&x[2 * i + 0]); m1 = _mm_loadu_ps(&x[2 * i + 4]); m2 = _mm_loadu_ps(&x[2 * i + 8]); m3 = _mm_loadu_ps(&x[2 * i + 12]); m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3)); m6 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m7 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3)); m0 = _mm_loadu_ps(&x[2 * i + 16]); m1 = _mm_loadu_ps(&x[2 * i + 20]); m2 = _mm_loadu_ps(&x[2 * i + 24]); m3 = _mm_loadu_ps(&x[2 * i + 28]); m8 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m9 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3)); m10 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m11 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3)); /* Quad multiply */ m0 = _mm_mul_ps(m4, m12); m1 = _mm_mul_ps(m5, m12); m2 = _mm_mul_ps(m6, m13); m3 = _mm_mul_ps(m7, m13); m4 = _mm_mul_ps(m8, m14); m5 = _mm_mul_ps(m9, m14); m6 = _mm_mul_ps(m10, m15); m7 = _mm_mul_ps(m11, m15); /* Sum and store */ m8 = _mm_add_ps(m0, m2); m9 = _mm_add_ps(m1, m3); m10 = _mm_add_ps(m4, m6); m11 = _mm_add_ps(m5, m7); m0 = _mm_add_ps(m8, m10); m1 = _mm_add_ps(m9, m11); m2 = _mm_hadd_ps(m0, m1); m3 = _mm_hadd_ps(m2, m2); _mm_store_ss(&y[2 * i + 0], m3); m3 = _mm_shuffle_ps(m3, m3, _MM_SHUFFLE(0, 3, 2, 1)); _mm_store_ss(&y[2 * i + 1], m3); } } /* 20-tap SSE complex-real convolution */ static void sse_conv_real20(float *restrict x, float *restrict h, float *restrict y, int len) { __m128 m0, m1, m2, m3, m4, m5, m6, m7; __m128 m8, m9, m11, m12, m13, m14, m15; /* Load (aligned) filter taps */ m0 = _mm_load_ps(&h[0]); m1 = _mm_load_ps(&h[4]); m2 = _mm_load_ps(&h[8]); m3 = _mm_load_ps(&h[12]); m4 = _mm_load_ps(&h[16]); m5 = _mm_load_ps(&h[20]); m6 = _mm_load_ps(&h[24]); m7 = _mm_load_ps(&h[28]); m8 = _mm_load_ps(&h[32]); m9 = _mm_load_ps(&h[36]); m11 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2)); m12 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2)); m13 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2)); m14 = _mm_shuffle_ps(m6, m7, _MM_SHUFFLE(0, 2, 0, 2)); m15 = _mm_shuffle_ps(m8, m9, _MM_SHUFFLE(0, 2, 0, 2)); for (int i = 0; i < len; i++) { /* Multiply-accumulate first 12 taps */ m0 = _mm_loadu_ps(