2013-08-20 23:31:14 +00:00
|
|
|
/*
|
|
|
|
* SSE Convolution
|
|
|
|
* Copyright (C) 2012, 2013 Thomas Tsou <tom@tsou.cc>
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with this library; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <malloc.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <stdio.h>
|
2013-10-31 01:24:40 +00:00
|
|
|
#include "convolve.h"
|
2013-08-20 23:31:14 +00:00
|
|
|
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
|
|
#include "config.h"
|
|
|
|
#endif
|
|
|
|
|
2013-10-31 01:24:40 +00:00
|
|
|
/* Forward declarations from base implementation */
|
2015-03-25 19:55:11 +00:00
|
|
|
int _base_convolve_real(const float *x, int x_len,
|
|
|
|
const float *h, int h_len,
|
2013-10-31 01:24:40 +00:00
|
|
|
float *y, int y_len,
|
|
|
|
int start, int len,
|
|
|
|
int step, int offset);
|
|
|
|
|
2015-03-25 19:55:11 +00:00
|
|
|
int _base_convolve_complex(const float *x, int x_len,
|
|
|
|
const float *h, int h_len,
|
2013-10-31 01:24:40 +00:00
|
|
|
float *y, int y_len,
|
|
|
|
int start, int len,
|
|
|
|
int step, int offset);
|
|
|
|
|
|
|
|
int bounds_check(int x_len, int h_len, int y_len,
|
|
|
|
int start, int len, int step);
|
|
|
|
|
2013-08-20 23:31:14 +00:00
|
|
|
#ifdef HAVE_SSE3
|
|
|
|
#include <xmmintrin.h>
|
|
|
|
#include <pmmintrin.h>
|
|
|
|
|
|
|
|
/* 4-tap SSE complex-real convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_real4(const float *restrict x,
|
|
|
|
const float *restrict h,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *restrict y,
|
|
|
|
int len)
|
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7;
|
|
|
|
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[0]);
|
|
|
|
m1 = _mm_load_ps(&h[4]);
|
|
|
|
m7 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 4]);
|
|
|
|
m2 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m3 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m4 = _mm_mul_ps(m2, m7);
|
|
|
|
m5 = _mm_mul_ps(m3, m7);
|
|
|
|
|
|
|
|
/* Sum and store */
|
|
|
|
m6 = _mm_hadd_ps(m4, m5);
|
|
|
|
m0 = _mm_hadd_ps(m6, m6);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m0);
|
|
|
|
m0 = _mm_shuffle_ps(m0, m0, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 8-tap SSE complex-real convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_real8(const float *restrict x,
|
|
|
|
const float *restrict h,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *restrict y,
|
|
|
|
int len)
|
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7, m8, m9;
|
|
|
|
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[0]);
|
|
|
|
m1 = _mm_load_ps(&h[4]);
|
|
|
|
m2 = _mm_load_ps(&h[8]);
|
|
|
|
m3 = _mm_load_ps(&h[12]);
|
|
|
|
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 4]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 8]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 12]);
|
|
|
|
|
|
|
|
m6 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m7 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m8 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m9 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m6 = _mm_mul_ps(m6, m4);
|
|
|
|
m7 = _mm_mul_ps(m7, m4);
|
|
|
|
m8 = _mm_mul_ps(m8, m5);
|
|
|
|
m9 = _mm_mul_ps(m9, m5);
|
|
|
|
|
|
|
|
/* Sum and store */
|
|
|
|
m6 = _mm_add_ps(m6, m8);
|
|
|
|
m7 = _mm_add_ps(m7, m9);
|
|
|
|
m6 = _mm_hadd_ps(m6, m7);
|
|
|
|
m6 = _mm_hadd_ps(m6, m6);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m6);
|
|
|
|
m6 = _mm_shuffle_ps(m6, m6, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m6);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 12-tap SSE complex-real convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_real12(const float *restrict x,
|
|
|
|
const float *restrict h,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *restrict y,
|
|
|
|
int len)
|
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7;
|
|
|
|
__m128 m8, m9, m10, m11, m12, m13, m14;
|
|
|
|
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[0]);
|
|
|
|
m1 = _mm_load_ps(&h[4]);
|
|
|
|
m2 = _mm_load_ps(&h[8]);
|
|
|
|
m3 = _mm_load_ps(&h[12]);
|
|
|
|
m4 = _mm_load_ps(&h[16]);
|
|
|
|
m5 = _mm_load_ps(&h[20]);
|
|
|
|
|
|
|
|
m12 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m13 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m14 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 4]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 8]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 12]);
|
|
|
|
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m6 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m7 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 16]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 20]);
|
|
|
|
|
|
|
|
m8 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m9 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m0 = _mm_mul_ps(m4, m12);
|
|
|
|
m1 = _mm_mul_ps(m5, m12);
|
|
|
|
m2 = _mm_mul_ps(m6, m13);
|
|
|
|
m3 = _mm_mul_ps(m7, m13);
|
|
|
|
m4 = _mm_mul_ps(m8, m14);
|
|
|
|
m5 = _mm_mul_ps(m9, m14);
|
|
|
|
|
|
|
|
/* Sum and store */
|
|
|
|
m8 = _mm_add_ps(m0, m2);
|
|
|
|
m9 = _mm_add_ps(m1, m3);
|
|
|
|
m10 = _mm_add_ps(m8, m4);
|
|
|
|
m11 = _mm_add_ps(m9, m5);
|
|
|
|
|
|
|
|
m2 = _mm_hadd_ps(m10, m11);
|
|
|
|
m3 = _mm_hadd_ps(m2, m2);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m3);
|
|
|
|
m3 = _mm_shuffle_ps(m3, m3, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m3);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 16-tap SSE complex-real convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_real16(const float *restrict x,
|
|
|
|
const float *restrict h,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *restrict y,
|
|
|
|
int len)
|
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7;
|
|
|
|
__m128 m8, m9, m10, m11, m12, m13, m14, m15;
|
|
|
|
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[0]);
|
|
|
|
m1 = _mm_load_ps(&h[4]);
|
|
|
|
m2 = _mm_load_ps(&h[8]);
|
|
|
|
m3 = _mm_load_ps(&h[12]);
|
|
|
|
|
|
|
|
m4 = _mm_load_ps(&h[16]);
|
|
|
|
m5 = _mm_load_ps(&h[20]);
|
|
|
|
m6 = _mm_load_ps(&h[24]);
|
|
|
|
m7 = _mm_load_ps(&h[28]);
|
|
|
|
|
|
|
|
m12 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m13 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m14 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m15 = _mm_shuffle_ps(m6, m7, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 4]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 8]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 12]);
|
|
|
|
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m6 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m7 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 16]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 20]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 24]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 28]);
|
|
|
|
|
|
|
|
m8 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m9 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m10 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m11 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m0 = _mm_mul_ps(m4, m12);
|
|
|
|
m1 = _mm_mul_ps(m5, m12);
|
|
|
|
m2 = _mm_mul_ps(m6, m13);
|
|
|
|
m3 = _mm_mul_ps(m7, m13);
|
|
|
|
|
|
|
|
m4 = _mm_mul_ps(m8, m14);
|
|
|
|
m5 = _mm_mul_ps(m9, m14);
|
|
|
|
m6 = _mm_mul_ps(m10, m15);
|
|
|
|
m7 = _mm_mul_ps(m11, m15);
|
|
|
|
|
|
|
|
/* Sum and store */
|
|
|
|
m8 = _mm_add_ps(m0, m2);
|
|
|
|
m9 = _mm_add_ps(m1, m3);
|
|
|
|
m10 = _mm_add_ps(m4, m6);
|
|
|
|
m11 = _mm_add_ps(m5, m7);
|
|
|
|
|
|
|
|
m0 = _mm_add_ps(m8, m10);
|
|
|
|
m1 = _mm_add_ps(m9, m11);
|
|
|
|
m2 = _mm_hadd_ps(m0, m1);
|
|
|
|
m3 = _mm_hadd_ps(m2, m2);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m3);
|
|
|
|
m3 = _mm_shuffle_ps(m3, m3, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m3);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 20-tap SSE complex-real convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_real20(const float *restrict x,
|
|
|
|
const float *restrict h,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *restrict y,
|
|
|
|
int len)
|
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7;
|
|
|
|
__m128 m8, m9, m11, m12, m13, m14, m15;
|
|
|
|
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[0]);
|
|
|
|
m1 = _mm_load_ps(&h[4]);
|
|
|
|
m2 = _mm_load_ps(&h[8]);
|
|
|
|
m3 = _mm_load_ps(&h[12]);
|
|
|
|
m4 = _mm_load_ps(&h[16]);
|
|
|
|
m5 = _mm_load_ps(&h[20]);
|
|
|
|
m6 = _mm_load_ps(&h[24]);
|
|
|
|
m7 = _mm_load_ps(&h[28]);
|
|
|
|
m8 = _mm_load_ps(&h[32]);
|
|
|
|
m9 = _mm_load_ps(&h[36]);
|
|
|
|
|
|
|
|
m11 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m12 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m13 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m14 = _mm_shuffle_ps(m6, m7, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m15 = _mm_shuffle_ps(m8, m9, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Multiply-accumulate first 12 taps */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 4]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 8]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 12]);
|
|
|
|
m4 = _mm_loadu_ps(&x[2 * i + 16]);
|
|
|
|
m5 = _mm_loadu_ps(&x[2 * i + 20]);
|
|
|
|
|
|
|
|
m6 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m7 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m8 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m9 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m0 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m1 = _mm_shuffle_ps(m4, m5, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
m2 = _mm_mul_ps(m6, m11);
|
|
|
|
m3 = _mm_mul_ps(m7, m11);
|
|
|
|
m4 = _mm_mul_ps(m8, m12);
|
|
|
|
m5 = _mm_mul_ps(m9, m12);
|
|
|
|
m6 = _mm_mul_ps(m0, m13);
|
|
|
|
m7 = _mm_mul_ps(m1, m13);
|
|
|
|
|
|
|
|
m0 = _mm_add_ps(m2, m4);
|
|
|
|
m1 = _mm_add_ps(m3, m5);
|
|
|
|
m8 = _mm_add_ps(m0, m6);
|
|
|
|
m9 = _mm_add_ps(m1, m7);
|
|
|
|
|
|
|
|
/* Multiply-accumulate last 8 taps */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 24]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 28]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 32]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 36]);
|
|
|
|
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m6 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m7 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
m0 = _mm_mul_ps(m4, m14);
|
|
|
|
m1 = _mm_mul_ps(m5, m14);
|
|
|
|
m2 = _mm_mul_ps(m6, m15);
|
|
|
|
m3 = _mm_mul_ps(m7, m15);
|
|
|
|
|
|
|
|
m4 = _mm_add_ps(m0, m2);
|
|
|
|
m5 = _mm_add_ps(m1, m3);
|
|
|
|
|
|
|
|
/* Final sum and store */
|
|
|
|
m0 = _mm_add_ps(m8, m4);
|
|
|
|
m1 = _mm_add_ps(m9, m5);
|
|
|
|
m2 = _mm_hadd_ps(m0, m1);
|
|
|
|
m3 = _mm_hadd_ps(m2, m2);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m3);
|
|
|
|
m3 = _mm_shuffle_ps(m3, m3, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m3);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 4*N-tap SSE complex-real convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_real4n(const float *x,
|
|
|
|
const float *h,
|
|
|
|
float *y,
|
|
|
|
int h_len, int len)
|
2013-08-20 23:31:14 +00:00
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m4, m5, m6, m7;
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Zero */
|
|
|
|
m6 = _mm_setzero_ps();
|
|
|
|
m7 = _mm_setzero_ps();
|
|
|
|
|
|
|
|
for (int n = 0; n < h_len / 4; n++) {
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[8 * n + 0]);
|
|
|
|
m1 = _mm_load_ps(&h[8 * n + 4]);
|
|
|
|
m2 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 8 * n + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 8 * n + 4]);
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m0 = _mm_mul_ps(m2, m4);
|
|
|
|
m1 = _mm_mul_ps(m2, m5);
|
|
|
|
|
|
|
|
/* Accumulate */
|
|
|
|
m6 = _mm_add_ps(m6, m0);
|
|
|
|
m7 = _mm_add_ps(m7, m1);
|
|
|
|
}
|
|
|
|
|
|
|
|
m0 = _mm_hadd_ps(m6, m7);
|
|
|
|
m0 = _mm_hadd_ps(m0, m0);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m0);
|
|
|
|
m0 = _mm_shuffle_ps(m0, m0, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 4*N-tap SSE complex-complex convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_cmplx_4n(const float *x,
|
|
|
|
const float *h,
|
|
|
|
float *y,
|
|
|
|
int h_len, int len)
|
2013-08-20 23:31:14 +00:00
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7;
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Zero */
|
|
|
|
m6 = _mm_setzero_ps();
|
|
|
|
m7 = _mm_setzero_ps();
|
|
|
|
|
|
|
|
for (int n = 0; n < h_len / 4; n++) {
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[8 * n + 0]);
|
|
|
|
m1 = _mm_load_ps(&h[8 * n + 4]);
|
|
|
|
m2 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m3 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 8 * n + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 8 * n + 4]);
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m0 = _mm_mul_ps(m2, m4);
|
|
|
|
m1 = _mm_mul_ps(m3, m5);
|
|
|
|
|
|
|
|
m2 = _mm_mul_ps(m2, m5);
|
|
|
|
m3 = _mm_mul_ps(m3, m4);
|
|
|
|
|
|
|
|
/* Sum */
|
|
|
|
m0 = _mm_sub_ps(m0, m1);
|
|
|
|
m2 = _mm_add_ps(m2, m3);
|
|
|
|
|
|
|
|
/* Accumulate */
|
|
|
|
m6 = _mm_add_ps(m6, m0);
|
|
|
|
m7 = _mm_add_ps(m7, m2);
|
|
|
|
}
|
|
|
|
|
|
|
|
m0 = _mm_hadd_ps(m6, m7);
|
|
|
|
m0 = _mm_hadd_ps(m0, m0);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m0);
|
|
|
|
m0 = _mm_shuffle_ps(m0, m0, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* 8*N-tap SSE complex-complex convolution */
|
2015-03-25 19:55:11 +00:00
|
|
|
static void sse_conv_cmplx_8n(const float *x,
|
|
|
|
const float *h,
|
|
|
|
float *y,
|
|
|
|
int h_len, int len)
|
2013-08-20 23:31:14 +00:00
|
|
|
{
|
|
|
|
__m128 m0, m1, m2, m3, m4, m5, m6, m7;
|
|
|
|
__m128 m8, m9, m10, m11, m12, m13, m14, m15;
|
|
|
|
|
|
|
|
for (int i = 0; i < len; i++) {
|
|
|
|
/* Zero */
|
|
|
|
m12 = _mm_setzero_ps();
|
|
|
|
m13 = _mm_setzero_ps();
|
|
|
|
m14 = _mm_setzero_ps();
|
|
|
|
m15 = _mm_setzero_ps();
|
|
|
|
|
|
|
|
for (int n = 0; n < h_len / 8; n++) {
|
|
|
|
/* Load (aligned) filter taps */
|
|
|
|
m0 = _mm_load_ps(&h[16 * n + 0]);
|
|
|
|
m1 = _mm_load_ps(&h[16 * n + 4]);
|
|
|
|
m2 = _mm_load_ps(&h[16 * n + 8]);
|
|
|
|
m3 = _mm_load_ps(&h[16 * n + 12]);
|
|
|
|
|
|
|
|
m4 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m5 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
2014-05-01 06:15:51 +00:00
|
|
|
m6 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
2013-08-20 23:31:14 +00:00
|
|
|
m7 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Load (unaligned) input data */
|
|
|
|
m0 = _mm_loadu_ps(&x[2 * i + 16 * n + 0]);
|
|
|
|
m1 = _mm_loadu_ps(&x[2 * i + 16 * n + 4]);
|
|
|
|
m2 = _mm_loadu_ps(&x[2 * i + 16 * n + 8]);
|
|
|
|
m3 = _mm_loadu_ps(&x[2 * i + 16 * n + 12]);
|
|
|
|
|
|
|
|
m8 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m9 = _mm_shuffle_ps(m0, m1, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
m10 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(0, 2, 0, 2));
|
|
|
|
m11 = _mm_shuffle_ps(m2, m3, _MM_SHUFFLE(1, 3, 1, 3));
|
|
|
|
|
|
|
|
/* Quad multiply */
|
|
|
|
m0 = _mm_mul_ps(m4, m8);
|
|
|
|
m1 = _mm_mul_ps(m5, m9);
|
|
|
|
m2 = _mm_mul_ps(m6, m10);
|
|
|
|
m3 = _mm_mul_ps(m7, m11);
|
|
|
|
|
|
|
|
m4 = _mm_mul_ps(m4, m9);
|
|
|
|
m5 = _mm_mul_ps(m5, m8);
|
|
|
|
m6 = _mm_mul_ps(m6, m11);
|
|
|
|
m7 = _mm_mul_ps(m7, m10);
|
|
|
|
|
|
|
|
/* Sum */
|
|
|
|
m0 = _mm_sub_ps(m0, m1);
|
|
|
|
m2 = _mm_sub_ps(m2, m3);
|
|
|
|
m4 = _mm_add_ps(m4, m5);
|
|
|
|
m6 = _mm_add_ps(m6, m7);
|
|
|
|
|
|
|
|
/* Accumulate */
|
|
|
|
m12 = _mm_add_ps(m12, m0);
|
|
|
|
m13 = _mm_add_ps(m13, m2);
|
|
|
|
m14 = _mm_add_ps(m14, m4);
|
|
|
|
m15 = _mm_add_ps(m15, m6);
|
|
|
|
}
|
|
|
|
|
|
|
|
m0 = _mm_add_ps(m12, m13);
|
|
|
|
m1 = _mm_add_ps(m14, m15);
|
|
|
|
m2 = _mm_hadd_ps(m0, m1);
|
|
|
|
m2 = _mm_hadd_ps(m2, m2);
|
|
|
|
|
|
|
|
_mm_store_ss(&y[2 * i + 0], m2);
|
|
|
|
m2 = _mm_shuffle_ps(m2, m2, _MM_SHUFFLE(0, 3, 2, 1));
|
|
|
|
_mm_store_ss(&y[2 * i + 1], m2);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* API: Aligned complex-real */
|
2015-03-25 19:55:11 +00:00
|
|
|
int convolve_real(const float *x, int x_len,
|
|
|
|
const float *h, int h_len,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *y, int y_len,
|
|
|
|
int start, int len,
|
|
|
|
int step, int offset)
|
|
|
|
{
|
2015-03-25 19:55:11 +00:00
|
|
|
void (*conv_func)(const float *, const float *,
|
|
|
|
float *, int) = NULL;
|
|
|
|
void (*conv_func_n)(const float *, const float *,
|
|
|
|
float *, int, int) = NULL;
|
2013-08-20 23:31:14 +00:00
|
|
|
|
|
|
|
if (bounds_check(x_len, h_len, y_len, start, len, step) < 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
memset(y, 0, len * 2 * sizeof(float));
|
|
|
|
|
|
|
|
#ifdef HAVE_SSE3
|
|
|
|
if (step <= 4) {
|
|
|
|
switch (h_len) {
|
|
|
|
case 4:
|
|
|
|
conv_func = sse_conv_real4;
|
|
|
|
break;
|
|
|
|
case 8:
|
|
|
|
conv_func = sse_conv_real8;
|
|
|
|
break;
|
|
|
|
case 12:
|
|
|
|
conv_func = sse_conv_real12;
|
|
|
|
break;
|
|
|
|
case 16:
|
|
|
|
conv_func = sse_conv_real16;
|
|
|
|
break;
|
|
|
|
case 20:
|
|
|
|
conv_func = sse_conv_real20;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
if (!(h_len % 4))
|
|
|
|
conv_func_n = sse_conv_real4n;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
if (conv_func) {
|
|
|
|
conv_func(&x[2 * (-(h_len - 1) + start)],
|
|
|
|
h, y, len);
|
|
|
|
} else if (conv_func_n) {
|
|
|
|
conv_func_n(&x[2 * (-(h_len - 1) + start)],
|
|
|
|
h, y, h_len, len);
|
|
|
|
} else {
|
|
|
|
_base_convolve_real(x, x_len,
|
|
|
|
h, h_len,
|
|
|
|
y, y_len,
|
|
|
|
start, len, step, offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
return len;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* API: Aligned complex-complex */
|
2015-03-25 19:55:11 +00:00
|
|
|
int convolve_complex(const float *x, int x_len,
|
|
|
|
const float *h, int h_len,
|
2013-08-20 23:31:14 +00:00
|
|
|
float *y, int y_len,
|
|
|
|
int start, int len,
|
|
|
|
int step, int offset)
|
|
|
|
{
|
2015-03-25 19:55:11 +00:00
|
|
|
void (*conv_func)(const float *, const float *,
|
|
|
|
float *, int, int) = NULL;
|
2013-08-20 23:31:14 +00:00
|
|
|
|
|
|
|
if (bounds_check(x_len, h_len, y_len, start, len, step) < 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
memset(y, 0, len * 2 * sizeof(float));
|
|
|
|
|
|
|
|
#ifdef HAVE_SSE3
|
|
|
|
if (step <= 4) {
|
|
|
|
if (!(h_len % 8))
|
|
|
|
conv_func = sse_conv_cmplx_8n;
|
|
|
|
else if (!(h_len % 4))
|
|
|
|
conv_func = sse_conv_cmplx_4n;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
if (conv_func) {
|
|
|
|
conv_func(&x[2 * (-(h_len - 1) + start)],
|
|
|
|
h, y, h_len, len);
|
|
|
|
} else {
|
|
|
|
_base_convolve_complex(x, x_len,
|
|
|
|
h, h_len,
|
|
|
|
y, y_len,
|
|
|
|
start, len, step, offset);
|
|
|
|
}
|
|
|
|
|
|
|
|
return len;
|
|
|
|
}
|