dustfft.c

/****************************************************************************\
* DustFFT - (c) Copyright pihlaja@signaldust.com 2014                        *
*----------------------------------------------------------------------------*
* You can use and/or redistribute this for whatever purpose, free of charge, *
* provided that the above copyright notice and this permission notice appear *
* in all copies of the software or it's associated documentation.            *
*                                                                            *
* THIS SOFTWARE IS PROVIDED "AS-IS" WITHOUT ANY WARRANTY. USE AT YOUR OWN    *
* RISK. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE HELD LIABLE FOR ANYTHING.  *
*----------------------------------------------------------------------------*
*                                                                            *
* This file is generated by a script, direct editing is a terrible idea.     *
\****************************************************************************/
#ifndef DUSTFFT_H
#define DUSTFFT_H
# ifdef __cplusplus
extern "C" {
# endif

/* Main interface:
   Use SSE2 packed processing for 16-byte aligned buffers (faster).
   For unaligned buffers, falls back to slower scalar code.
*/
void DustFFT_fwdD(double * buf, unsigned n);
void DustFFT_revD(double * buf, unsigned n);

/* These assume 16 bytes alignment without checking anything.
   Use these if you want to crash (for debug/testing) on unaligned buffers.
*/
void DustFFT_fwdDa(double * buf, unsigned n);
void DustFFT_revDa(double * buf, unsigned n);

/* These do real to half-complex FFT and half-complex to real IFFT.
   Implemented as simple wrappers around half-size complex FFT.
*/
void DustFFT_r2cD(double * buf, unsigned n);
void DustFFT_c2rD(double * buf, unsigned n);

# ifdef __cplusplus
}; /* extern C */
# endif
#endif /* DUSTFFT_H */

/* -- cut header here -- implementation follows --- */
#ifndef DUSTFFT_HEADER_ONLY

#ifdef _MSC_VER
#  include <stddef.h> /* for uintptr_t */
typedef unsigned uint32_t;
#else /* _MSC_VER  */
#  include <stdint.h> /* for uintptr_t */
#endif /* _MSC_VER */

#include <math.h> /* sin() */
#include <emmintrin.h>
#ifdef WIN32
# define ALIGN16 __declspec(align(16))
#else
# define ALIGN16 __attribute__ ((aligned (16)))
#endif

static const double _constant_pi = 3.141592653589793;
static const uint32_t ALIGN16 _mask_pd_sign_lo[4] = { 0, 1U << 31, 0, 0 };
static const double ALIGN16 _constant_sqrt05[2] = { 0.7071067811865476, 0.7071067811865476 };
static const double ALIGN16 _constant_cos8[2] = { 0.9238795325112867, 0.9238795325112867 };
static const double ALIGN16 _constant_sin8[2] = { 0.3826834323650898, 0.3826834323650898 };


    

static void _br_sd(double * buf, unsigned n)
{
  switch(n)
  {
  case 1:
  case 2: return;
  case 4: 
    { double r,i;
      r = buf[4]; i = buf[1+4];
      buf[4] = buf[2]; buf[1+4] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    return;
  case 8:
    { double r,i;
      r = buf[8]; i = buf[1+8];
      buf[8] = buf[2]; buf[1+8] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    { double r,i;
      r = buf[12]; i = buf[1+12];
      buf[12] = buf[6]; buf[1+12] = buf[1+6];
      buf[6] = r; buf[1+6] = i;}
      
    return;
  case 16:
    { double r,i;
      r = buf[8]; i = buf[1+8];
      buf[8] = buf[4]; buf[1+8] = buf[1+4];
      buf[4] = r; buf[1+4] = i;}
      
    { double r,i;
      r = buf[16]; i = buf[1+16];
      buf[16] = buf[2]; buf[1+16] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    { double r,i;
      r = buf[20]; i = buf[1+20];
      buf[20] = buf[10]; buf[1+20] = buf[1+10];
      buf[10] = r; buf[1+10] = i;}
      
    { double r,i;
      r = buf[24]; i = buf[1+24];
      buf[24] = buf[6]; buf[1+24] = buf[1+6];
      buf[6] = r; buf[1+6] = i;}
      
    { double r,i;
      r = buf[26]; i = buf[1+26];
      buf[26] = buf[22]; buf[1+26] = buf[1+22];
      buf[22] = r; buf[1+22] = i;}
      
    { double r,i;
      r = buf[28]; i = buf[1+28];
      buf[28] = buf[14]; buf[1+28] = buf[1+14];
      buf[14] = r; buf[1+14] = i;}
      
    return;
    
  case 32:
    { double r,i;
      r = buf[16]; i = buf[1+16];
      buf[16] = buf[4]; buf[1+16] = buf[1+4];
      buf[4] = r; buf[1+4] = i;}
      
    { double r,i;
      r = buf[24]; i = buf[1+24];
      buf[24] = buf[12]; buf[1+24] = buf[1+12];
      buf[12] = r; buf[1+12] = i;}
      
    { double r,i;
      r = buf[32]; i = buf[1+32];
      buf[32] = buf[2]; buf[1+32] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    { double r,i;
      r = buf[36]; i = buf[1+36];
      buf[36] = buf[18]; buf[1+36] = buf[1+18];
      buf[18] = r; buf[1+18] = i;}
      
    { double r,i;
      r = buf[40]; i = buf[1+40];
      buf[40] = buf[10]; buf[1+40] = buf[1+10];
      buf[10] = r; buf[1+10] = i;}
      
    { double r,i;
      r = buf[44]; i = buf[1+44];
      buf[44] = buf[26]; buf[1+44] = buf[1+26];
      buf[26] = r; buf[1+26] = i;}
      
    { double r,i;
      r = buf[48]; i = buf[1+48];
      buf[48] = buf[6]; buf[1+48] = buf[1+6];
      buf[6] = r; buf[1+6] = i;}
      
    { double r,i;
      r = buf[50]; i = buf[1+50];
      buf[50] = buf[38]; buf[1+50] = buf[1+38];
      buf[38] = r; buf[1+38] = i;}
      
    { double r,i;
      r = buf[52]; i = buf[1+52];
      buf[52] = buf[22]; buf[1+52] = buf[1+22];
      buf[22] = r; buf[1+22] = i;}
      
    { double r,i;
      r = buf[56]; i = buf[1+56];
      buf[56] = buf[14]; buf[1+56] = buf[1+14];
      buf[14] = r; buf[1+14] = i;}
      
    { double r,i;
      r = buf[58]; i = buf[1+58];
      buf[58] = buf[46]; buf[1+58] = buf[1+46];
      buf[46] = r; buf[1+46] = i;}
      
    { double r,i;
      r = buf[60]; i = buf[1+60];
      buf[60] = buf[30]; buf[1+60] = buf[1+30];
      buf[30] = r; buf[1+30] = i;}
      
    return;
  case 64:
    { double r,i;
      r = buf[16]; i = buf[1+16];
      buf[16] = buf[8]; buf[1+16] = buf[1+8];
      buf[8] = r; buf[1+8] = i;}
      
    { double r,i;
      r = buf[32]; i = buf[1+32];
      buf[32] = buf[4]; buf[1+32] = buf[1+4];
      buf[4] = r; buf[1+4] = i;}
      
    { double r,i;
      r = buf[40]; i = buf[1+40];
      buf[40] = buf[20]; buf[1+40] = buf[1+20];
      buf[20] = r; buf[1+20] = i;}
      
    { double r,i;
      r = buf[48]; i = buf[1+48];
      buf[48] = buf[12]; buf[1+48] = buf[1+12];
      buf[12] = r; buf[1+12] = i;}
      
    { double r,i;
      r = buf[52]; i = buf[1+52];
      buf[52] = buf[44]; buf[1+52] = buf[1+44];
      buf[44] = r; buf[1+44] = i;}
      
    { double r,i;
      r = buf[56]; i = buf[1+56];
      buf[56] = buf[28]; buf[1+56] = buf[1+28];
      buf[28] = r; buf[1+28] = i;}
      
    { double r,i;
      r = buf[64]; i = buf[1+64];
      buf[64] = buf[2]; buf[1+64] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    { double r,i;
      r = buf[68]; i = buf[1+68];
      buf[68] = buf[34]; buf[1+68] = buf[1+34];
      buf[34] = r; buf[1+34] = i;}
      
    { double r,i;
      r = buf[72]; i = buf[1+72];
      buf[72] = buf[18]; buf[1+72] = buf[1+18];
      buf[18] = r; buf[1+18] = i;}
      
    { double r,i;
      r = buf[76]; i = buf[1+76];
      buf[76] = buf[50]; buf[1+76] = buf[1+50];
      buf[50] = r; buf[1+50] = i;}
      
    { double r,i;
      r = buf[80]; i = buf[1+80];
      buf[80] = buf[10]; buf[1+80] = buf[1+10];
      buf[10] = r; buf[1+10] = i;}
      
    { double r,i;
      r = buf[82]; i = buf[1+82];
      buf[82] = buf[74]; buf[1+82] = buf[1+74];
      buf[74] = r; buf[1+74] = i;}
      
    { double r,i;
      r = buf[84]; i = buf[1+84];
      buf[84] = buf[42]; buf[1+84] = buf[1+42];
      buf[42] = r; buf[1+42] = i;}
      
    { double r,i;
      r = buf[88]; i = buf[1+88];
      buf[88] = buf[26]; buf[1+88] = buf[1+26];
      buf[26] = r; buf[1+26] = i;}
      
    { double r,i;
      r = buf[92]; i = buf[1+92];
      buf[92] = buf[58]; buf[1+92] = buf[1+58];
      buf[58] = r; buf[1+58] = i;}
      
    { double r,i;
      r = buf[96]; i = buf[1+96];
      buf[96] = buf[6]; buf[1+96] = buf[1+6];
      buf[6] = r; buf[1+6] = i;}
      
    { double r,i;
      r = buf[98]; i = buf[1+98];
      buf[98] = buf[70]; buf[1+98] = buf[1+70];
      buf[70] = r; buf[1+70] = i;}
      
    { double r,i;
      r = buf[100]; i = buf[1+100];
      buf[100] = buf[38]; buf[1+100] = buf[1+38];
      buf[38] = r; buf[1+38] = i;}
      
    { double r,i;
      r = buf[104]; i = buf[1+104];
      buf[104] = buf[22]; buf[1+104] = buf[1+22];
      buf[22] = r; buf[1+22] = i;}
      
    { double r,i;
      r = buf[106]; i = buf[1+106];
      buf[106] = buf[86]; buf[1+106] = buf[1+86];
      buf[86] = r; buf[1+86] = i;}
      
    { double r,i;
      r = buf[108]; i = buf[1+108];
      buf[108] = buf[54]; buf[1+108] = buf[1+54];
      buf[54] = r; buf[1+54] = i;}
      
    { double r,i;
      r = buf[112]; i = buf[1+112];
      buf[112] = buf[14]; buf[1+112] = buf[1+14];
      buf[14] = r; buf[1+14] = i;}
      
    { double r,i;
      r = buf[114]; i = buf[1+114];
      buf[114] = buf[78]; buf[1+114] = buf[1+78];
      buf[78] = r; buf[1+78] = i;}
      
    { double r,i;
      r = buf[116]; i = buf[1+116];
      buf[116] = buf[46]; buf[1+116] = buf[1+46];
      buf[46] = r; buf[1+46] = i;}
      
    { double r,i;
      r = buf[118]; i = buf[1+118];
      buf[118] = buf[110]; buf[1+118] = buf[1+110];
      buf[110] = r; buf[1+110] = i;}
      
    { double r,i;
      r = buf[120]; i = buf[1+120];
      buf[120] = buf[30]; buf[1+120] = buf[1+30];
      buf[30] = r; buf[1+30] = i;}
      
    { double r,i;
      r = buf[122]; i = buf[1+122];
      buf[122] = buf[94]; buf[1+122] = buf[1+94];
      buf[94] = r; buf[1+94] = i;}
      
    { double r,i;
      r = buf[124]; i = buf[1+124];
      buf[124] = buf[62]; buf[1+124] = buf[1+62];
      buf[62] = r; buf[1+62] = i;}
      
    return;
  case 128:
    { double r,i;
      r = buf[32]; i = buf[1+32];
      buf[32] = buf[8]; buf[1+32] = buf[1+8];
      buf[8] = r; buf[1+8] = i;}
      
    { double r,i;
      r = buf[48]; i = buf[1+48];
      buf[48] = buf[24]; buf[1+48] = buf[1+24];
      buf[24] = r; buf[1+24] = i;}
      
    { double r,i;
      r = buf[64]; i = buf[1+64];
      buf[64] = buf[4]; buf[1+64] = buf[1+4];
      buf[4] = r; buf[1+4] = i;}
      
    { double r,i;
      r = buf[72]; i = buf[1+72];
      buf[72] = buf[36]; buf[1+72] = buf[1+36];
      buf[36] = r; buf[1+36] = i;}
      
    { double r,i;
      r = buf[80]; i = buf[1+80];
      buf[80] = buf[20]; buf[1+80] = buf[1+20];
      buf[20] = r; buf[1+20] = i;}
      
    { double r,i;
      r = buf[88]; i = buf[1+88];
      buf[88] = buf[52]; buf[1+88] = buf[1+52];
      buf[52] = r; buf[1+52] = i;}
      
    { double r,i;
      r = buf[96]; i = buf[1+96];
      buf[96] = buf[12]; buf[1+96] = buf[1+12];
      buf[12] = r; buf[1+12] = i;}
      
    { double r,i;
      r = buf[100]; i = buf[1+100];
      buf[100] = buf[76]; buf[1+100] = buf[1+76];
      buf[76] = r; buf[1+76] = i;}
      
    { double r,i;
      r = buf[104]; i = buf[1+104];
      buf[104] = buf[44]; buf[1+104] = buf[1+44];
      buf[44] = r; buf[1+44] = i;}
      
    { double r,i;
      r = buf[112]; i = buf[1+112];
      buf[112] = buf[28]; buf[1+112] = buf[1+28];
      buf[28] = r; buf[1+28] = i;}
      
    { double r,i;
      r = buf[116]; i = buf[1+116];
      buf[116] = buf[92]; buf[1+116] = buf[1+92];
      buf[92] = r; buf[1+92] = i;}
      
    { double r,i;
      r = buf[120]; i = buf[1+120];
      buf[120] = buf[60]; buf[1+120] = buf[1+60];
      buf[60] = r; buf[1+60] = i;}
      
    { double r,i;
      r = buf[128]; i = buf[1+128];
      buf[128] = buf[2]; buf[1+128] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    { double r,i;
      r = buf[132]; i = buf[1+132];
      buf[132] = buf[66]; buf[1+132] = buf[1+66];
      buf[66] = r; buf[1+66] = i;}
      
    { double r,i;
      r = buf[136]; i = buf[1+136];
      buf[136] = buf[34]; buf[1+136] = buf[1+34];
      buf[34] = r; buf[1+34] = i;}
      
    { double r,i;
      r = buf[140]; i = buf[1+140];
      buf[140] = buf[98]; buf[1+140] = buf[1+98];
      buf[98] = r; buf[1+98] = i;}
      
    { double r,i;
      r = buf[144]; i = buf[1+144];
      buf[144] = buf[18]; buf[1+144] = buf[1+18];
      buf[18] = r; buf[1+18] = i;}
      
    { double r,i;
      r = buf[148]; i = buf[1+148];
      buf[148] = buf[82]; buf[1+148] = buf[1+82];
      buf[82] = r; buf[1+82] = i;}
      
    { double r,i;
      r = buf[152]; i = buf[1+152];
      buf[152] = buf[50]; buf[1+152] = buf[1+50];
      buf[50] = r; buf[1+50] = i;}
      
    { double r,i;
      r = buf[156]; i = buf[1+156];
      buf[156] = buf[114]; buf[1+156] = buf[1+114];
      buf[114] = r; buf[1+114] = i;}
      
    { double r,i;
      r = buf[160]; i = buf[1+160];
      buf[160] = buf[10]; buf[1+160] = buf[1+10];
      buf[10] = r; buf[1+10] = i;}
      
    { double r,i;
      r = buf[162]; i = buf[1+162];
      buf[162] = buf[138]; buf[1+162] = buf[1+138];
      buf[138] = r; buf[1+138] = i;}
      
    { double r,i;
      r = buf[164]; i = buf[1+164];
      buf[164] = buf[74]; buf[1+164] = buf[1+74];
      buf[74] = r; buf[1+74] = i;}
      
    { double r,i;
      r = buf[168]; i = buf[1+168];
      buf[168] = buf[42]; buf[1+168] = buf[1+42];
      buf[42] = r; buf[1+42] = i;}
      
    { double r,i;
      r = buf[172]; i = buf[1+172];
      buf[172] = buf[106]; buf[1+172] = buf[1+106];
      buf[106] = r; buf[1+106] = i;}
      
    { double r,i;
      r = buf[176]; i = buf[1+176];
      buf[176] = buf[26]; buf[1+176] = buf[1+26];
      buf[26] = r; buf[1+26] = i;}
      
    { double r,i;
      r = buf[178]; i = buf[1+178];
      buf[178] = buf[154]; buf[1+178] = buf[1+154];
      buf[154] = r; buf[1+154] = i;}
      
    { double r,i;
      r = buf[180]; i = buf[1+180];
      buf[180] = buf[90]; buf[1+180] = buf[1+90];
      buf[90] = r; buf[1+90] = i;}
      
    { double r,i;
      r = buf[184]; i = buf[1+184];
      buf[184] = buf[58]; buf[1+184] = buf[1+58];
      buf[58] = r; buf[1+58] = i;}
      
    { double r,i;
      r = buf[188]; i = buf[1+188];
      buf[188] = buf[122]; buf[1+188] = buf[1+122];
      buf[122] = r; buf[1+122] = i;}
      
    { double r,i;
      r = buf[192]; i = buf[1+192];
      buf[192] = buf[6]; buf[1+192] = buf[1+6];
      buf[6] = r; buf[1+6] = i;}
      
    { double r,i;
      r = buf[194]; i = buf[1+194];
      buf[194] = buf[134]; buf[1+194] = buf[1+134];
      buf[134] = r; buf[1+134] = i;}
      
    { double r,i;
      r = buf[196]; i = buf[1+196];
      buf[196] = buf[70]; buf[1+196] = buf[1+70];
      buf[70] = r; buf[1+70] = i;}
      
    { double r,i;
      r = buf[200]; i = buf[1+200];
      buf[200] = buf[38]; buf[1+200] = buf[1+38];
      buf[38] = r; buf[1+38] = i;}
      
    { double r,i;
      r = buf[202]; i = buf[1+202];
      buf[202] = buf[166]; buf[1+202] = buf[1+166];
      buf[166] = r; buf[1+166] = i;}
      
    { double r,i;
      r = buf[204]; i = buf[1+204];
      buf[204] = buf[102]; buf[1+204] = buf[1+102];
      buf[102] = r; buf[1+102] = i;}
      
    { double r,i;
      r = buf[208]; i = buf[1+208];
      buf[208] = buf[22]; buf[1+208] = buf[1+22];
      buf[22] = r; buf[1+22] = i;}
      
    { double r,i;
      r = buf[210]; i = buf[1+210];
      buf[210] = buf[150]; buf[1+210] = buf[1+150];
      buf[150] = r; buf[1+150] = i;}
      
    { double r,i;
      r = buf[212]; i = buf[1+212];
      buf[212] = buf[86]; buf[1+212] = buf[1+86];
      buf[86] = r; buf[1+86] = i;}
      
    { double r,i;
      r = buf[216]; i = buf[1+216];
      buf[216] = buf[54]; buf[1+216] = buf[1+54];
      buf[54] = r; buf[1+54] = i;}
      
    { double r,i;
      r = buf[218]; i = buf[1+218];
      buf[218] = buf[182]; buf[1+218] = buf[1+182];
      buf[182] = r; buf[1+182] = i;}
      
    { double r,i;
      r = buf[220]; i = buf[1+220];
      buf[220] = buf[118]; buf[1+220] = buf[1+118];
      buf[118] = r; buf[1+118] = i;}
      
    { double r,i;
      r = buf[224]; i = buf[1+224];
      buf[224] = buf[14]; buf[1+224] = buf[1+14];
      buf[14] = r; buf[1+14] = i;}
      
    { double r,i;
      r = buf[226]; i = buf[1+226];
      buf[226] = buf[142]; buf[1+226] = buf[1+142];
      buf[142] = r; buf[1+142] = i;}
      
    { double r,i;
      r = buf[228]; i = buf[1+228];
      buf[228] = buf[78]; buf[1+228] = buf[1+78];
      buf[78] = r; buf[1+78] = i;}
      
    { double r,i;
      r = buf[230]; i = buf[1+230];
      buf[230] = buf[206]; buf[1+230] = buf[1+206];
      buf[206] = r; buf[1+206] = i;}
      
    { double r,i;
      r = buf[232]; i = buf[1+232];
      buf[232] = buf[46]; buf[1+232] = buf[1+46];
      buf[46] = r; buf[1+46] = i;}
      
    { double r,i;
      r = buf[234]; i = buf[1+234];
      buf[234] = buf[174]; buf[1+234] = buf[1+174];
      buf[174] = r; buf[1+174] = i;}
      
    { double r,i;
      r = buf[236]; i = buf[1+236];
      buf[236] = buf[110]; buf[1+236] = buf[1+110];
      buf[110] = r; buf[1+110] = i;}
      
    { double r,i;
      r = buf[240]; i = buf[1+240];
      buf[240] = buf[30]; buf[1+240] = buf[1+30];
      buf[30] = r; buf[1+30] = i;}
      
    { double r,i;
      r = buf[242]; i = buf[1+242];
      buf[242] = buf[158]; buf[1+242] = buf[1+158];
      buf[158] = r; buf[1+158] = i;}
      
    { double r,i;
      r = buf[244]; i = buf[1+244];
      buf[244] = buf[94]; buf[1+244] = buf[1+94];
      buf[94] = r; buf[1+94] = i;}
      
    { double r,i;
      r = buf[246]; i = buf[1+246];
      buf[246] = buf[222]; buf[1+246] = buf[1+222];
      buf[222] = r; buf[1+222] = i;}
      
    { double r,i;
      r = buf[248]; i = buf[1+248];
      buf[248] = buf[62]; buf[1+248] = buf[1+62];
      buf[62] = r; buf[1+62] = i;}
      
    { double r,i;
      r = buf[250]; i = buf[1+250];
      buf[250] = buf[190]; buf[1+250] = buf[1+190];
      buf[190] = r; buf[1+190] = i;}
      
    { double r,i;
      r = buf[252]; i = buf[1+252];
      buf[252] = buf[126]; buf[1+252] = buf[1+126];
      buf[126] = r; buf[1+126] = i;}
      
    return;
  case 256:
    { double r,i;
      r = buf[32]; i = buf[1+32];
      buf[32] = buf[16]; buf[1+32] = buf[1+16];
      buf[16] = r; buf[1+16] = i;}
      
    { double r,i;
      r = buf[64]; i = buf[1+64];
      buf[64] = buf[8]; buf[1+64] = buf[1+8];
      buf[8] = r; buf[1+8] = i;}
      
    { double r,i;
      r = buf[80]; i = buf[1+80];
      buf[80] = buf[40]; buf[1+80] = buf[1+40];
      buf[40] = r; buf[1+40] = i;}
      
    { double r,i;
      r = buf[96]; i = buf[1+96];
      buf[96] = buf[24]; buf[1+96] = buf[1+24];
      buf[24] = r; buf[1+24] = i;}
      
    { double r,i;
      r = buf[104]; i = buf[1+104];
      buf[104] = buf[88]; buf[1+104] = buf[1+88];
      buf[88] = r; buf[1+88] = i;}
      
    { double r,i;
      r = buf[112]; i = buf[1+112];
      buf[112] = buf[56]; buf[1+112] = buf[1+56];
      buf[56] = r; buf[1+56] = i;}
      
    { double r,i;
      r = buf[128]; i = buf[1+128];
      buf[128] = buf[4]; buf[1+128] = buf[1+4];
      buf[4] = r; buf[1+4] = i;}
      
    { double r,i;
      r = buf[136]; i = buf[1+136];
      buf[136] = buf[68]; buf[1+136] = buf[1+68];
      buf[68] = r; buf[1+68] = i;}
      
    { double r,i;
      r = buf[144]; i = buf[1+144];
      buf[144] = buf[36]; buf[1+144] = buf[1+36];
      buf[36] = r; buf[1+36] = i;}
      
    { double r,i;
      r = buf[152]; i = buf[1+152];
      buf[152] = buf[100]; buf[1+152] = buf[1+100];
      buf[100] = r; buf[1+100] = i;}
      
    { double r,i;
      r = buf[160]; i = buf[1+160];
      buf[160] = buf[20]; buf[1+160] = buf[1+20];
      buf[20] = r; buf[1+20] = i;}
      
    { double r,i;
      r = buf[164]; i = buf[1+164];
      buf[164] = buf[148]; buf[1+164] = buf[1+148];
      buf[148] = r; buf[1+148] = i;}
      
    { double r,i;
      r = buf[168]; i = buf[1+168];
      buf[168] = buf[84]; buf[1+168] = buf[1+84];
      buf[84] = r; buf[1+84] = i;}
      
    { double r,i;
      r = buf[176]; i = buf[1+176];
      buf[176] = buf[52]; buf[1+176] = buf[1+52];
      buf[52] = r; buf[1+52] = i;}
      
    { double r,i;
      r = buf[184]; i = buf[1+184];
      buf[184] = buf[116]; buf[1+184] = buf[1+116];
      buf[116] = r; buf[1+116] = i;}
      
    { double r,i;
      r = buf[192]; i = buf[1+192];
      buf[192] = buf[12]; buf[1+192] = buf[1+12];
      buf[12] = r; buf[1+12] = i;}
      
    { double r,i;
      r = buf[196]; i = buf[1+196];
      buf[196] = buf[140]; buf[1+196] = buf[1+140];
      buf[140] = r; buf[1+140] = i;}
      
    { double r,i;
      r = buf[200]; i = buf[1+200];
      buf[200] = buf[76]; buf[1+200] = buf[1+76];
      buf[76] = r; buf[1+76] = i;}
      
    { double r,i;
      r = buf[208]; i = buf[1+208];
      buf[208] = buf[44]; buf[1+208] = buf[1+44];
      buf[44] = r; buf[1+44] = i;}
      
    { double r,i;
      r = buf[212]; i = buf[1+212];
      buf[212] = buf[172]; buf[1+212] = buf[1+172];
      buf[172] = r; buf[1+172] = i;}
      
    { double r,i;
      r = buf[216]; i = buf[1+216];
      buf[216] = buf[108]; buf[1+216] = buf[1+108];
      buf[108] = r; buf[1+108] = i;}
      
    { double r,i;
      r = buf[224]; i = buf[1+224];
      buf[224] = buf[28]; buf[1+224] = buf[1+28];
      buf[28] = r; buf[1+28] = i;}
      
    { double r,i;
      r = buf[228]; i = buf[1+228];
      buf[228] = buf[156]; buf[1+228] = buf[1+156];
      buf[156] = r; buf[1+156] = i;}
      
    { double r,i;
      r = buf[232]; i = buf[1+232];
      buf[232] = buf[92]; buf[1+232] = buf[1+92];
      buf[92] = r; buf[1+92] = i;}
      
    { double r,i;
      r = buf[236]; i = buf[1+236];
      buf[236] = buf[220]; buf[1+236] = buf[1+220];
      buf[220] = r; buf[1+220] = i;}
      
    { double r,i;
      r = buf[240]; i = buf[1+240];
      buf[240] = buf[60]; buf[1+240] = buf[1+60];
      buf[60] = r; buf[1+60] = i;}
      
    { double r,i;
      r = buf[244]; i = buf[1+244];
      buf[244] = buf[188]; buf[1+244] = buf[1+188];
      buf[188] = r; buf[1+188] = i;}
      
    { double r,i;
      r = buf[248]; i = buf[1+248];
      buf[248] = buf[124]; buf[1+248] = buf[1+124];
      buf[124] = r; buf[1+124] = i;}
      
    { double r,i;
      r = buf[256]; i = buf[1+256];
      buf[256] = buf[2]; buf[1+256] = buf[1+2];
      buf[2] = r; buf[1+2] = i;}
      
    { double r,i;
      r = buf[260]; i = buf[1+260];
      buf[260] = buf[130]; buf[1+260] = buf[1+130];
      buf[130] = r; buf[1+130] = i;}
      
    { double r,i;
      r = buf[264]; i = buf[1+264];
      buf[264] = buf[66]; buf[1+264] = buf[1+66];
      buf[66] = r; buf[1+66] = i;}
      
    { double r,i;
      r = buf[268]; i = buf[1+268];
      buf[268] = buf[194]; buf[1+268] = buf[1+194];
      buf[194] = r; buf[1+194] = i;}
      
    { double r,i;
      r = buf[272]; i = buf[1+272];
      buf[272] = buf[34]; buf[1+272] = buf[1+34];
      buf[34] = r; buf[1+34] = i;}
      
    { double r,i;
      r = buf[276]; i = buf[1+276];
      buf[276] = buf[162]; buf[1+276] = buf[1+162];
      buf[162] = r; buf[1+162] = i;}
      
    { double r,i;
      r = buf[280]; i = buf[1+280];
      buf[280] = buf[98]; buf[1+280] = buf[1+98];
      buf[98] = r; buf[1+98] = i;}
      
    { double r,i;
      r = buf[284]; i = buf[1+284];
      buf[284] = buf[226]; buf[1+284] = buf[1+226];
      buf[226] = r; buf[1+226] = i;}
      
    { double r,i;
      r = buf[288]; i = buf[1+288];
      buf[288] = buf[18]; buf[1+288] = buf[1+18];
      buf[18] = r; buf[1+18] = i;}
      
    { double r,i;
      r = buf[290]; i = buf[1+290];
      buf[290] = buf[274]; buf[1+290] = buf[1+274];
      buf[274] = r; buf[1+274] = i;}
      
    { double r,i;
      r = buf[292]; i = buf[1+292];
      buf[292] = buf[146]; buf[1+292] = buf[1+146];
      buf[146] = r; buf[1+146] = i;}
      
    { double r,i;
      r = buf[296]; i = buf[1+296];
      buf[296] = buf[82]; buf[1+296] = buf[1+82];
      buf[82] = r; buf[1+82] = i;}
      
    { double r,i;
      r = buf[300]; i = buf[1+300];
      buf[300] = buf[210]; buf[1+300] = buf[1+210];
      buf[210] = r; buf[1+210] = i;}
      
    { double r,i;
      r = buf[304]; i = buf[1+304];
      buf[304] = buf[50]; buf[1+304] = buf[1+50];
      buf[50] = r; buf[1+50] = i;}
      
    { double r,i;
      r = buf[308]; i = buf[1+308];
      buf[308] = buf[178]; buf[1+308] = buf[1+178];
      buf[178] = r; buf[1+178] = i;}
      
    { double r,i;
      r = buf[312]; i = buf[1+312];
      buf[312] = buf[114]; buf[1+312] = buf[1+114];
      buf[114] = r; buf[1+114] = i;}
      
    { double r,i;
      r = buf[316]; i = buf[1+316];
      buf[316] = buf[242]; buf[1+316] = buf[1+242];
      buf[242] = r; buf[1+242] = i;}
      
    { double r,i;
      r = buf[320]; i = buf[1+320];
      buf[320] = buf[10]; buf[1+320] = buf[1+10];
      buf[10] = r; buf[1+10] = i;}
      
    { double r,i;
      r = buf[322]; i = buf[1+322];
      buf[322] = buf[266]; buf[1+322] = buf[1+266];
      buf[266] = r; buf[1+266] = i;}
      
    { double r,i;
      r = buf[324]; i = buf[1+324];
      buf[324] = buf[138]; buf[1+324] = buf[1+138];
      buf[138] = r; buf[1+138] = i;}
      
    { double r,i;
      r = buf[328]; i = buf[1+328];
      buf[328] = buf[74]; buf[1+328] = buf[1+74];
      buf[74] = r; buf[1+74] = i;}
      
    { double r,i;
      r = buf[332]; i = buf[1+332];
      buf[332] = buf[202]; buf[1+332] = buf[1+202];
      buf[202] = r; buf[1+202] = i;}
      
    { double r,i;
      r = buf[336]; i = buf[1+336];
      buf[336] = buf[42]; buf[1+336] = buf[1+42];
      buf[42] = r; buf[1+42] = i;}
      
    { double r,i;
      r = buf[338]; i = buf[1+338];
      buf[338] = buf[298]; buf[1+338] = buf[1+298];
      buf[298] = r; buf[1+298] = i;}
      
    { double r,i;
      r = buf[340]; i = buf[1+340];
      buf[340] = buf[170]; buf[1+340] = buf[1+170];
      buf[170] = r; buf[1+170] = i;}
      
    { double r,i;
      r = buf[344]; i = buf[1+344];
      buf[344] = buf[106]; buf[1+344] = buf[1+106];
      buf[106] = r; buf[1+106] = i;}
      
    { double r,i;
      r = buf[348]; i = buf[1+348];
      buf[348] = buf[234]; buf[1+348] = buf[1+234];
      buf[234] = r; buf[1+234] = i;}
      
    { double r,i;
      r = buf[352]; i = buf[1+352];
      buf[352] = buf[26]; buf[1+352] = buf[1+26];
      buf[26] = r; buf[1+26] = i;}
      
    { double r,i;
      r = buf[354]; i = buf[1+354];
      buf[354] = buf[282]; buf[1+354] = buf[1+282];
      buf[282] = r; buf[1+282] = i;}
      
    { double r,i;
      r = buf[356]; i = buf[1+356];
      buf[356] = buf[154]; buf[1+356] = buf[1+154];
      buf[154] = r; buf[1+154] = i;}
      
    { double r,i;
      r = buf[360]; i = buf[1+360];
      buf[360] = buf[90]; buf[1+360] = buf[1+90];
      buf[90] = r; buf[1+90] = i;}
      
    { double r,i;
      r = buf[362]; i = buf[1+362];
      buf[362] = buf[346]; buf[1+362] = buf[1+346];
      buf[346] = r; buf[1+346] = i;}
      
    { double r,i;
      r = buf[364]; i = buf[1+364];
      buf[364] = buf[218]; buf[1+364] = buf[1+218];
      buf[218] = r; buf[1+218] = i;}
      
    { double r,i;
      r = buf[368]; i = buf[1+368];
      buf[368] = buf[58]; buf[1+368] = buf[1+58];
      buf[58] = r; buf[1+58] = i;}
      
    { double r,i;
      r = buf[370]; i = buf[1+370];
      buf[370] = buf[314]; buf[1+370] = buf[1+314];
      buf[314] = r; buf[1+314] = i;}
      
    { double r,i;
      r = buf[372]; i = buf[1+372];
      buf[372] = buf[186]; buf[1+372] = buf[1+186];
      buf[186] = r; buf[1+186] = i;}
      
    { double r,i;
      r = buf[376]; i = buf[1+376];
      buf[376] = buf[122]; buf[1+376] = buf[1+122];
      buf[122] = r; buf[1+122] = i;}
      
    { double r,i;
      r = buf[380]; i = buf[1+380];
      buf[380] = buf[250]; buf[1+380] = buf[1+250];
      buf[250] = r; buf[1+250] = i;}
      
    { double r,i;
      r = buf[384]; i = buf[1+384];
      buf[384] = buf[6]; buf[1+384] = buf[1+6];
      buf[6] = r; buf[1+6] = i;}
      
    { double r,i;
      r = buf[386]; i = buf[1+386];
      buf[386] = buf[262]; buf[1+386] = buf[1+262];
      buf[262] = r; buf[1+262] = i;}
      
    { double r,i;
      r = buf[388]; i = buf[1+388];
      buf[388] = buf[134]; buf[1+388] = buf[1+134];
      buf[134] = r; buf[1+134] = i;}
      
    { double r,i;
      r = buf[392]; i = buf[1+392];
      buf[392] = buf[70]; buf[1+392] = buf[1+70];
      buf[70] = r; buf[1+70] = i;}
      
    { double r,i;
      r = buf[394]; i = buf[1+394];
      buf[394] = buf[326]; buf[1+394] = buf[1+326];
      buf[326] = r; buf[1+326] = i;}
      
    { double r,i;
      r = buf[396]; i = buf[1+396];
      buf[396] = buf[198]; buf[1+396] = buf[1+198];
      buf[198] = r; buf[1+198] = i;}
      
    { double r,i;
      r = buf[400]; i = buf[1+400];
      buf[400] = buf[38]; buf[1+400] = buf[1+38];
      buf[38] = r; buf[1+38] = i;}
      
    { double r,i;
      r = buf[402]; i = buf[1+402];
      buf[402] = buf[294]; buf[1+402] = buf[1+294];
      buf[294] = r; buf[1+294] = i;}
      
    { double r,i;
      r = buf[404]; i = buf[1+404];
      buf[404] = buf[166]; buf[1+404] = buf[1+166];
      buf[166] = r; buf[1+166] = i;}
      
    { double r,i;
      r = buf[408]; i = buf[1+408];
      buf[408] = buf[102]; buf[1+408] = buf[1+102];
      buf[102] = r; buf[1+102] = i;}
      
    { double r,i;
      r = buf[410]; i = buf[1+410];
      buf[410] = buf[358]; buf[1+410] = buf[1+358];
      buf[358] = r; buf[1+358] = i;}
      
    { double r,i;
      r = buf[412]; i = buf[1+412];
      buf[412] = buf[230]; buf[1+412] = buf[1+230];
      buf[230] = r; buf[1+230] = i;}
      
    { double r,i;
      r = buf[416]; i = buf[1+416];
      buf[416] = buf[22]; buf[1+416] = buf[1+22];
      buf[22] = r; buf[1+22] = i;}
      
    { double r,i;
      r = buf[418]; i = buf[1+418];
      buf[418] = buf[278]; buf[1+418] = buf[1+278];
      buf[278] = r; buf[1+278] = i;}
      
    { double r,i;
      r = buf[420]; i = buf[1+420];
      buf[420] = buf[150]; buf[1+420] = buf[1+150];
      buf[150] = r; buf[1+150] = i;}
      
    { double r,i;
      r = buf[422]; i = buf[1+422];
      buf[422] = buf[406]; buf[1+422] = buf[1+406];
      buf[406] = r; buf[1+406] = i;}
      
    { double r,i;
      r = buf[424]; i = buf[1+424];
      buf[424] = buf[86]; buf[1+424] = buf[1+86];
      buf[86] = r; buf[1+86] = i;}
      
    { double r,i;
      r = buf[426]; i = buf[1+426];
      buf[426] = buf[342]; buf[1+426] = buf[1+342];
      buf[342] = r; buf[1+342] = i;}
      
    { double r,i;
      r = buf[428]; i = buf[1+428];
      buf[428] = buf[214]; buf[1+428] = buf[1+214];
      buf[214] = r; buf[1+214] = i;}
      
    { double r,i;
      r = buf[432]; i = buf[1+432];
      buf[432] = buf[54]; buf[1+432] = buf[1+54];
      buf[54] = r; buf[1+54] = i;}
      
    { double r,i;
      r = buf[434]; i = buf[1+434];
      buf[434] = buf[310]; buf[1+434] = buf[1+310];
      buf[310] = r; buf[1+310] = i;}
      
    { double r,i;
      r = buf[436]; i = buf[1+436];
      buf[436] = buf[182]; buf[1+436] = buf[1+182];
      buf[182] = r; buf[1+182] = i;}
      
    { double r,i;
      r = buf[440]; i = buf[1+440];
      buf[440] = buf[118]; buf[1+440] = buf[1+118];
      buf[118] = r; buf[1+118] = i;}
      
    { double r,i;
      r = buf[442]; i = buf[1+442];
      buf[442] = buf[374]; buf[1+442] = buf[1+374];
      buf[374] = r; buf[1+374] = i;}
      
    { double r,i;
      r = buf[444]; i = buf[1+444];
      buf[444] = buf[246]; buf[1+444] = buf[1+246];
      buf[246] = r; buf[1+246] = i;}
      
    { double r,i;
      r = buf[448]; i = buf[1+448];
      buf[448] = buf[14]; buf[1+448] = buf[1+14];
      buf[14] = r; buf[1+14] = i;}
      
    { double r,i;
      r = buf[450]; i = buf[1+450];
      buf[450] = buf[270]; buf[1+450] = buf[1+270];
      buf[270] = r; buf[1+270] = i;}
      
    { double r,i;
      r = buf[452]; i = buf[1+452];
      buf[452] = buf[142]; buf[1+452] = buf[1+142];
      buf[142] = r; buf[1+142] = i;}
      
    { double r,i;
      r = buf[454]; i = buf[1+454];
      buf[454] = buf[398]; buf[1+454] = buf[1+398];
      buf[398] = r; buf[1+398] = i;}
      
    { double r,i;
      r = buf[456]; i = buf[1+456];
      buf[456] = buf[78]; buf[1+456] = buf[1+78];
      buf[78] = r; buf[1+78] = i;}
      
    { double r,i;
      r = buf[458]; i = buf[1+458];
      buf[458] = buf[334]; buf[1+458] = buf[1+334];
      buf[334] = r; buf[1+334] = i;}
      
    { double r,i;
      r = buf[460]; i = buf[1+460];
      buf[460] = buf[206]; buf[1+460] = buf[1+206];
      buf[206] = r; buf[1+206] = i;}
      
    { double r,i;
      r = buf[464]; i = buf[1+464];
      buf[464] = buf[46]; buf[1+464] = buf[1+46];
      buf[46] = r; buf[1+46] = i;}
      
    { double r,i;
      r = buf[466]; i = buf[1+466];
      buf[466] = buf[302]; buf[1+466] = buf[1+302];
      buf[302] = r; buf[1+302] = i;}
      
    { double r,i;
      r = buf[468]; i = buf[1+468];
      buf[468] = buf[174]; buf[1+468] = buf[1+174];
      buf[174] = r; buf[1+174] = i;}
      
    { double r,i;
      r = buf[470]; i = buf[1+470];
      buf[470] = buf[430]; buf[1+470] = buf[1+430];
      buf[430] = r; buf[1+430] = i;}
      
    { double r,i;
      r = buf[472]; i = buf[1+472];
      buf[472] = buf[110]; buf[1+472] = buf[1+110];
      buf[110] = r; buf[1+110] = i;}
      
    { double r,i;
      r = buf[474]; i = buf[1+474];
      buf[474] = buf[366]; buf[1+474] = buf[1+366];
      buf[366] = r; buf[1+366] = i;}
      
    { double r,i;
      r = buf[476]; i = buf[1+476];
      buf[476] = buf[238]; buf[1+476] = buf[1+238];
      buf[238] = r; buf[1+238] = i;}
      
    { double r,i;
      r = buf[480]; i = buf[1+480];
      buf[480] = buf[30]; buf[1+480] = buf[1+30];
      buf[30] = r; buf[1+30] = i;}
      
    { double r,i;
      r = buf[482]; i = buf[1+482];
      buf[482] = buf[286]; buf[1+482] = buf[1+286];
      buf[286] = r; buf[1+286] = i;}
      
    { double r,i;
      r = buf[484]; i = buf[1+484];
      buf[484] = buf[158]; buf[1+484] = buf[1+158];
      buf[158] = r; buf[1+158] = i;}
      
    { double r,i;
      r = buf[486]; i = buf[1+486];
      buf[486] = buf[414]; buf[1+486] = buf[1+414];
      buf[414] = r; buf[1+414] = i;}
      
    { double r,i;
      r = buf[488]; i = buf[1+488];
      buf[488] = buf[94]; buf[1+488] = buf[1+94];
      buf[94] = r; buf[1+94] = i;}
      
    { double r,i;
      r = buf[490]; i = buf[1+490];
      buf[490] = buf[350]; buf[1+490] = buf[1+350];
      buf[350] = r; buf[1+350] = i;}
      
    { double r,i;
      r = buf[492]; i = buf[1+492];
      buf[492] = buf[222]; buf[1+492] = buf[1+222];
      buf[222] = r; buf[1+222] = i;}
      
    { double r,i;
      r = buf[494]; i = buf[1+494];
      buf[494] = buf[478]; buf[1+494] = buf[1+478];
      buf[478] = r; buf[1+478] = i;}
      
    { double r,i;
      r = buf[496]; i = buf[1+496];
      buf[496] = buf[62]; buf[1+496] = buf[1+62];
      buf[62] = r; buf[1+62] = i;}
      
    { double r,i;
      r = buf[498]; i = buf[1+498];
      buf[498] = buf[318]; buf[1+498] = buf[1+318];
      buf[318] = r; buf[1+318] = i;}
      
    { double r,i;
      r = buf[500]; i = buf[1+500];
      buf[500] = buf[190]; buf[1+500] = buf[1+190];
      buf[190] = r; buf[1+190] = i;}
      
    { double r,i;
      r = buf[502]; i = buf[1+502];
      buf[502] = buf[446]; buf[1+502] = buf[1+446];
      buf[446] = r; buf[1+446] = i;}
      
    { double r,i;
      r = buf[504]; i = buf[1+504];
      buf[504] = buf[126]; buf[1+504] = buf[1+126];
      buf[126] = r; buf[1+126] = i;}
      
    { double r,i;
      r = buf[506]; i = buf[1+506];
      buf[506] = buf[382]; buf[1+506] = buf[1+382];
      buf[382] = r; buf[1+382] = i;}
      
    { double r,i;
      r = buf[508]; i = buf[1+508];
      buf[508] = buf[254]; buf[1+508] = buf[1+254];
      buf[254] = r; buf[1+254] = i;}
      
    return;
  default:
    {
      unsigned bits = 0;
      { unsigned x = n; /* count bits, log(N) algo */
        { unsigned y = (x >> 16); if(y) { bits += 16; x = y; } }
        { unsigned y = (x >> 8); if(y) { bits += 8; x = y; } }
        { unsigned y = (x >> 4); if(y) { bits += 4; x = y; } }
        { unsigned y = (x >> 2); if(y) { bits += 2; x = y; } }
        { unsigned y = (x >> 1); if(y) { bits += 1; x = y; } }
      }
      {
        unsigned a, b, c, ii = (1 << bits);
        for(a = 1, b = 0; a < ii; ++a)
        {
          /* this increments b in bitreversed order */
          /* add by xor, probagate carries manually */
          /* worst case O(NlogN) but amortized O(N) */
          /* in fact it runs only 2*n-1 times total */
          /* could improve this by unroll & predict */
          for(c = ii >> 1; c; c >>= 1) { if((b^=c) & c) break; }
          if(a > b)
          {
            { double r,i;
      r = buf[(a<<1)]; i = buf[1+(a<<1)];
      buf[(a<<1)] = buf[(b<<1)]; buf[1+(a<<1)] = buf[1+(b<<1)];
      buf[(b<<1)] = r; buf[1+(b<<1)] = i;}
    
          }
        }
      }
    }
    return;
  }
}


static void _br_pd(double * buf, unsigned n)
{
  switch(n)
  {
  case 1:
  case 2: return;
  case 4: 
    { __m128d tmp;
      tmp = ((__m128d*)buf)[2];
      ((__m128d*)buf)[2] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    return;
  case 8:
    { __m128d tmp;
      tmp = ((__m128d*)buf)[4];
      ((__m128d*)buf)[4] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[6];
      ((__m128d*)buf)[6] = ((__m128d*)buf)[3];
      ((__m128d*)buf)[3] = tmp;}
      
    return;
  case 16:
    { __m128d tmp;
      tmp = ((__m128d*)buf)[4];
      ((__m128d*)buf)[4] = ((__m128d*)buf)[2];
      ((__m128d*)buf)[2] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[8];
      ((__m128d*)buf)[8] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[10];
      ((__m128d*)buf)[10] = ((__m128d*)buf)[5];
      ((__m128d*)buf)[5] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[12];
      ((__m128d*)buf)[12] = ((__m128d*)buf)[3];
      ((__m128d*)buf)[3] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[13];
      ((__m128d*)buf)[13] = ((__m128d*)buf)[11];
      ((__m128d*)buf)[11] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[14];
      ((__m128d*)buf)[14] = ((__m128d*)buf)[7];
      ((__m128d*)buf)[7] = tmp;}
      
    return;
    
  case 32:
    { __m128d tmp;
      tmp = ((__m128d*)buf)[8];
      ((__m128d*)buf)[8] = ((__m128d*)buf)[2];
      ((__m128d*)buf)[2] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[12];
      ((__m128d*)buf)[12] = ((__m128d*)buf)[6];
      ((__m128d*)buf)[6] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[16];
      ((__m128d*)buf)[16] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[18];
      ((__m128d*)buf)[18] = ((__m128d*)buf)[9];
      ((__m128d*)buf)[9] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[20];
      ((__m128d*)buf)[20] = ((__m128d*)buf)[5];
      ((__m128d*)buf)[5] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[22];
      ((__m128d*)buf)[22] = ((__m128d*)buf)[13];
      ((__m128d*)buf)[13] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[24];
      ((__m128d*)buf)[24] = ((__m128d*)buf)[3];
      ((__m128d*)buf)[3] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[25];
      ((__m128d*)buf)[25] = ((__m128d*)buf)[19];
      ((__m128d*)buf)[19] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[26];
      ((__m128d*)buf)[26] = ((__m128d*)buf)[11];
      ((__m128d*)buf)[11] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[28];
      ((__m128d*)buf)[28] = ((__m128d*)buf)[7];
      ((__m128d*)buf)[7] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[29];
      ((__m128d*)buf)[29] = ((__m128d*)buf)[23];
      ((__m128d*)buf)[23] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[30];
      ((__m128d*)buf)[30] = ((__m128d*)buf)[15];
      ((__m128d*)buf)[15] = tmp;}
      
    return;
  case 64:
    { __m128d tmp;
      tmp = ((__m128d*)buf)[8];
      ((__m128d*)buf)[8] = ((__m128d*)buf)[4];
      ((__m128d*)buf)[4] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[16];
      ((__m128d*)buf)[16] = ((__m128d*)buf)[2];
      ((__m128d*)buf)[2] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[20];
      ((__m128d*)buf)[20] = ((__m128d*)buf)[10];
      ((__m128d*)buf)[10] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[24];
      ((__m128d*)buf)[24] = ((__m128d*)buf)[6];
      ((__m128d*)buf)[6] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[26];
      ((__m128d*)buf)[26] = ((__m128d*)buf)[22];
      ((__m128d*)buf)[22] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[28];
      ((__m128d*)buf)[28] = ((__m128d*)buf)[14];
      ((__m128d*)buf)[14] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[32];
      ((__m128d*)buf)[32] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[34];
      ((__m128d*)buf)[34] = ((__m128d*)buf)[17];
      ((__m128d*)buf)[17] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[36];
      ((__m128d*)buf)[36] = ((__m128d*)buf)[9];
      ((__m128d*)buf)[9] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[38];
      ((__m128d*)buf)[38] = ((__m128d*)buf)[25];
      ((__m128d*)buf)[25] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[40];
      ((__m128d*)buf)[40] = ((__m128d*)buf)[5];
      ((__m128d*)buf)[5] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[41];
      ((__m128d*)buf)[41] = ((__m128d*)buf)[37];
      ((__m128d*)buf)[37] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[42];
      ((__m128d*)buf)[42] = ((__m128d*)buf)[21];
      ((__m128d*)buf)[21] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[44];
      ((__m128d*)buf)[44] = ((__m128d*)buf)[13];
      ((__m128d*)buf)[13] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[46];
      ((__m128d*)buf)[46] = ((__m128d*)buf)[29];
      ((__m128d*)buf)[29] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[48];
      ((__m128d*)buf)[48] = ((__m128d*)buf)[3];
      ((__m128d*)buf)[3] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[49];
      ((__m128d*)buf)[49] = ((__m128d*)buf)[35];
      ((__m128d*)buf)[35] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[50];
      ((__m128d*)buf)[50] = ((__m128d*)buf)[19];
      ((__m128d*)buf)[19] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[52];
      ((__m128d*)buf)[52] = ((__m128d*)buf)[11];
      ((__m128d*)buf)[11] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[53];
      ((__m128d*)buf)[53] = ((__m128d*)buf)[43];
      ((__m128d*)buf)[43] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[54];
      ((__m128d*)buf)[54] = ((__m128d*)buf)[27];
      ((__m128d*)buf)[27] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[56];
      ((__m128d*)buf)[56] = ((__m128d*)buf)[7];
      ((__m128d*)buf)[7] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[57];
      ((__m128d*)buf)[57] = ((__m128d*)buf)[39];
      ((__m128d*)buf)[39] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[58];
      ((__m128d*)buf)[58] = ((__m128d*)buf)[23];
      ((__m128d*)buf)[23] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[59];
      ((__m128d*)buf)[59] = ((__m128d*)buf)[55];
      ((__m128d*)buf)[55] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[60];
      ((__m128d*)buf)[60] = ((__m128d*)buf)[15];
      ((__m128d*)buf)[15] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[61];
      ((__m128d*)buf)[61] = ((__m128d*)buf)[47];
      ((__m128d*)buf)[47] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[62];
      ((__m128d*)buf)[62] = ((__m128d*)buf)[31];
      ((__m128d*)buf)[31] = tmp;}
      
    return;
  case 128:
    { __m128d tmp;
      tmp = ((__m128d*)buf)[16];
      ((__m128d*)buf)[16] = ((__m128d*)buf)[4];
      ((__m128d*)buf)[4] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[24];
      ((__m128d*)buf)[24] = ((__m128d*)buf)[12];
      ((__m128d*)buf)[12] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[32];
      ((__m128d*)buf)[32] = ((__m128d*)buf)[2];
      ((__m128d*)buf)[2] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[36];
      ((__m128d*)buf)[36] = ((__m128d*)buf)[18];
      ((__m128d*)buf)[18] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[40];
      ((__m128d*)buf)[40] = ((__m128d*)buf)[10];
      ((__m128d*)buf)[10] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[44];
      ((__m128d*)buf)[44] = ((__m128d*)buf)[26];
      ((__m128d*)buf)[26] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[48];
      ((__m128d*)buf)[48] = ((__m128d*)buf)[6];
      ((__m128d*)buf)[6] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[50];
      ((__m128d*)buf)[50] = ((__m128d*)buf)[38];
      ((__m128d*)buf)[38] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[52];
      ((__m128d*)buf)[52] = ((__m128d*)buf)[22];
      ((__m128d*)buf)[22] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[56];
      ((__m128d*)buf)[56] = ((__m128d*)buf)[14];
      ((__m128d*)buf)[14] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[58];
      ((__m128d*)buf)[58] = ((__m128d*)buf)[46];
      ((__m128d*)buf)[46] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[60];
      ((__m128d*)buf)[60] = ((__m128d*)buf)[30];
      ((__m128d*)buf)[30] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[64];
      ((__m128d*)buf)[64] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[66];
      ((__m128d*)buf)[66] = ((__m128d*)buf)[33];
      ((__m128d*)buf)[33] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[68];
      ((__m128d*)buf)[68] = ((__m128d*)buf)[17];
      ((__m128d*)buf)[17] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[70];
      ((__m128d*)buf)[70] = ((__m128d*)buf)[49];
      ((__m128d*)buf)[49] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[72];
      ((__m128d*)buf)[72] = ((__m128d*)buf)[9];
      ((__m128d*)buf)[9] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[74];
      ((__m128d*)buf)[74] = ((__m128d*)buf)[41];
      ((__m128d*)buf)[41] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[76];
      ((__m128d*)buf)[76] = ((__m128d*)buf)[25];
      ((__m128d*)buf)[25] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[78];
      ((__m128d*)buf)[78] = ((__m128d*)buf)[57];
      ((__m128d*)buf)[57] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[80];
      ((__m128d*)buf)[80] = ((__m128d*)buf)[5];
      ((__m128d*)buf)[5] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[81];
      ((__m128d*)buf)[81] = ((__m128d*)buf)[69];
      ((__m128d*)buf)[69] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[82];
      ((__m128d*)buf)[82] = ((__m128d*)buf)[37];
      ((__m128d*)buf)[37] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[84];
      ((__m128d*)buf)[84] = ((__m128d*)buf)[21];
      ((__m128d*)buf)[21] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[86];
      ((__m128d*)buf)[86] = ((__m128d*)buf)[53];
      ((__m128d*)buf)[53] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[88];
      ((__m128d*)buf)[88] = ((__m128d*)buf)[13];
      ((__m128d*)buf)[13] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[89];
      ((__m128d*)buf)[89] = ((__m128d*)buf)[77];
      ((__m128d*)buf)[77] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[90];
      ((__m128d*)buf)[90] = ((__m128d*)buf)[45];
      ((__m128d*)buf)[45] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[92];
      ((__m128d*)buf)[92] = ((__m128d*)buf)[29];
      ((__m128d*)buf)[29] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[94];
      ((__m128d*)buf)[94] = ((__m128d*)buf)[61];
      ((__m128d*)buf)[61] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[96];
      ((__m128d*)buf)[96] = ((__m128d*)buf)[3];
      ((__m128d*)buf)[3] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[97];
      ((__m128d*)buf)[97] = ((__m128d*)buf)[67];
      ((__m128d*)buf)[67] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[98];
      ((__m128d*)buf)[98] = ((__m128d*)buf)[35];
      ((__m128d*)buf)[35] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[100];
      ((__m128d*)buf)[100] = ((__m128d*)buf)[19];
      ((__m128d*)buf)[19] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[101];
      ((__m128d*)buf)[101] = ((__m128d*)buf)[83];
      ((__m128d*)buf)[83] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[102];
      ((__m128d*)buf)[102] = ((__m128d*)buf)[51];
      ((__m128d*)buf)[51] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[104];
      ((__m128d*)buf)[104] = ((__m128d*)buf)[11];
      ((__m128d*)buf)[11] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[105];
      ((__m128d*)buf)[105] = ((__m128d*)buf)[75];
      ((__m128d*)buf)[75] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[106];
      ((__m128d*)buf)[106] = ((__m128d*)buf)[43];
      ((__m128d*)buf)[43] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[108];
      ((__m128d*)buf)[108] = ((__m128d*)buf)[27];
      ((__m128d*)buf)[27] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[109];
      ((__m128d*)buf)[109] = ((__m128d*)buf)[91];
      ((__m128d*)buf)[91] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[110];
      ((__m128d*)buf)[110] = ((__m128d*)buf)[59];
      ((__m128d*)buf)[59] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[112];
      ((__m128d*)buf)[112] = ((__m128d*)buf)[7];
      ((__m128d*)buf)[7] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[113];
      ((__m128d*)buf)[113] = ((__m128d*)buf)[71];
      ((__m128d*)buf)[71] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[114];
      ((__m128d*)buf)[114] = ((__m128d*)buf)[39];
      ((__m128d*)buf)[39] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[115];
      ((__m128d*)buf)[115] = ((__m128d*)buf)[103];
      ((__m128d*)buf)[103] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[116];
      ((__m128d*)buf)[116] = ((__m128d*)buf)[23];
      ((__m128d*)buf)[23] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[117];
      ((__m128d*)buf)[117] = ((__m128d*)buf)[87];
      ((__m128d*)buf)[87] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[118];
      ((__m128d*)buf)[118] = ((__m128d*)buf)[55];
      ((__m128d*)buf)[55] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[120];
      ((__m128d*)buf)[120] = ((__m128d*)buf)[15];
      ((__m128d*)buf)[15] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[121];
      ((__m128d*)buf)[121] = ((__m128d*)buf)[79];
      ((__m128d*)buf)[79] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[122];
      ((__m128d*)buf)[122] = ((__m128d*)buf)[47];
      ((__m128d*)buf)[47] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[123];
      ((__m128d*)buf)[123] = ((__m128d*)buf)[111];
      ((__m128d*)buf)[111] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[124];
      ((__m128d*)buf)[124] = ((__m128d*)buf)[31];
      ((__m128d*)buf)[31] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[125];
      ((__m128d*)buf)[125] = ((__m128d*)buf)[95];
      ((__m128d*)buf)[95] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[126];
      ((__m128d*)buf)[126] = ((__m128d*)buf)[63];
      ((__m128d*)buf)[63] = tmp;}
      
    return;
  case 256:
    { __m128d tmp;
      tmp = ((__m128d*)buf)[16];
      ((__m128d*)buf)[16] = ((__m128d*)buf)[8];
      ((__m128d*)buf)[8] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[32];
      ((__m128d*)buf)[32] = ((__m128d*)buf)[4];
      ((__m128d*)buf)[4] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[40];
      ((__m128d*)buf)[40] = ((__m128d*)buf)[20];
      ((__m128d*)buf)[20] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[48];
      ((__m128d*)buf)[48] = ((__m128d*)buf)[12];
      ((__m128d*)buf)[12] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[52];
      ((__m128d*)buf)[52] = ((__m128d*)buf)[44];
      ((__m128d*)buf)[44] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[56];
      ((__m128d*)buf)[56] = ((__m128d*)buf)[28];
      ((__m128d*)buf)[28] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[64];
      ((__m128d*)buf)[64] = ((__m128d*)buf)[2];
      ((__m128d*)buf)[2] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[68];
      ((__m128d*)buf)[68] = ((__m128d*)buf)[34];
      ((__m128d*)buf)[34] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[72];
      ((__m128d*)buf)[72] = ((__m128d*)buf)[18];
      ((__m128d*)buf)[18] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[76];
      ((__m128d*)buf)[76] = ((__m128d*)buf)[50];
      ((__m128d*)buf)[50] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[80];
      ((__m128d*)buf)[80] = ((__m128d*)buf)[10];
      ((__m128d*)buf)[10] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[82];
      ((__m128d*)buf)[82] = ((__m128d*)buf)[74];
      ((__m128d*)buf)[74] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[84];
      ((__m128d*)buf)[84] = ((__m128d*)buf)[42];
      ((__m128d*)buf)[42] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[88];
      ((__m128d*)buf)[88] = ((__m128d*)buf)[26];
      ((__m128d*)buf)[26] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[92];
      ((__m128d*)buf)[92] = ((__m128d*)buf)[58];
      ((__m128d*)buf)[58] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[96];
      ((__m128d*)buf)[96] = ((__m128d*)buf)[6];
      ((__m128d*)buf)[6] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[98];
      ((__m128d*)buf)[98] = ((__m128d*)buf)[70];
      ((__m128d*)buf)[70] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[100];
      ((__m128d*)buf)[100] = ((__m128d*)buf)[38];
      ((__m128d*)buf)[38] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[104];
      ((__m128d*)buf)[104] = ((__m128d*)buf)[22];
      ((__m128d*)buf)[22] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[106];
      ((__m128d*)buf)[106] = ((__m128d*)buf)[86];
      ((__m128d*)buf)[86] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[108];
      ((__m128d*)buf)[108] = ((__m128d*)buf)[54];
      ((__m128d*)buf)[54] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[112];
      ((__m128d*)buf)[112] = ((__m128d*)buf)[14];
      ((__m128d*)buf)[14] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[114];
      ((__m128d*)buf)[114] = ((__m128d*)buf)[78];
      ((__m128d*)buf)[78] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[116];
      ((__m128d*)buf)[116] = ((__m128d*)buf)[46];
      ((__m128d*)buf)[46] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[118];
      ((__m128d*)buf)[118] = ((__m128d*)buf)[110];
      ((__m128d*)buf)[110] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[120];
      ((__m128d*)buf)[120] = ((__m128d*)buf)[30];
      ((__m128d*)buf)[30] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[122];
      ((__m128d*)buf)[122] = ((__m128d*)buf)[94];
      ((__m128d*)buf)[94] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[124];
      ((__m128d*)buf)[124] = ((__m128d*)buf)[62];
      ((__m128d*)buf)[62] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[128];
      ((__m128d*)buf)[128] = ((__m128d*)buf)[1];
      ((__m128d*)buf)[1] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[130];
      ((__m128d*)buf)[130] = ((__m128d*)buf)[65];
      ((__m128d*)buf)[65] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[132];
      ((__m128d*)buf)[132] = ((__m128d*)buf)[33];
      ((__m128d*)buf)[33] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[134];
      ((__m128d*)buf)[134] = ((__m128d*)buf)[97];
      ((__m128d*)buf)[97] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[136];
      ((__m128d*)buf)[136] = ((__m128d*)buf)[17];
      ((__m128d*)buf)[17] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[138];
      ((__m128d*)buf)[138] = ((__m128d*)buf)[81];
      ((__m128d*)buf)[81] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[140];
      ((__m128d*)buf)[140] = ((__m128d*)buf)[49];
      ((__m128d*)buf)[49] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[142];
      ((__m128d*)buf)[142] = ((__m128d*)buf)[113];
      ((__m128d*)buf)[113] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[144];
      ((__m128d*)buf)[144] = ((__m128d*)buf)[9];
      ((__m128d*)buf)[9] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[145];
      ((__m128d*)buf)[145] = ((__m128d*)buf)[137];
      ((__m128d*)buf)[137] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[146];
      ((__m128d*)buf)[146] = ((__m128d*)buf)[73];
      ((__m128d*)buf)[73] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[148];
      ((__m128d*)buf)[148] = ((__m128d*)buf)[41];
      ((__m128d*)buf)[41] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[150];
      ((__m128d*)buf)[150] = ((__m128d*)buf)[105];
      ((__m128d*)buf)[105] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[152];
      ((__m128d*)buf)[152] = ((__m128d*)buf)[25];
      ((__m128d*)buf)[25] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[154];
      ((__m128d*)buf)[154] = ((__m128d*)buf)[89];
      ((__m128d*)buf)[89] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[156];
      ((__m128d*)buf)[156] = ((__m128d*)buf)[57];
      ((__m128d*)buf)[57] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[158];
      ((__m128d*)buf)[158] = ((__m128d*)buf)[121];
      ((__m128d*)buf)[121] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[160];
      ((__m128d*)buf)[160] = ((__m128d*)buf)[5];
      ((__m128d*)buf)[5] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[161];
      ((__m128d*)buf)[161] = ((__m128d*)buf)[133];
      ((__m128d*)buf)[133] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[162];
      ((__m128d*)buf)[162] = ((__m128d*)buf)[69];
      ((__m128d*)buf)[69] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[164];
      ((__m128d*)buf)[164] = ((__m128d*)buf)[37];
      ((__m128d*)buf)[37] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[166];
      ((__m128d*)buf)[166] = ((__m128d*)buf)[101];
      ((__m128d*)buf)[101] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[168];
      ((__m128d*)buf)[168] = ((__m128d*)buf)[21];
      ((__m128d*)buf)[21] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[169];
      ((__m128d*)buf)[169] = ((__m128d*)buf)[149];
      ((__m128d*)buf)[149] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[170];
      ((__m128d*)buf)[170] = ((__m128d*)buf)[85];
      ((__m128d*)buf)[85] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[172];
      ((__m128d*)buf)[172] = ((__m128d*)buf)[53];
      ((__m128d*)buf)[53] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[174];
      ((__m128d*)buf)[174] = ((__m128d*)buf)[117];
      ((__m128d*)buf)[117] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[176];
      ((__m128d*)buf)[176] = ((__m128d*)buf)[13];
      ((__m128d*)buf)[13] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[177];
      ((__m128d*)buf)[177] = ((__m128d*)buf)[141];
      ((__m128d*)buf)[141] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[178];
      ((__m128d*)buf)[178] = ((__m128d*)buf)[77];
      ((__m128d*)buf)[77] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[180];
      ((__m128d*)buf)[180] = ((__m128d*)buf)[45];
      ((__m128d*)buf)[45] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[181];
      ((__m128d*)buf)[181] = ((__m128d*)buf)[173];
      ((__m128d*)buf)[173] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[182];
      ((__m128d*)buf)[182] = ((__m128d*)buf)[109];
      ((__m128d*)buf)[109] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[184];
      ((__m128d*)buf)[184] = ((__m128d*)buf)[29];
      ((__m128d*)buf)[29] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[185];
      ((__m128d*)buf)[185] = ((__m128d*)buf)[157];
      ((__m128d*)buf)[157] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[186];
      ((__m128d*)buf)[186] = ((__m128d*)buf)[93];
      ((__m128d*)buf)[93] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[188];
      ((__m128d*)buf)[188] = ((__m128d*)buf)[61];
      ((__m128d*)buf)[61] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[190];
      ((__m128d*)buf)[190] = ((__m128d*)buf)[125];
      ((__m128d*)buf)[125] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[192];
      ((__m128d*)buf)[192] = ((__m128d*)buf)[3];
      ((__m128d*)buf)[3] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[193];
      ((__m128d*)buf)[193] = ((__m128d*)buf)[131];
      ((__m128d*)buf)[131] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[194];
      ((__m128d*)buf)[194] = ((__m128d*)buf)[67];
      ((__m128d*)buf)[67] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[196];
      ((__m128d*)buf)[196] = ((__m128d*)buf)[35];
      ((__m128d*)buf)[35] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[197];
      ((__m128d*)buf)[197] = ((__m128d*)buf)[163];
      ((__m128d*)buf)[163] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[198];
      ((__m128d*)buf)[198] = ((__m128d*)buf)[99];
      ((__m128d*)buf)[99] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[200];
      ((__m128d*)buf)[200] = ((__m128d*)buf)[19];
      ((__m128d*)buf)[19] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[201];
      ((__m128d*)buf)[201] = ((__m128d*)buf)[147];
      ((__m128d*)buf)[147] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[202];
      ((__m128d*)buf)[202] = ((__m128d*)buf)[83];
      ((__m128d*)buf)[83] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[204];
      ((__m128d*)buf)[204] = ((__m128d*)buf)[51];
      ((__m128d*)buf)[51] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[205];
      ((__m128d*)buf)[205] = ((__m128d*)buf)[179];
      ((__m128d*)buf)[179] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[206];
      ((__m128d*)buf)[206] = ((__m128d*)buf)[115];
      ((__m128d*)buf)[115] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[208];
      ((__m128d*)buf)[208] = ((__m128d*)buf)[11];
      ((__m128d*)buf)[11] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[209];
      ((__m128d*)buf)[209] = ((__m128d*)buf)[139];
      ((__m128d*)buf)[139] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[210];
      ((__m128d*)buf)[210] = ((__m128d*)buf)[75];
      ((__m128d*)buf)[75] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[211];
      ((__m128d*)buf)[211] = ((__m128d*)buf)[203];
      ((__m128d*)buf)[203] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[212];
      ((__m128d*)buf)[212] = ((__m128d*)buf)[43];
      ((__m128d*)buf)[43] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[213];
      ((__m128d*)buf)[213] = ((__m128d*)buf)[171];
      ((__m128d*)buf)[171] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[214];
      ((__m128d*)buf)[214] = ((__m128d*)buf)[107];
      ((__m128d*)buf)[107] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[216];
      ((__m128d*)buf)[216] = ((__m128d*)buf)[27];
      ((__m128d*)buf)[27] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[217];
      ((__m128d*)buf)[217] = ((__m128d*)buf)[155];
      ((__m128d*)buf)[155] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[218];
      ((__m128d*)buf)[218] = ((__m128d*)buf)[91];
      ((__m128d*)buf)[91] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[220];
      ((__m128d*)buf)[220] = ((__m128d*)buf)[59];
      ((__m128d*)buf)[59] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[221];
      ((__m128d*)buf)[221] = ((__m128d*)buf)[187];
      ((__m128d*)buf)[187] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[222];
      ((__m128d*)buf)[222] = ((__m128d*)buf)[123];
      ((__m128d*)buf)[123] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[224];
      ((__m128d*)buf)[224] = ((__m128d*)buf)[7];
      ((__m128d*)buf)[7] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[225];
      ((__m128d*)buf)[225] = ((__m128d*)buf)[135];
      ((__m128d*)buf)[135] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[226];
      ((__m128d*)buf)[226] = ((__m128d*)buf)[71];
      ((__m128d*)buf)[71] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[227];
      ((__m128d*)buf)[227] = ((__m128d*)buf)[199];
      ((__m128d*)buf)[199] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[228];
      ((__m128d*)buf)[228] = ((__m128d*)buf)[39];
      ((__m128d*)buf)[39] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[229];
      ((__m128d*)buf)[229] = ((__m128d*)buf)[167];
      ((__m128d*)buf)[167] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[230];
      ((__m128d*)buf)[230] = ((__m128d*)buf)[103];
      ((__m128d*)buf)[103] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[232];
      ((__m128d*)buf)[232] = ((__m128d*)buf)[23];
      ((__m128d*)buf)[23] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[233];
      ((__m128d*)buf)[233] = ((__m128d*)buf)[151];
      ((__m128d*)buf)[151] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[234];
      ((__m128d*)buf)[234] = ((__m128d*)buf)[87];
      ((__m128d*)buf)[87] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[235];
      ((__m128d*)buf)[235] = ((__m128d*)buf)[215];
      ((__m128d*)buf)[215] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[236];
      ((__m128d*)buf)[236] = ((__m128d*)buf)[55];
      ((__m128d*)buf)[55] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[237];
      ((__m128d*)buf)[237] = ((__m128d*)buf)[183];
      ((__m128d*)buf)[183] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[238];
      ((__m128d*)buf)[238] = ((__m128d*)buf)[119];
      ((__m128d*)buf)[119] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[240];
      ((__m128d*)buf)[240] = ((__m128d*)buf)[15];
      ((__m128d*)buf)[15] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[241];
      ((__m128d*)buf)[241] = ((__m128d*)buf)[143];
      ((__m128d*)buf)[143] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[242];
      ((__m128d*)buf)[242] = ((__m128d*)buf)[79];
      ((__m128d*)buf)[79] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[243];
      ((__m128d*)buf)[243] = ((__m128d*)buf)[207];
      ((__m128d*)buf)[207] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[244];
      ((__m128d*)buf)[244] = ((__m128d*)buf)[47];
      ((__m128d*)buf)[47] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[245];
      ((__m128d*)buf)[245] = ((__m128d*)buf)[175];
      ((__m128d*)buf)[175] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[246];
      ((__m128d*)buf)[246] = ((__m128d*)buf)[111];
      ((__m128d*)buf)[111] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[247];
      ((__m128d*)buf)[247] = ((__m128d*)buf)[239];
      ((__m128d*)buf)[239] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[248];
      ((__m128d*)buf)[248] = ((__m128d*)buf)[31];
      ((__m128d*)buf)[31] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[249];
      ((__m128d*)buf)[249] = ((__m128d*)buf)[159];
      ((__m128d*)buf)[159] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[250];
      ((__m128d*)buf)[250] = ((__m128d*)buf)[95];
      ((__m128d*)buf)[95] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[251];
      ((__m128d*)buf)[251] = ((__m128d*)buf)[223];
      ((__m128d*)buf)[223] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[252];
      ((__m128d*)buf)[252] = ((__m128d*)buf)[63];
      ((__m128d*)buf)[63] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[253];
      ((__m128d*)buf)[253] = ((__m128d*)buf)[191];
      ((__m128d*)buf)[191] = tmp;}
      
    { __m128d tmp;
      tmp = ((__m128d*)buf)[254];
      ((__m128d*)buf)[254] = ((__m128d*)buf)[127];
      ((__m128d*)buf)[127] = tmp;}
      
    return;
  default:
    {
      unsigned bits = 0;
      { unsigned x = n; /* count bits, log(N) algo */
        { unsigned y = (x >> 16); if(y) { bits += 16; x = y; } }
        { unsigned y = (x >> 8); if(y) { bits += 8; x = y; } }
        { unsigned y = (x >> 4); if(y) { bits += 4; x = y; } }
        { unsigned y = (x >> 2); if(y) { bits += 2; x = y; } }
        { unsigned y = (x >> 1); if(y) { bits += 1; x = y; } }
      }
      {
        unsigned a, b, c, ii = (1 << bits);
        for(a = 1, b = 0; a < ii; ++a)
        {
          /* this increments b in bitreversed order */
          /* add by xor, probagate carries manually */
          /* worst case O(NlogN) but amortized O(N) */
          /* in fact it runs only 2*n-1 times total */
          /* could improve this by unroll & predict */
          for(c = ii >> 1; c; c >>= 1) { if((b^=c) & c) break; }
          if(a > b)
          {
            { __m128d tmp;
      tmp = ((__m128d*)buf)[a];
      ((__m128d*)buf)[a] = ((__m128d*)buf)[b];
      ((__m128d*)buf)[b] = tmp;}
      
          }
        }
      }
    }
    return;
  }
}


static void fft_2_sd(double * buf)
{
    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((1)<<1)]; it1 = buf[1+((1)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((1)<<1)] = rs1; buf[1+((1)<<1)] = is1;
    }

}
static void fft_4_sd(double * buf)
{
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((1)<<1)]; it1 = buf[1+((1)<<1)];
      rt2 = buf[0+((2)<<1)]; it2 = buf[1+((2)<<1)];
      rt3 = buf[0+((3)<<1)]; it3 = buf[1+((3)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((2)<<1)] = rt2; buf[1+((2)<<1)] = it2;
      buf[0+((1)<<1)] = rt1; buf[1+((1)<<1)] = it1;
      buf[0+((3)<<1)] = rt3; buf[1+((3)<<1)] = it3;
    }

}


static void fft_8_sd(double * buf)
{
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((2)<<1)]; it1 = buf[1+((2)<<1)];
      rt2 = buf[0+((4)<<1)]; it2 = buf[1+((4)<<1)];
      rt3 = buf[0+((6)<<1)]; it3 = buf[1+((6)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((2)<<1)] = rt2; buf[1+((2)<<1)] = it2;
      buf[0+((4)<<1)] = rt1; buf[1+((4)<<1)] = it1;
      buf[0+((6)<<1)] = rt3; buf[1+((6)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((1)<<1)]; it0 = buf[1+((1)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rt2 = buf[0+((5)<<1)]; it2 = buf[1+((5)<<1)];
      rt3 = buf[0+((7)<<1)]; it3 = buf[1+((7)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = rt1; rt1 += it1; it1 -= tmp;}
      rt1 *= _constant_sqrt05[0];
      it1 *= _constant_sqrt05[0];
       {double tmp = rt2; rt2 = it2; it2 = -tmp;}
       {double tmp = rt3; rt3 = it3 - rt3; it3 = -tmp - it3;}
      rt3 *= _constant_sqrt05[0];
      it3 *= _constant_sqrt05[0];
      
      buf[0+((1)<<1)] = rt0; buf[1+((1)<<1)] = it0;
      buf[0+((3)<<1)] = rt2; buf[1+((3)<<1)] = it2;
      buf[0+((5)<<1)] = rt1; buf[1+((5)<<1)] = it1;
      buf[0+((7)<<1)] = rt3; buf[1+((7)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((1)<<1)]; it1 = buf[1+((1)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((1)<<1)] = rs1; buf[1+((1)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((2)<<1)]; it0 = buf[1+((2)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((2)<<1)] = rs0; buf[1+((2)<<1)] = is0;
      buf[0+((3)<<1)] = rs1; buf[1+((3)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((4)<<1)]; it0 = buf[1+((4)<<1)];
      rt1 = buf[0+((5)<<1)]; it1 = buf[1+((5)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((4)<<1)] = rs0; buf[1+((4)<<1)] = is0;
      buf[0+((5)<<1)] = rs1; buf[1+((5)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((6)<<1)]; it0 = buf[1+((6)<<1)];
      rt1 = buf[0+((7)<<1)]; it1 = buf[1+((7)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((6)<<1)] = rs0; buf[1+((6)<<1)] = is0;
      buf[0+((7)<<1)] = rs1; buf[1+((7)<<1)] = is1;
    }

}
static void fft_16_sd(double * buf)
{
    
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((4)<<1)]; it1 = buf[1+((4)<<1)];
      rt2 = buf[0+((8)<<1)]; it2 = buf[1+((8)<<1)];
      rt3 = buf[0+((12)<<1)]; it3 = buf[1+((12)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((4)<<1)] = rs2; buf[1+((4)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
       
      buf[0+((8)<<1)] = rt1; buf[1+((8)<<1)] = it1;
      buf[0+((12)<<1)] = rt3; buf[1+((12)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((1)<<1)]; it0 = buf[1+((1)<<1)];
      rt1 = buf[0+((5)<<1)]; it1 = buf[1+((5)<<1)];
      rt2 = buf[0+((9)<<1)]; it2 = buf[1+((9)<<1)];
      rt3 = buf[0+((13)<<1)]; it3 = buf[1+((13)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((1)<<1)] = rs0; buf[1+((1)<<1)] = is0;
      buf[0+((5)<<1)] = rs2; buf[1+((5)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      { double rtmp, itmp;
      
          rtmp = _constant_cos8[0];
          itmp = -_constant_sin8[0];
      {double tmp = (rt1 * rtmp - it1 * itmp);
    it1 = (rt1 * itmp) + (rtmp * it1); rt1 = tmp;
    }
      } { double rtmp, itmp;
      
          rtmp = _constant_sin8[0];
          itmp = -_constant_cos8[0];
      {double tmp = (rt3 * rtmp - it3 * itmp);
    it3 = (rt3 * itmp) + (rtmp * it3); rt3 = tmp;
    }
      }
      buf[0+((9)<<1)] = rt1; buf[1+((9)<<1)] = it1;
      buf[0+((13)<<1)] = rt3; buf[1+((13)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((2)<<1)]; it0 = buf[1+((2)<<1)];
      rt1 = buf[0+((6)<<1)]; it1 = buf[1+((6)<<1)];
      rt2 = buf[0+((10)<<1)]; it2 = buf[1+((10)<<1)];
      rt3 = buf[0+((14)<<1)]; it3 = buf[1+((14)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((2)<<1)] = rs0; buf[1+((2)<<1)] = is0;
      buf[0+((6)<<1)] = rs2; buf[1+((6)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = rt1; rt1 += it1; it1 -= tmp;}
      rt1 *= _constant_sqrt05[0];
      it1 *= _constant_sqrt05[0];
       {double tmp = rt3; rt3 = it3 - rt3; it3 = -tmp - it3;}
      rt3 *= _constant_sqrt05[0];
      it3 *= _constant_sqrt05[0];
      
      buf[0+((10)<<1)] = rt1; buf[1+((10)<<1)] = it1;
      buf[0+((14)<<1)] = rt3; buf[1+((14)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((3)<<1)]; it0 = buf[1+((3)<<1)];
      rt1 = buf[0+((7)<<1)]; it1 = buf[1+((7)<<1)];
      rt2 = buf[0+((11)<<1)]; it2 = buf[1+((11)<<1)];
      rt3 = buf[0+((15)<<1)]; it3 = buf[1+((15)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((3)<<1)] = rs0; buf[1+((3)<<1)] = is0;
      buf[0+((7)<<1)] = rs2; buf[1+((7)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      { double rtmp, itmp;
      
          rtmp = _constant_sin8[0];
          itmp = -_constant_cos8[0];
      {double tmp = (rt1 * rtmp - it1 * itmp);
    it1 = (rt1 * itmp) + (rtmp * it1); rt1 = tmp;
    }
      } { double rtmp, itmp;
      
          rtmp = -_constant_cos8[0];
          itmp = _constant_sin8[0];
      {double tmp = (rt3 * rtmp - it3 * itmp);
    it3 = (rt3 * itmp) + (rtmp * it3); rt3 = tmp;
    }
      }
      buf[0+((11)<<1)] = rt1; buf[1+((11)<<1)] = it1;
      buf[0+((15)<<1)] = rt3; buf[1+((15)<<1)] = it3;
    }

    /* 8-point sub */
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((2)<<1)]; it1 = buf[1+((2)<<1)];
      rt2 = buf[0+((4)<<1)]; it2 = buf[1+((4)<<1)];
      rt3 = buf[0+((6)<<1)]; it3 = buf[1+((6)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((2)<<1)] = rt2; buf[1+((2)<<1)] = it2;
      buf[0+((4)<<1)] = rt1; buf[1+((4)<<1)] = it1;
      buf[0+((6)<<1)] = rt3; buf[1+((6)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((1)<<1)]; it0 = buf[1+((1)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rt2 = buf[0+((5)<<1)]; it2 = buf[1+((5)<<1)];
      rt3 = buf[0+((7)<<1)]; it3 = buf[1+((7)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = rt1; rt1 += it1; it1 -= tmp;}
      rt1 *= _constant_sqrt05[0];
      it1 *= _constant_sqrt05[0];
       {double tmp = rt2; rt2 = it2; it2 = -tmp;}
       {double tmp = rt3; rt3 = it3 - rt3; it3 = -tmp - it3;}
      rt3 *= _constant_sqrt05[0];
      it3 *= _constant_sqrt05[0];
      
      buf[0+((1)<<1)] = rt0; buf[1+((1)<<1)] = it0;
      buf[0+((3)<<1)] = rt2; buf[1+((3)<<1)] = it2;
      buf[0+((5)<<1)] = rt1; buf[1+((5)<<1)] = it1;
      buf[0+((7)<<1)] = rt3; buf[1+((7)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((1)<<1)]; it1 = buf[1+((1)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((1)<<1)] = rs1; buf[1+((1)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((2)<<1)]; it0 = buf[1+((2)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((2)<<1)] = rs0; buf[1+((2)<<1)] = is0;
      buf[0+((3)<<1)] = rs1; buf[1+((3)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((4)<<1)]; it0 = buf[1+((4)<<1)];
      rt1 = buf[0+((5)<<1)]; it1 = buf[1+((5)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((4)<<1)] = rs0; buf[1+((4)<<1)] = is0;
      buf[0+((5)<<1)] = rs1; buf[1+((5)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((6)<<1)]; it0 = buf[1+((6)<<1)];
      rt1 = buf[0+((7)<<1)]; it1 = buf[1+((7)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((6)<<1)] = rs0; buf[1+((6)<<1)] = is0;
      buf[0+((7)<<1)] = rs1; buf[1+((7)<<1)] = is1;
    }

    /* 2 x 4-point subs */
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((8)<<1)]; it0 = buf[1+((8)<<1)];
      rt1 = buf[0+((9)<<1)]; it1 = buf[1+((9)<<1)];
      rt2 = buf[0+((10)<<1)]; it2 = buf[1+((10)<<1)];
      rt3 = buf[0+((11)<<1)]; it3 = buf[1+((11)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((8)<<1)] = rt0; buf[1+((8)<<1)] = it0;
      buf[0+((9)<<1)] = rt2; buf[1+((9)<<1)] = it2;
      buf[0+((10)<<1)] = rt1; buf[1+((10)<<1)] = it1;
      buf[0+((11)<<1)] = rt3; buf[1+((11)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((12)<<1)]; it0 = buf[1+((12)<<1)];
      rt1 = buf[0+((13)<<1)]; it1 = buf[1+((13)<<1)];
      rt2 = buf[0+((14)<<1)]; it2 = buf[1+((14)<<1)];
      rt3 = buf[0+((15)<<1)]; it3 = buf[1+((15)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((12)<<1)] = rt0; buf[1+((12)<<1)] = it0;
      buf[0+((13)<<1)] = rt2; buf[1+((13)<<1)] = it2;
      buf[0+((14)<<1)] = rt1; buf[1+((14)<<1)] = it1;
      buf[0+((15)<<1)] = rt3; buf[1+((15)<<1)] = it3;
    }

    
}

static void fft_8n_sd(double * buf, unsigned n)
{

  unsigned nn;
  
  for(nn = 0; nn < n; nn += 8)
  {

    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((2)<<1)]; it1 = buf[1+((2)<<1)];
      rt2 = buf[0+((4)<<1)]; it2 = buf[1+((4)<<1)];
      rt3 = buf[0+((6)<<1)]; it3 = buf[1+((6)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((2)<<1)] = rt2; buf[1+((2)<<1)] = it2;
      buf[0+((4)<<1)] = rt1; buf[1+((4)<<1)] = it1;
      buf[0+((6)<<1)] = rt3; buf[1+((6)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((1)<<1)]; it0 = buf[1+((1)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rt2 = buf[0+((5)<<1)]; it2 = buf[1+((5)<<1)];
      rt3 = buf[0+((7)<<1)]; it3 = buf[1+((7)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = rt1; rt1 += it1; it1 -= tmp;}
      rt1 *= _constant_sqrt05[0];
      it1 *= _constant_sqrt05[0];
       {double tmp = rt2; rt2 = it2; it2 = -tmp;}
       {double tmp = rt3; rt3 = it3 - rt3; it3 = -tmp - it3;}
      rt3 *= _constant_sqrt05[0];
      it3 *= _constant_sqrt05[0];
      
      buf[0+((1)<<1)] = rt0; buf[1+((1)<<1)] = it0;
      buf[0+((3)<<1)] = rt2; buf[1+((3)<<1)] = it2;
      buf[0+((5)<<1)] = rt1; buf[1+((5)<<1)] = it1;
      buf[0+((7)<<1)] = rt3; buf[1+((7)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((1)<<1)]; it1 = buf[1+((1)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((1)<<1)] = rs1; buf[1+((1)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((2)<<1)]; it0 = buf[1+((2)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((2)<<1)] = rs0; buf[1+((2)<<1)] = is0;
      buf[0+((3)<<1)] = rs1; buf[1+((3)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((4)<<1)]; it0 = buf[1+((4)<<1)];
      rt1 = buf[0+((5)<<1)]; it1 = buf[1+((5)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((4)<<1)] = rs0; buf[1+((4)<<1)] = is0;
      buf[0+((5)<<1)] = rs1; buf[1+((5)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((6)<<1)]; it0 = buf[1+((6)<<1)];
      rt1 = buf[0+((7)<<1)]; it1 = buf[1+((7)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((6)<<1)] = rs0; buf[1+((6)<<1)] = is0;
      buf[0+((7)<<1)] = rs1; buf[1+((7)<<1)] = is1;
    }

  buf += 16;
  
  }
  
}


static void fft_16n_sd(double * buf, unsigned n)
{

  unsigned nn;
  
  for(nn = 0; nn < n; nn += 16)
  {

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((4)<<1)]; it1 = buf[1+((4)<<1)];
      rt2 = buf[0+((8)<<1)]; it2 = buf[1+((8)<<1)];
      rt3 = buf[0+((12)<<1)]; it3 = buf[1+((12)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((4)<<1)] = rs2; buf[1+((4)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
       
      buf[0+((8)<<1)] = rt1; buf[1+((8)<<1)] = it1;
      buf[0+((12)<<1)] = rt3; buf[1+((12)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((1)<<1)]; it0 = buf[1+((1)<<1)];
      rt1 = buf[0+((5)<<1)]; it1 = buf[1+((5)<<1)];
      rt2 = buf[0+((9)<<1)]; it2 = buf[1+((9)<<1)];
      rt3 = buf[0+((13)<<1)]; it3 = buf[1+((13)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((1)<<1)] = rs0; buf[1+((1)<<1)] = is0;
      buf[0+((5)<<1)] = rs2; buf[1+((5)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      { double rtmp, itmp;
      
          rtmp = _constant_cos8[0];
          itmp = -_constant_sin8[0];
      {double tmp = (rt1 * rtmp - it1 * itmp);
    it1 = (rt1 * itmp) + (rtmp * it1); rt1 = tmp;
    }
      } { double rtmp, itmp;
      
          rtmp = _constant_sin8[0];
          itmp = -_constant_cos8[0];
      {double tmp = (rt3 * rtmp - it3 * itmp);
    it3 = (rt3 * itmp) + (rtmp * it3); rt3 = tmp;
    }
      }
      buf[0+((9)<<1)] = rt1; buf[1+((9)<<1)] = it1;
      buf[0+((13)<<1)] = rt3; buf[1+((13)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((2)<<1)]; it0 = buf[1+((2)<<1)];
      rt1 = buf[0+((6)<<1)]; it1 = buf[1+((6)<<1)];
      rt2 = buf[0+((10)<<1)]; it2 = buf[1+((10)<<1)];
      rt3 = buf[0+((14)<<1)]; it3 = buf[1+((14)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((2)<<1)] = rs0; buf[1+((2)<<1)] = is0;
      buf[0+((6)<<1)] = rs2; buf[1+((6)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = rt1; rt1 += it1; it1 -= tmp;}
      rt1 *= _constant_sqrt05[0];
      it1 *= _constant_sqrt05[0];
       {double tmp = rt3; rt3 = it3 - rt3; it3 = -tmp - it3;}
      rt3 *= _constant_sqrt05[0];
      it3 *= _constant_sqrt05[0];
      
      buf[0+((10)<<1)] = rt1; buf[1+((10)<<1)] = it1;
      buf[0+((14)<<1)] = rt3; buf[1+((14)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((3)<<1)]; it0 = buf[1+((3)<<1)];
      rt1 = buf[0+((7)<<1)]; it1 = buf[1+((7)<<1)];
      rt2 = buf[0+((11)<<1)]; it2 = buf[1+((11)<<1)];
      rt3 = buf[0+((15)<<1)]; it3 = buf[1+((15)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      buf[0+((3)<<1)] = rs0; buf[1+((3)<<1)] = is0;
      buf[0+((7)<<1)] = rs2; buf[1+((7)<<1)] = is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      { double rtmp, itmp;
      
          rtmp = _constant_sin8[0];
          itmp = -_constant_cos8[0];
      {double tmp = (rt1 * rtmp - it1 * itmp);
    it1 = (rt1 * itmp) + (rtmp * it1); rt1 = tmp;
    }
      } { double rtmp, itmp;
      
          rtmp = -_constant_cos8[0];
          itmp = _constant_sin8[0];
      {double tmp = (rt3 * rtmp - it3 * itmp);
    it3 = (rt3 * itmp) + (rtmp * it3); rt3 = tmp;
    }
      }
      buf[0+((11)<<1)] = rt1; buf[1+((11)<<1)] = it1;
      buf[0+((15)<<1)] = rt3; buf[1+((15)<<1)] = it3;
    }

    /* 8-point sub */
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((2)<<1)]; it1 = buf[1+((2)<<1)];
      rt2 = buf[0+((4)<<1)]; it2 = buf[1+((4)<<1)];
      rt3 = buf[0+((6)<<1)]; it3 = buf[1+((6)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((2)<<1)] = rt2; buf[1+((2)<<1)] = it2;
      buf[0+((4)<<1)] = rt1; buf[1+((4)<<1)] = it1;
      buf[0+((6)<<1)] = rt3; buf[1+((6)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((1)<<1)]; it0 = buf[1+((1)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rt2 = buf[0+((5)<<1)]; it2 = buf[1+((5)<<1)];
      rt3 = buf[0+((7)<<1)]; it3 = buf[1+((7)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = rt1; rt1 += it1; it1 -= tmp;}
      rt1 *= _constant_sqrt05[0];
      it1 *= _constant_sqrt05[0];
       {double tmp = rt2; rt2 = it2; it2 = -tmp;}
       {double tmp = rt3; rt3 = it3 - rt3; it3 = -tmp - it3;}
      rt3 *= _constant_sqrt05[0];
      it3 *= _constant_sqrt05[0];
      
      buf[0+((1)<<1)] = rt0; buf[1+((1)<<1)] = it0;
      buf[0+((3)<<1)] = rt2; buf[1+((3)<<1)] = it2;
      buf[0+((5)<<1)] = rt1; buf[1+((5)<<1)] = it1;
      buf[0+((7)<<1)] = rt3; buf[1+((7)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((1)<<1)]; it1 = buf[1+((1)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((0)<<1)] = rs0; buf[1+((0)<<1)] = is0;
      buf[0+((1)<<1)] = rs1; buf[1+((1)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((2)<<1)]; it0 = buf[1+((2)<<1)];
      rt1 = buf[0+((3)<<1)]; it1 = buf[1+((3)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((2)<<1)] = rs0; buf[1+((2)<<1)] = is0;
      buf[0+((3)<<1)] = rs1; buf[1+((3)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((4)<<1)]; it0 = buf[1+((4)<<1)];
      rt1 = buf[0+((5)<<1)]; it1 = buf[1+((5)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((4)<<1)] = rs0; buf[1+((4)<<1)] = is0;
      buf[0+((5)<<1)] = rs1; buf[1+((5)<<1)] = is1;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rs0, is0;
      double rs1, is1;
      rt0 = buf[0+((6)<<1)]; it0 = buf[1+((6)<<1)];
      rt1 = buf[0+((7)<<1)]; it1 = buf[1+((7)<<1)];
      rs0 = rt0 + rt1; is0 = it0 + it1;
      rs1 = rt0 - rt1; is1 = it0 - it1;
      buf[0+((6)<<1)] = rs0; buf[1+((6)<<1)] = is0;
      buf[0+((7)<<1)] = rs1; buf[1+((7)<<1)] = is1;
    }

    /* 2 x 4-point subs */
    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((8)<<1)]; it0 = buf[1+((8)<<1)];
      rt1 = buf[0+((9)<<1)]; it1 = buf[1+((9)<<1)];
      rt2 = buf[0+((10)<<1)]; it2 = buf[1+((10)<<1)];
      rt3 = buf[0+((11)<<1)]; it3 = buf[1+((11)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((8)<<1)] = rt0; buf[1+((8)<<1)] = it0;
      buf[0+((9)<<1)] = rt2; buf[1+((9)<<1)] = it2;
      buf[0+((10)<<1)] = rt1; buf[1+((10)<<1)] = it1;
      buf[0+((11)<<1)] = rt3; buf[1+((11)<<1)] = it3;
    }

    
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((12)<<1)]; it0 = buf[1+((12)<<1)];
      rt1 = buf[0+((13)<<1)]; it1 = buf[1+((13)<<1)];
      rt2 = buf[0+((14)<<1)]; it2 = buf[1+((14)<<1)];
      rt3 = buf[0+((15)<<1)]; it3 = buf[1+((15)<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
        
      buf[0+((12)<<1)] = rt0; buf[1+((12)<<1)] = it0;
      buf[0+((13)<<1)] = rt2; buf[1+((13)<<1)] = it2;
      buf[0+((14)<<1)] = rt1; buf[1+((14)<<1)] = it1;
      buf[0+((15)<<1)] = rt3; buf[1+((15)<<1)] = it3;
    }

    
  buf += 32;
  
  }
  
}

  

    
static void fft_r4_sd(double * buf, unsigned n)
{

      double rww, iww;
      double rw1, iw1;
      double rw2, iw2;
      double rw3, iw3;
      unsigned ni, n4 = n >> 2;
      {double tmp = sin(_constant_pi/n);
    rww = -2*tmp*tmp; iww = -sin((2*_constant_pi)/n);
    rw1 = 1; iw1 = 0;
    }
      
  for(ni = 0; ni < n4; ni += 1)
  {

        rw2 = rw1; iw2 = iw1;
    
        {double tmp = (rw2 * rw2 - iw2 * iw2);
    iw2 = (rw2 * iw2) + (rw2 * iw2); rw2 = tmp;
    }
        rw3 = rw2; iw3 = iw2;
    
        {double tmp = (rw3 * rw1 - iw3 * iw1);
    iw3 = (rw3 * iw1) + (rw1 * iw3); rw3 = tmp;
    }
        
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((n4)<<1)]; it1 = buf[1+((n4)<<1)];
      rt2 = buf[0+(((n4<<1))<<1)]; it2 = buf[1+(((n4<<1))<<1)];
      rt3 = buf[0+((((n4<<1)+n4))<<1)]; it3 = buf[1+((((n4<<1)+n4))<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = (rt1 * rw1 - it1 * iw1);
    it1 = (rt1 * iw1) + (rw1 * it1); rt1 = tmp;
    } {double tmp = (rt2 * rw2 - it2 * iw2);
    it2 = (rt2 * iw2) + (rw2 * it2); rt2 = tmp;
    } {double tmp = (rt3 * rw3 - it3 * iw3);
    it3 = (rt3 * iw3) + (rw3 * it3); rt3 = tmp;
    }
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((n4)<<1)] = rt2; buf[1+((n4)<<1)] = it2;
      buf[0+(((n4<<1))<<1)] = rt1; buf[1+(((n4<<1))<<1)] = it1;
      buf[0+((((n4<<1)+n4))<<1)] = rt3; buf[1+((((n4<<1)+n4))<<1)] = it3;
    }

        buf += 2;
        {double tmp = (rw1 * rww - iw1 * iww);
    iw1 += (rw1 * iww) + (rww * iw1); rw1 += tmp;
    }
        
  }
  
}

    
static void fft_r4s_sd(double * bufr, unsigned n, unsigned rn)
{

      double rww, iww;
      double rw1, iw1;
      double rw2, iw2;
      double rw3, iw3;
      unsigned ni, ri, n4 = n >> 2;
      {double tmp = sin(_constant_pi/n);
    rww = -2*tmp*tmp; iww = -sin((2*_constant_pi)/n);
    rw1 = 1; iw1 = 0;
    }
      
  for(ni = 0; ni < n4; ni += 1)
  {

        double * buf = bufr;
        rw2 = rw1; iw2 = iw1;
    
        {double tmp = (rw2 * rw2 - iw2 * iw2);
    iw2 = (rw2 * iw2) + (rw2 * iw2); rw2 = tmp;
    }
        rw3 = rw2; iw3 = iw2;
    
        {double tmp = (rw3 * rw1 - iw3 * iw1);
    iw3 = (rw3 * iw1) + (rw1 * iw3); rw3 = tmp;
    }
        
        
  for(ri = 0; ri < rn; ri += 1)
  {

          
    {
      double rt0, it0;
      double rt1, it1;
      double rt2, it2;
      double rt3, it3;
      double rs0, is0;
      double rs1, is1;
      double rs2, is2;
      double rs3, is3;
      rt0 = buf[0+((0)<<1)]; it0 = buf[1+((0)<<1)];
      rt1 = buf[0+((n4)<<1)]; it1 = buf[1+((n4)<<1)];
      rt2 = buf[0+(((n4<<1))<<1)]; it2 = buf[1+(((n4<<1))<<1)];
      rt3 = buf[0+((((n4<<1)+n4))<<1)]; it3 = buf[1+((((n4<<1)+n4))<<1)];
      rs0 = rt0 + rt2; is0 = it0 + it2;
      rs1 = rt0 - rt2; is1 = it0 - it2;
      rs2 = rt1 + rt3; is2 = it1 + it3;
      rs3 = it1 - it3; is3 = rt3 - rt1;
      rt0 = rs0 + rs2; it0 = is0 + is2;
      rt2 = rs0 - rs2; it2 = is0 - is2;
      rt1 = rs1 + rs3; it1 = is1 + is3;
      rt3 = rs1 - rs3; it3 = is1 - is3;
      {double tmp = (rt1 * rw1 - it1 * iw1);
    it1 = (rt1 * iw1) + (rw1 * it1); rt1 = tmp;
    } {double tmp = (rt2 * rw2 - it2 * iw2);
    it2 = (rt2 * iw2) + (rw2 * it2); rt2 = tmp;
    } {double tmp = (rt3 * rw3 - it3 * iw3);
    it3 = (rt3 * iw3) + (rw3 * it3); rt3 = tmp;
    }
      buf[0+((0)<<1)] = rt0; buf[1+((0)<<1)] = it0;
      buf[0+((n4)<<1)] = rt2; buf[1+((n4)<<1)] = it2;
      buf[0+(((n4<<1))<<1)] = rt1; buf[1+(((n4<<1))<<1)] = it1;
      buf[0+((((n4<<1)+n4))<<1)] = rt3; buf[1+((((n4<<1)+n4))<<1)] = it3;
    }

            buf += 2*n;
  }

        bufr += 2;
        {double tmp = (rw1 * rww - iw1 * iww);
    iw1 += (rw1 * iww) + (rww * iw1); rw1 += tmp;
    }
        
  }
  
}

  


static void fft_2_pd(double * buf)
{
    
    {
      __m128d t0;
      __m128d t1;
      __m128d s0;
      __m128d s1;
      t0 = ((__m128d*)buf)[0];
      t1 = ((__m128d*)buf)[1];
      s0 = _mm_add_pd(t0, t1);
      s1 = _mm_sub_pd(t0, t1);
      ((__m128d*)buf)[0] = s0;
      ((__m128d*)buf)[1] = s1;
    }

}
static void fft_4_pd(double * buf)
{
    
    {
      __m128d t0;
      __m128d t1;
      __m128d t2;
      __m128d t3;
      __m128d s0;
      __m128d s1;
      __m128d s2;
      __m128d s3;
      t0 = ((__m128d*)buf)[0];
      t1 = ((__m128d*)buf)[1];
      t2 = ((__m128d*)buf)[2];
      t3 = ((__m128d*)buf)[3];
      s0 = _mm_add_pd(t0, t2);
      s1 = _mm_sub_pd(t0, t2);
      s2 = _mm_add_pd(t1, t3);
      s3 = _mm_sub_pd(t1, t3);
        s3 = _mm_xor_pd(s3, *((__m128d*)_mask_pd_sign_lo));
        s3 = _mm_shuffle_pd(s3,s3,1);
      t0 = _mm_add_pd(s0, s2);
      t2 = _mm_sub_pd(s0, s2);
      t1 = _mm_add_pd(s1, s3);
      t3 = _mm_sub_pd(s1, s3);
        
      ((__m128d*)buf)[0] = t0;
      ((__m128d*)buf)[2] = t2;
      ((__m128d*)buf)[1] = t1;
      ((__m128d*)buf)[3] = t3;
    }

}

static void fft_8_pd2(double * buf)
{
    
    {
      __m128d rt0, it0; /* x04 c01 */
      __m128d rt1, it1; /* x15 c45 */
      __m128d rt2, it2; /* x26 c23 */
      __m128d rt3, it3; /* x37 c67 */
      __m128d rs0, is0; /* y01 */
      __m128d rs1, is1; /* y23 */
      __m128d rs2, is2; /* y45 */
      __m128d rs3, is3; /* y67 */
      
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+2]; it1 = ((__m128d*)buf)[1+2];
      rt2 = ((__m128d*)buf)[0+4]; it2 = ((__m128d*)buf)[1+4];
      rt3 = ((__m128d*)buf)[0+6]; it3 = ((__m128d*)buf)[1+6];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      rt0 = _mm_add_pd(rs0, rs2); it0 = _mm_add_pd(is0, is2);
      rt2 = _mm_sub_pd(rs0, rs2); it2 = _mm_sub_pd(is0, is2);
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* unpacking the cmplx2 format */
      {
       __m128d tmp = _mm_unpackhi_pd(rt0,it0);
       rt0 = _mm_unpacklo_pd(rt0,it0); it0 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt2,it2);
       rt2 = _mm_unpacklo_pd(rt2,it2); it2 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      /* now rt0 it0 is the first pair, etc   */
      /* radix-4 twiddles for c3,c5,c7        */
      /* these are now found in it1 it2 it3   */
      { __m128d tmp = it1;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it1 = _mm_add_pd(it1, tmp);}
      it1 = _mm_mul_pd(it1, *((__m128d*)_constant_sqrt05));
      
      it2 = _mm_xor_pd(it2, *((__m128d*)_mask_pd_sign_lo));
      it2 = _mm_shuffle_pd(it2,it2,1);
      { __m128d tmp = it3;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it3 = _mm_sub_pd(tmp, it3);}
      it3 = _mm_mul_pd(it3, *((__m128d*)_constant_sqrt05));
      
      rs0 = _mm_add_pd(rt0, it0);
      is0 = _mm_sub_pd(rt0, it0);
      ((__m128d*)buf)[0] = rs0;
      ((__m128d*)buf)[1] = is0;
      rs2 = _mm_add_pd(rt2, it2);
      is2 = _mm_sub_pd(rt2, it2);
      ((__m128d*)buf)[2] = rs2;
      ((__m128d*)buf)[3] = is2;
      rs1 = _mm_add_pd(rt1, it1);
      is1 = _mm_sub_pd(rt1, it1);
      ((__m128d*)buf)[4] = rs1;
      ((__m128d*)buf)[5] = is1;
      rs3 = _mm_add_pd(rt3, it3);
      is3 = _mm_sub_pd(rt3, it3);
      ((__m128d*)buf)[6] = rs3;
      ((__m128d*)buf)[7] = is3;
    }

}
static void fft_16_pd2(double * buf)
{
    
    {
      __m128d rt0, it0;
      __m128d rt1, it1;
      __m128d rt2, it2;
      __m128d rt3, it3;
      __m128d rs0, is0;
      __m128d rs1, is1;
      __m128d rs2, is2;
      __m128d rs3, is3;
      
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+4]; it1 = ((__m128d*)buf)[1+4];
      rt2 = ((__m128d*)buf)[0+8]; it2 = ((__m128d*)buf)[1+8];
      rt3 = ((__m128d*)buf)[0+12]; it3 = ((__m128d*)buf)[1+12];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      ((__m128d*)buf)[0+0] = rs0; ((__m128d*)buf)[1+0] = is0;
      ((__m128d*)buf)[0+4] = rs2; ((__m128d*)buf)[1+4] = is2;
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* unpack, then twiddle in unpacked format */
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_cos8), it1);
      it1 = _mm_shuffle_pd(it1,it1,1);
      it1 = _mm_xor_pd(it1, *((__m128d*)_mask_pd_sign_lo));
      it1 = _mm_mul_pd(*((__m128d*)_constant_sin8), it1);
      it1 = _mm_sub_pd(tmp, it1);
      }
      ((__m128d*)buf)[0+8] = rt1; ((__m128d*)buf)[1+8] = it1;
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_sin8), it3);
      it3 = _mm_shuffle_pd(it3,it3,1);
      it3 = _mm_xor_pd(it3, *((__m128d*)_mask_pd_sign_lo));
      it3 = _mm_mul_pd(*((__m128d*)_constant_cos8), it3);
      it3 = _mm_sub_pd(tmp, it3);
      }
      ((__m128d*)buf)[0+12] = rt3; ((__m128d*)buf)[1+12] = it3;
    
      rt0 = ((__m128d*)buf)[0+2]; it0 = ((__m128d*)buf)[1+2];
      rt1 = ((__m128d*)buf)[0+6]; it1 = ((__m128d*)buf)[1+6];
      rt2 = ((__m128d*)buf)[0+10]; it2 = ((__m128d*)buf)[1+10];
      rt3 = ((__m128d*)buf)[0+14]; it3 = ((__m128d*)buf)[1+14];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      ((__m128d*)buf)[0+2] = rs0; ((__m128d*)buf)[1+2] = is0;
      ((__m128d*)buf)[0+6] = rs2; ((__m128d*)buf)[1+6] = is2;
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* do pi/8 twiddles, unpack, patch the second set with pi/16 */
      /* not sure if this is actually fastest, could profile more  */
      /* {__m128d tmp = rt1; 
      rt1 = _mm_add_pd(rt1,it1); 
      it1 = _mm_sub_pd(it1,tmp);
      }
      rt1 = _mm_mul_pd(rt1, *((__m128d*)_constant_sqrt05));
    it1 = _mm_mul_pd(it1, *((__m128d*)_constant_sqrt05));
    
       */
      /* {__m128d tmp = rt3;
      rt3 = _mm_sub_pd(it3, rt3);
      it3 = _mm_sub_pd(_mm_setzero_pd(), _mm_add_pd(tmp, it3));}
      rt3 = _mm_mul_pd(rt3, *((__m128d*)_constant_sqrt05));
    it3 = _mm_mul_pd(it3, *((__m128d*)_constant_sqrt05));
    
       */
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      { __m128d tmp = rt1;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      rt1 = _mm_add_pd(rt1, tmp);}
      rt1 = _mm_mul_pd(rt1, *((__m128d*)_constant_sqrt05));
      
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_sin8), it1);
      it1 = _mm_shuffle_pd(it1,it1,1);
      it1 = _mm_xor_pd(it1, *((__m128d*)_mask_pd_sign_lo));
      it1 = _mm_mul_pd(*((__m128d*)_constant_cos8), it1);
      it1 = _mm_sub_pd(tmp, it1);
      }
      ((__m128d*)buf)[0+10] = rt1; ((__m128d*)buf)[1+10] = it1;
      { __m128d tmp = rt3;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      rt3 = _mm_sub_pd(tmp, rt3);}
      rt3 = _mm_mul_pd(rt3, *((__m128d*)_constant_sqrt05));
      
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_cos8), it3);
      it3 = _mm_shuffle_pd(it3,it3,1);
      it3 = _mm_xor_pd(it3, *((__m128d*)_mask_pd_sign_lo));
      it3 = _mm_mul_pd(*((__m128d*)_constant_sin8), it3);
      it3 = _mm_sub_pd(it3, tmp);
      }
      ((__m128d*)buf)[0+14] = rt3; ((__m128d*)buf)[1+14] = it3;
    }
    
    {
      __m128d t0;
      __m128d t1;
      __m128d t2;
      __m128d t3;
      __m128d s0;
      __m128d s1;
      __m128d s2;
      __m128d s3;
      t0 = ((__m128d*)buf)[8];
      t1 = ((__m128d*)buf)[9];
      t2 = ((__m128d*)buf)[10];
      t3 = ((__m128d*)buf)[11];
      s0 = _mm_add_pd(t0, t2);
      s1 = _mm_sub_pd(t0, t2);
      s2 = _mm_add_pd(t1, t3);
      s3 = _mm_sub_pd(t1, t3);
        s3 = _mm_xor_pd(s3, *((__m128d*)_mask_pd_sign_lo));
        s3 = _mm_shuffle_pd(s3,s3,1);
      t0 = _mm_add_pd(s0, s2);
      t2 = _mm_sub_pd(s0, s2);
      t1 = _mm_add_pd(s1, s3);
      t3 = _mm_sub_pd(s1, s3);
        
      ((__m128d*)buf)[8] = t0;
      ((__m128d*)buf)[9] = t2;
      ((__m128d*)buf)[10] = t1;
      ((__m128d*)buf)[11] = t3;
    }

    
    {
      __m128d t0;
      __m128d t1;
      __m128d t2;
      __m128d t3;
      __m128d s0;
      __m128d s1;
      __m128d s2;
      __m128d s3;
      t0 = ((__m128d*)buf)[12];
      t1 = ((__m128d*)buf)[13];
      t2 = ((__m128d*)buf)[14];
      t3 = ((__m128d*)buf)[15];
      s0 = _mm_add_pd(t0, t2);
      s1 = _mm_sub_pd(t0, t2);
      s2 = _mm_add_pd(t1, t3);
      s3 = _mm_sub_pd(t1, t3);
        s3 = _mm_xor_pd(s3, *((__m128d*)_mask_pd_sign_lo));
        s3 = _mm_shuffle_pd(s3,s3,1);
      t0 = _mm_add_pd(s0, s2);
      t2 = _mm_sub_pd(s0, s2);
      t1 = _mm_add_pd(s1, s3);
      t3 = _mm_sub_pd(s1, s3);
        
      ((__m128d*)buf)[12] = t0;
      ((__m128d*)buf)[13] = t2;
      ((__m128d*)buf)[14] = t1;
      ((__m128d*)buf)[15] = t3;
    }


    {
      __m128d rt0, it0; /* x04 c01 */
      __m128d rt1, it1; /* x15 c45 */
      __m128d rt2, it2; /* x26 c23 */
      __m128d rt3, it3; /* x37 c67 */
      __m128d rs0, is0; /* y01 */
      __m128d rs1, is1; /* y23 */
      __m128d rs2, is2; /* y45 */
      __m128d rs3, is3; /* y67 */
      
      /* sub 8 point for the split radix */
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+2]; it1 = ((__m128d*)buf)[1+2];
      rt2 = ((__m128d*)buf)[0+4]; it2 = ((__m128d*)buf)[1+4];
      rt3 = ((__m128d*)buf)[0+6]; it3 = ((__m128d*)buf)[1+6];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      rt0 = _mm_add_pd(rs0, rs2); it0 = _mm_add_pd(is0, is2);
      rt2 = _mm_sub_pd(rs0, rs2); it2 = _mm_sub_pd(is0, is2);
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* unpacking the cmplx2 format */
      {
       __m128d tmp = _mm_unpackhi_pd(rt0,it0);
       rt0 = _mm_unpacklo_pd(rt0,it0); it0 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt2,it2);
       rt2 = _mm_unpacklo_pd(rt2,it2); it2 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      /* now rt0 it0 is the first pair, etc   */
      /* radix-4 twiddles for c3,c5,c7        */
      /* these are now found in it1 it2 it3   */
      { __m128d tmp = it1;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it1 = _mm_add_pd(it1, tmp);}
      it1 = _mm_mul_pd(it1, *((__m128d*)_constant_sqrt05));
      
      it2 = _mm_xor_pd(it2, *((__m128d*)_mask_pd_sign_lo));
      it2 = _mm_shuffle_pd(it2,it2,1);
      { __m128d tmp = it3;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it3 = _mm_sub_pd(tmp, it3);}
      it3 = _mm_mul_pd(it3, *((__m128d*)_constant_sqrt05));
      
      rs0 = _mm_add_pd(rt0, it0);
      is0 = _mm_sub_pd(rt0, it0);
      ((__m128d*)buf)[0] = rs0;
      ((__m128d*)buf)[1] = is0;
      rs2 = _mm_add_pd(rt2, it2);
      is2 = _mm_sub_pd(rt2, it2);
      ((__m128d*)buf)[2] = rs2;
      ((__m128d*)buf)[3] = is2;
      rs1 = _mm_add_pd(rt1, it1);
      is1 = _mm_sub_pd(rt1, it1);
      ((__m128d*)buf)[4] = rs1;
      ((__m128d*)buf)[5] = is1;
      rs3 = _mm_add_pd(rt3, it3);
      is3 = _mm_sub_pd(rt3, it3);
      ((__m128d*)buf)[6] = rs3;
      ((__m128d*)buf)[7] = is3;
    }

}

static void fft_8n_pd2(double * buf, unsigned n)
{

  unsigned nn;
  
  for(nn = 0; nn < n; nn += 8)
  {

    {
      __m128d rt0, it0; /* x04 c01 */
      __m128d rt1, it1; /* x15 c45 */
      __m128d rt2, it2; /* x26 c23 */
      __m128d rt3, it3; /* x37 c67 */
      __m128d rs0, is0; /* y01 */
      __m128d rs1, is1; /* y23 */
      __m128d rs2, is2; /* y45 */
      __m128d rs3, is3; /* y67 */
      
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+2]; it1 = ((__m128d*)buf)[1+2];
      rt2 = ((__m128d*)buf)[0+4]; it2 = ((__m128d*)buf)[1+4];
      rt3 = ((__m128d*)buf)[0+6]; it3 = ((__m128d*)buf)[1+6];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      rt0 = _mm_add_pd(rs0, rs2); it0 = _mm_add_pd(is0, is2);
      rt2 = _mm_sub_pd(rs0, rs2); it2 = _mm_sub_pd(is0, is2);
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* unpacking the cmplx2 format */
      {
       __m128d tmp = _mm_unpackhi_pd(rt0,it0);
       rt0 = _mm_unpacklo_pd(rt0,it0); it0 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt2,it2);
       rt2 = _mm_unpacklo_pd(rt2,it2); it2 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      /* now rt0 it0 is the first pair, etc   */
      /* radix-4 twiddles for c3,c5,c7        */
      /* these are now found in it1 it2 it3   */
      { __m128d tmp = it1;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it1 = _mm_add_pd(it1, tmp);}
      it1 = _mm_mul_pd(it1, *((__m128d*)_constant_sqrt05));
      
      it2 = _mm_xor_pd(it2, *((__m128d*)_mask_pd_sign_lo));
      it2 = _mm_shuffle_pd(it2,it2,1);
      { __m128d tmp = it3;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it3 = _mm_sub_pd(tmp, it3);}
      it3 = _mm_mul_pd(it3, *((__m128d*)_constant_sqrt05));
      
      rs0 = _mm_add_pd(rt0, it0);
      is0 = _mm_sub_pd(rt0, it0);
      ((__m128d*)buf)[0] = rs0;
      ((__m128d*)buf)[1] = is0;
      rs2 = _mm_add_pd(rt2, it2);
      is2 = _mm_sub_pd(rt2, it2);
      ((__m128d*)buf)[2] = rs2;
      ((__m128d*)buf)[3] = is2;
      rs1 = _mm_add_pd(rt1, it1);
      is1 = _mm_sub_pd(rt1, it1);
      ((__m128d*)buf)[4] = rs1;
      ((__m128d*)buf)[5] = is1;
      rs3 = _mm_add_pd(rt3, it3);
      is3 = _mm_sub_pd(rt3, it3);
      ((__m128d*)buf)[6] = rs3;
      ((__m128d*)buf)[7] = is3;
    }

  buf += 16;
  
  }
  
}


static void fft_16n_pd2(double * buf, unsigned n)
{

  unsigned nn;
  
  for(nn = 0; nn < n; nn += 16)
  {

    {
      __m128d rt0, it0;
      __m128d rt1, it1;
      __m128d rt2, it2;
      __m128d rt3, it3;
      __m128d rs0, is0;
      __m128d rs1, is1;
      __m128d rs2, is2;
      __m128d rs3, is3;
      
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+4]; it1 = ((__m128d*)buf)[1+4];
      rt2 = ((__m128d*)buf)[0+8]; it2 = ((__m128d*)buf)[1+8];
      rt3 = ((__m128d*)buf)[0+12]; it3 = ((__m128d*)buf)[1+12];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      ((__m128d*)buf)[0+0] = rs0; ((__m128d*)buf)[1+0] = is0;
      ((__m128d*)buf)[0+4] = rs2; ((__m128d*)buf)[1+4] = is2;
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* unpack, then twiddle in unpacked format */
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_cos8), it1);
      it1 = _mm_shuffle_pd(it1,it1,1);
      it1 = _mm_xor_pd(it1, *((__m128d*)_mask_pd_sign_lo));
      it1 = _mm_mul_pd(*((__m128d*)_constant_sin8), it1);
      it1 = _mm_sub_pd(tmp, it1);
      }
      ((__m128d*)buf)[0+8] = rt1; ((__m128d*)buf)[1+8] = it1;
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_sin8), it3);
      it3 = _mm_shuffle_pd(it3,it3,1);
      it3 = _mm_xor_pd(it3, *((__m128d*)_mask_pd_sign_lo));
      it3 = _mm_mul_pd(*((__m128d*)_constant_cos8), it3);
      it3 = _mm_sub_pd(tmp, it3);
      }
      ((__m128d*)buf)[0+12] = rt3; ((__m128d*)buf)[1+12] = it3;
    
      rt0 = ((__m128d*)buf)[0+2]; it0 = ((__m128d*)buf)[1+2];
      rt1 = ((__m128d*)buf)[0+6]; it1 = ((__m128d*)buf)[1+6];
      rt2 = ((__m128d*)buf)[0+10]; it2 = ((__m128d*)buf)[1+10];
      rt3 = ((__m128d*)buf)[0+14]; it3 = ((__m128d*)buf)[1+14];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      ((__m128d*)buf)[0+2] = rs0; ((__m128d*)buf)[1+2] = is0;
      ((__m128d*)buf)[0+6] = rs2; ((__m128d*)buf)[1+6] = is2;
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* do pi/8 twiddles, unpack, patch the second set with pi/16 */
      /* not sure if this is actually fastest, could profile more  */
      /* {__m128d tmp = rt1; 
      rt1 = _mm_add_pd(rt1,it1); 
      it1 = _mm_sub_pd(it1,tmp);
      }
      rt1 = _mm_mul_pd(rt1, *((__m128d*)_constant_sqrt05));
    it1 = _mm_mul_pd(it1, *((__m128d*)_constant_sqrt05));
    
       */
      /* {__m128d tmp = rt3;
      rt3 = _mm_sub_pd(it3, rt3);
      it3 = _mm_sub_pd(_mm_setzero_pd(), _mm_add_pd(tmp, it3));}
      rt3 = _mm_mul_pd(rt3, *((__m128d*)_constant_sqrt05));
    it3 = _mm_mul_pd(it3, *((__m128d*)_constant_sqrt05));
    
       */
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      { __m128d tmp = rt1;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      rt1 = _mm_add_pd(rt1, tmp);}
      rt1 = _mm_mul_pd(rt1, *((__m128d*)_constant_sqrt05));
      
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_sin8), it1);
      it1 = _mm_shuffle_pd(it1,it1,1);
      it1 = _mm_xor_pd(it1, *((__m128d*)_mask_pd_sign_lo));
      it1 = _mm_mul_pd(*((__m128d*)_constant_cos8), it1);
      it1 = _mm_sub_pd(tmp, it1);
      }
      ((__m128d*)buf)[0+10] = rt1; ((__m128d*)buf)[1+10] = it1;
      { __m128d tmp = rt3;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      rt3 = _mm_sub_pd(tmp, rt3);}
      rt3 = _mm_mul_pd(rt3, *((__m128d*)_constant_sqrt05));
      
      { __m128d tmp;
      tmp = _mm_mul_pd(*((__m128d*)_constant_cos8), it3);
      it3 = _mm_shuffle_pd(it3,it3,1);
      it3 = _mm_xor_pd(it3, *((__m128d*)_mask_pd_sign_lo));
      it3 = _mm_mul_pd(*((__m128d*)_constant_sin8), it3);
      it3 = _mm_sub_pd(it3, tmp);
      }
      ((__m128d*)buf)[0+14] = rt3; ((__m128d*)buf)[1+14] = it3;
    }
    
    {
      __m128d t0;
      __m128d t1;
      __m128d t2;
      __m128d t3;
      __m128d s0;
      __m128d s1;
      __m128d s2;
      __m128d s3;
      t0 = ((__m128d*)buf)[8];
      t1 = ((__m128d*)buf)[9];
      t2 = ((__m128d*)buf)[10];
      t3 = ((__m128d*)buf)[11];
      s0 = _mm_add_pd(t0, t2);
      s1 = _mm_sub_pd(t0, t2);
      s2 = _mm_add_pd(t1, t3);
      s3 = _mm_sub_pd(t1, t3);
        s3 = _mm_xor_pd(s3, *((__m128d*)_mask_pd_sign_lo));
        s3 = _mm_shuffle_pd(s3,s3,1);
      t0 = _mm_add_pd(s0, s2);
      t2 = _mm_sub_pd(s0, s2);
      t1 = _mm_add_pd(s1, s3);
      t3 = _mm_sub_pd(s1, s3);
        
      ((__m128d*)buf)[8] = t0;
      ((__m128d*)buf)[9] = t2;
      ((__m128d*)buf)[10] = t1;
      ((__m128d*)buf)[11] = t3;
    }

    
    {
      __m128d t0;
      __m128d t1;
      __m128d t2;
      __m128d t3;
      __m128d s0;
      __m128d s1;
      __m128d s2;
      __m128d s3;
      t0 = ((__m128d*)buf)[12];
      t1 = ((__m128d*)buf)[13];
      t2 = ((__m128d*)buf)[14];
      t3 = ((__m128d*)buf)[15];
      s0 = _mm_add_pd(t0, t2);
      s1 = _mm_sub_pd(t0, t2);
      s2 = _mm_add_pd(t1, t3);
      s3 = _mm_sub_pd(t1, t3);
        s3 = _mm_xor_pd(s3, *((__m128d*)_mask_pd_sign_lo));
        s3 = _mm_shuffle_pd(s3,s3,1);
      t0 = _mm_add_pd(s0, s2);
      t2 = _mm_sub_pd(s0, s2);
      t1 = _mm_add_pd(s1, s3);
      t3 = _mm_sub_pd(s1, s3);
        
      ((__m128d*)buf)[12] = t0;
      ((__m128d*)buf)[13] = t2;
      ((__m128d*)buf)[14] = t1;
      ((__m128d*)buf)[15] = t3;
    }


    {
      __m128d rt0, it0; /* x04 c01 */
      __m128d rt1, it1; /* x15 c45 */
      __m128d rt2, it2; /* x26 c23 */
      __m128d rt3, it3; /* x37 c67 */
      __m128d rs0, is0; /* y01 */
      __m128d rs1, is1; /* y23 */
      __m128d rs2, is2; /* y45 */
      __m128d rs3, is3; /* y67 */
      
      /* sub 8 point for the split radix */
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+2]; it1 = ((__m128d*)buf)[1+2];
      rt2 = ((__m128d*)buf)[0+4]; it2 = ((__m128d*)buf)[1+4];
      rt3 = ((__m128d*)buf)[0+6]; it3 = ((__m128d*)buf)[1+6];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      rt0 = _mm_add_pd(rs0, rs2); it0 = _mm_add_pd(is0, is2);
      rt2 = _mm_sub_pd(rs0, rs2); it2 = _mm_sub_pd(is0, is2);
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      /* unpacking the cmplx2 format */
      {
       __m128d tmp = _mm_unpackhi_pd(rt0,it0);
       rt0 = _mm_unpacklo_pd(rt0,it0); it0 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt1,it1);
       rt1 = _mm_unpacklo_pd(rt1,it1); it1 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt2,it2);
       rt2 = _mm_unpacklo_pd(rt2,it2); it2 = tmp;
      }
      {
       __m128d tmp = _mm_unpackhi_pd(rt3,it3);
       rt3 = _mm_unpacklo_pd(rt3,it3); it3 = tmp;
      }
      /* now rt0 it0 is the first pair, etc   */
      /* radix-4 twiddles for c3,c5,c7        */
      /* these are now found in it1 it2 it3   */
      { __m128d tmp = it1;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it1 = _mm_add_pd(it1, tmp);}
      it1 = _mm_mul_pd(it1, *((__m128d*)_constant_sqrt05));
      
      it2 = _mm_xor_pd(it2, *((__m128d*)_mask_pd_sign_lo));
      it2 = _mm_shuffle_pd(it2,it2,1);
      { __m128d tmp = it3;
      tmp = _mm_xor_pd(tmp, *((__m128d*)_mask_pd_sign_lo));
      tmp = _mm_shuffle_pd(tmp,tmp,1);
      it3 = _mm_sub_pd(tmp, it3);}
      it3 = _mm_mul_pd(it3, *((__m128d*)_constant_sqrt05));
      
      rs0 = _mm_add_pd(rt0, it0);
      is0 = _mm_sub_pd(rt0, it0);
      ((__m128d*)buf)[0] = rs0;
      ((__m128d*)buf)[1] = is0;
      rs2 = _mm_add_pd(rt2, it2);
      is2 = _mm_sub_pd(rt2, it2);
      ((__m128d*)buf)[2] = rs2;
      ((__m128d*)buf)[3] = is2;
      rs1 = _mm_add_pd(rt1, it1);
      is1 = _mm_sub_pd(rt1, it1);
      ((__m128d*)buf)[4] = rs1;
      ((__m128d*)buf)[5] = is1;
      rs3 = _mm_add_pd(rt3, it3);
      is3 = _mm_sub_pd(rt3, it3);
      ((__m128d*)buf)[6] = rs3;
      ((__m128d*)buf)[7] = is3;
    }

  buf += 32;
  
  }
  
}


    
static void fft_r4_pd2(double * buf, unsigned n)
{

      __m128d rww, iww;
      __m128d rw1, iw1;
      __m128d rw2, iw2;
      __m128d rw3, iw3;
      unsigned ni, n4 = n >> 2;
      {double rtmp, itmp;
    rtmp = sin(_constant_pi/n); rtmp *= -2*rtmp; itmp = (-sin((2*_constant_pi)/n));
    rw1 = _mm_setr_pd(1,1+rtmp); iw1 = _mm_setr_pd(0,itmp);
    rww = _mm_set1_pd(rtmp); iww = _mm_set1_pd(itmp);
    {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rww,rww),_mm_mul_pd(iww,iww));
    iww = _mm_add_pd(_mm_add_pd(iww, iww), 
          _mm_add_pd(_mm_mul_pd(rww,iww),_mm_mul_pd(rww,iww)));
    rww = _mm_add_pd(_mm_add_pd(rww, rww), tmp);
    }
    }
      
  for(ni = 0; ni < n4; ni += 2)
  {

        rw2 = rw1; iw2 = iw1;
    
        {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rw2,rw2),_mm_mul_pd(iw2,iw2));
    iw2 = _mm_add_pd(_mm_mul_pd(rw2,iw2),_mm_mul_pd(rw2,iw2));
    rw2 = tmp;
    }
        rw3 = rw2; iw3 = iw2;
    
        {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rw3,rw1),_mm_mul_pd(iw3,iw1));
    iw3 = _mm_add_pd(_mm_mul_pd(rw3,iw1),_mm_mul_pd(rw1,iw3));
    rw3 = tmp;
    }
        
    {
      __m128d rt0, it0;
      __m128d rt1, it1;
      __m128d rt2, it2;
      __m128d rt3, it3;
      __m128d rs0, is0;
      __m128d rs1, is1;
      __m128d rs2, is2;
      __m128d rs3, is3;
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+n4]; it1 = ((__m128d*)buf)[1+n4];
      rt2 = ((__m128d*)buf)[0+(n4<<1)]; it2 = ((__m128d*)buf)[1+(n4<<1)];
      rt3 = ((__m128d*)buf)[0+((n4<<1)+n4)]; it3 = ((__m128d*)buf)[1+((n4<<1)+n4)];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      rt0 = _mm_add_pd(rs0, rs2); it0 = _mm_add_pd(is0, is2);
      rt2 = _mm_sub_pd(rs0, rs2); it2 = _mm_sub_pd(is0, is2);
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rt1,rw1),_mm_mul_pd(it1,iw1));
    it1 = _mm_add_pd(_mm_mul_pd(rt1,iw1),_mm_mul_pd(rw1,it1));
    rt1 = tmp;
    } {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rt2,rw2),_mm_mul_pd(it2,iw2));
    it2 = _mm_add_pd(_mm_mul_pd(rt2,iw2),_mm_mul_pd(rw2,it2));
    rt2 = tmp;
    } {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rt3,rw3),_mm_mul_pd(it3,iw3));
    it3 = _mm_add_pd(_mm_mul_pd(rt3,iw3),_mm_mul_pd(rw3,it3));
    rt3 = tmp;
    }
      ((__m128d*)buf)[0+0] = rt0; ((__m128d*)buf)[1+0] = it0;
      ((__m128d*)buf)[0+n4] = rt2; ((__m128d*)buf)[1+n4] = it2;
      ((__m128d*)buf)[0+(n4<<1)] = rt1; ((__m128d*)buf)[1+(n4<<1)] = it1;
      ((__m128d*)buf)[0+((n4<<1)+n4)] = rt3; ((__m128d*)buf)[1+((n4<<1)+n4)] = it3;
    }

        buf += 4;
        {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rw1,rww),_mm_mul_pd(iw1,iww));
    iw1 = _mm_add_pd(iw1,_mm_add_pd(_mm_mul_pd(rw1,iww),_mm_mul_pd(rww,iw1)));
    rw1 = _mm_add_pd(rw1,tmp);
    }
        
  }
  
}

    
static void fft_r4s_pd2(double * bufr, unsigned n, unsigned rn)
{

      __m128d rww, iww;
      __m128d rw1, iw1;
      __m128d rw2, iw2;
      __m128d rw3, iw3;
      unsigned ni, ri, n4 = n >> 2;
      {double rtmp, itmp;
    rtmp = sin(_constant_pi/n); rtmp *= -2*rtmp; itmp = (-sin((2*_constant_pi)/n));
    rw1 = _mm_setr_pd(1,1+rtmp); iw1 = _mm_setr_pd(0,itmp);
    rww = _mm_set1_pd(rtmp); iww = _mm_set1_pd(itmp);
    {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rww,rww),_mm_mul_pd(iww,iww));
    iww = _mm_add_pd(_mm_add_pd(iww, iww), 
          _mm_add_pd(_mm_mul_pd(rww,iww),_mm_mul_pd(rww,iww)));
    rww = _mm_add_pd(_mm_add_pd(rww, rww), tmp);
    }
    }
      
  for(ni = 0; ni < n4; ni += 2)
  {

        double * buf = bufr;
        rw2 = rw1; iw2 = iw1;
    
        {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rw2,rw2),_mm_mul_pd(iw2,iw2));
    iw2 = _mm_add_pd(_mm_mul_pd(rw2,iw2),_mm_mul_pd(rw2,iw2));
    rw2 = tmp;
    }
        rw3 = rw2; iw3 = iw2;
    
        {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rw3,rw1),_mm_mul_pd(iw3,iw1));
    iw3 = _mm_add_pd(_mm_mul_pd(rw3,iw1),_mm_mul_pd(rw1,iw3));
    rw3 = tmp;
    }
        
        
  for(ri = 0; ri < rn; ri += 1)
  {

          
    {
      __m128d rt0, it0;
      __m128d rt1, it1;
      __m128d rt2, it2;
      __m128d rt3, it3;
      __m128d rs0, is0;
      __m128d rs1, is1;
      __m128d rs2, is2;
      __m128d rs3, is3;
      rt0 = ((__m128d*)buf)[0+0]; it0 = ((__m128d*)buf)[1+0];
      rt1 = ((__m128d*)buf)[0+n4]; it1 = ((__m128d*)buf)[1+n4];
      rt2 = ((__m128d*)buf)[0+(n4<<1)]; it2 = ((__m128d*)buf)[1+(n4<<1)];
      rt3 = ((__m128d*)buf)[0+((n4<<1)+n4)]; it3 = ((__m128d*)buf)[1+((n4<<1)+n4)];
      rs0 = _mm_add_pd(rt0, rt2); is0 = _mm_add_pd(it0, it2);
      rs1 = _mm_sub_pd(rt0, rt2); is1 = _mm_sub_pd(it0, it2);
      rs2 = _mm_add_pd(rt1, rt3); is2 = _mm_add_pd(it1, it3);
      rs3 = _mm_sub_pd(it1, it3); is3 = _mm_sub_pd(rt3, rt1);
      rt0 = _mm_add_pd(rs0, rs2); it0 = _mm_add_pd(is0, is2);
      rt2 = _mm_sub_pd(rs0, rs2); it2 = _mm_sub_pd(is0, is2);
      rt1 = _mm_add_pd(rs1, rs3); it1 = _mm_add_pd(is1, is3);
      rt3 = _mm_sub_pd(rs1, rs3); it3 = _mm_sub_pd(is1, is3);
      {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rt1,rw1),_mm_mul_pd(it1,iw1));
    it1 = _mm_add_pd(_mm_mul_pd(rt1,iw1),_mm_mul_pd(rw1,it1));
    rt1 = tmp;
    } {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rt2,rw2),_mm_mul_pd(it2,iw2));
    it2 = _mm_add_pd(_mm_mul_pd(rt2,iw2),_mm_mul_pd(rw2,it2));
    rt2 = tmp;
    } {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rt3,rw3),_mm_mul_pd(it3,iw3));
    it3 = _mm_add_pd(_mm_mul_pd(rt3,iw3),_mm_mul_pd(rw3,it3));
    rt3 = tmp;
    }
      ((__m128d*)buf)[0+0] = rt0; ((__m128d*)buf)[1+0] = it0;
      ((__m128d*)buf)[0+n4] = rt2; ((__m128d*)buf)[1+n4] = it2;
      ((__m128d*)buf)[0+(n4<<1)] = rt1; ((__m128d*)buf)[1+(n4<<1)] = it1;
      ((__m128d*)buf)[0+((n4<<1)+n4)] = rt3; ((__m128d*)buf)[1+((n4<<1)+n4)] = it3;
    }

            buf += 2*n;
  }

        bufr += 4;
        {__m128d tmp;
    tmp = _mm_sub_pd(_mm_mul_pd(rw1,rww),_mm_mul_pd(iw1,iww));
    iw1 = _mm_add_pd(iw1,_mm_add_pd(_mm_mul_pd(rw1,iww),_mm_mul_pd(rww,iw1)));
    rw1 = _mm_add_pd(rw1,tmp);
    }
        
  }
  
}

  

static void _fft_str_Da(double * buf, unsigned n, unsigned r)
{
  while(n > 16)
  {
    fft_r4s_pd2(buf, n, r);
    n >>= 2; r <<= 2;
  }
  if(n == 16)
      fft_16n_pd2(buf, n*r);
    else
      fft_8n_pd2(buf, n*r);
}

static void _fft_str_Du(double * buf, unsigned n, unsigned r)
{
  while(n > 16)
  {
    fft_r4s_sd(buf, n, r);
    n >>= 2; r <<= 2;
  }
  if(n == 16)
      fft_16n_sd(buf, n*r);
    else
      fft_8n_sd(buf, n*r);
}

static void _fft_rec_Da(double * buf, unsigned n)
{
  if(n > 16)
  {
    if(n >= 1024)
    {
      fft_r4_pd2(buf, n);
      n >>= 2;
      if(n >= 1024)
      {
        _fft_rec_Da(buf, n);
        _fft_rec_Da(buf + 2*n, n);
        _fft_rec_Da(buf + 4*n, n);
        _fft_rec_Da(buf + 6*n, n);
      }
      else
      {
        _fft_str_Da(buf, n, 4);
      }
    }
    else
    {
      _fft_str_Da(buf, n, 1);
    }
  }
  else
  {
    if(n == 16)
      fft_16_pd2(buf);
    else
      fft_8_pd2(buf);
  }
}

static void _fft_rec_Du(double * buf, unsigned n)
{
  if(n > 16)
  {
    if(n >= 1024)
    {
      fft_r4_sd(buf, n);
      n >>= 2;
      if(n >= 1024)
      {
        _fft_rec_Du(buf, n);
        _fft_rec_Du(buf + 2*n, n);
        _fft_rec_Du(buf + 4*n, n);
        _fft_rec_Du(buf + 6*n, n);
      }
      else
      {
        _fft_str_Du(buf, n, 4);
      }
    }
    else
    {
      _fft_str_Du(buf, n, 1);
    }
  }
  else
  {
    if(n == 16)
      fft_16_sd(buf);
    else
      fft_8_sd(buf);
  }
}

static void _fwd_Da(double * buf, unsigned n)
{
  switch(n)
  {
  case 2: fft_2_pd(buf); return;
  case 4: fft_4_pd(buf); return;
  default:
    {
       /* repack for nicer layout */
       double tmp;
       unsigned i, n2 = n<<1;
       for(i = 1; i < n2; i += 4)
       {
          tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
       }
       _fft_rec_Da(buf, n);
    }
  }
  _br_pd(buf, n);
}

static void _fwd_Du(double * buf, unsigned n)
{
  switch(n)
  {
  case 2: fft_2_sd(buf); return;
  case 4: fft_4_sd(buf); return;
  default: _fft_rec_Du(buf, n); break;
  }
  _br_sd(buf, n);
}

void DustFFT_fwdD(double * buf, unsigned n)
{
  if(!(((uintptr_t)buf)&0xf)) 
  {
    _fwd_Da(buf, n);
  }
  else
  {
    _fwd_Du(buf, n);
  }
}
/* reverse wrapper */
void DustFFT_revD(double * buf, unsigned n)
{
  unsigned i, n2 = n<<1;
  for(i = 0; i < n2; i += 2)
  {
    double tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
  }
  DustFFT_fwdD(buf, n);
  for(i = 0; i < n2; i += 2)
  {
    double tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
  }
}
void DustFFT_fwdDa(double * buf, unsigned n)
{
  _fwd_Da(buf, n);
}
/* reverse wrapper */
void DustFFT_revDa(double * buf, unsigned n)
{
  unsigned i, n2 = n<<1;
  for(i = 0; i < n2; i += 2)
  {
    double tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
  }
  DustFFT_fwdDa(buf, n);
  for(i = 0; i < n2; i += 2)
  {
    double tmp = buf[i]; buf[i] = buf[i+1]; buf[i+1] = tmp;
  }
}

/* real to half-buffer complex wrapper */
void DustFFT_r2cD(double * buf, unsigned n)
{
  unsigned i, n2 = n >> 1, n4 = n >> 2;
  DustFFT_fwdD(buf, n2);
  
  double rww, iww, rw1, iw1;
  {
      double tmp = sin(_constant_pi/n);
      rww = -2*tmp*tmp; iww = -sin((2*_constant_pi)/n);
      rw1 = 1; iw1 = 0;
  }
  
  for(i = 1; i < n4; ++i)
  {
    double r1, i1, r2, i2, rt, it;
    double f1, f2, f3, f4;
    unsigned a1, a2, a3, a4;
    
    {double tmp = (rw1 * rww - iw1 * iww);
    iw1 += (rw1 * iww) + (rww * iw1); rw1 += tmp;
    }

    a1 = 2*i; a2 = a1 + 1; a3 = n - a1; a4 = a3 + 1;

    f1 = buf[a1]; f2 = buf[a2]; f3 = buf[a3]; f4 = buf[a4];

    r1 = .5*(f3 + f1); i2 = .5*(f3 - f1);
    r2 = .5*(f2 + f4); i1 = .5*(f2 - f4);

    rt = r2 * rw1 - i2 * iw1; it = r2 * iw1 + i2 * rw1;

    buf[a1] = r1 + rt; buf[a3] = r1 - rt;
    buf[a2] = it + i1; buf[a4] = it - i1;
  }

  {
    double x0 = buf[0];
    double x1 = buf[1];
    buf[0] = x0 + x1;
    buf[1] = x0 - x1;
  }    
}  

/* half-buffer complex to real wrapper, inverse of r2cD */
void DustFFT_c2rD(double * buf, unsigned n)
{
  unsigned i, n2 = n >> 1, n4 = n >> 2;
  
  double rww, iww, rw1, iw1;
  {
      double tmp = sin(_constant_pi/n);
      rww = -2*tmp*tmp; iww = sin((2*_constant_pi)/n);
      rw1 = 1; iw1 = 0;
  }
  
  for(i = 1; i < n4; ++i)
  {
    double r1, i1, r2, i2, rt, it;
    double f1, f2, f3, f4;
    unsigned a1, a2, a3, a4;
    
    {double tmp = (rw1 * rww - iw1 * iww);
    iw1 += (rw1 * iww) + (rww * iw1); rw1 += tmp;
    }

    a1 = 2*i; a2 = a1 + 1; a3 = n - a1; a4 = a3 + 1;

    f1 = buf[a1]; f2 = buf[a2]; f3 = buf[a3]; f4 = buf[a4];

    r1 = (f1 + f3); i2 = (f1 - f3);
    r2 = -(f2 + f4); i1 = (f2 - f4);

    rt = r2 * rw1 - i2 * iw1; it = r2 * iw1 + i2 * rw1;

    buf[a1] = r1 + rt; buf[a3] = r1 - rt;
    buf[a2] = it + i1; buf[a4] = it - i1;
  }

  {
    double x0 = buf[0];
    double x1 = buf[1];
    buf[0] = x0 + x1;
    buf[1] = x0 - x1;
  }
  if(n2>=2) { buf[n2] *= 2; buf[n2+1] *= 2; }
  DustFFT_revD(buf, n2);
}
#endif