Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Consistently use FMA in neon butterfly3 #136

Merged
merged 1 commit into from
Feb 29, 2024
Merged

Consistently use FMA in neon butterfly3 #136

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions src/neon/neon_butterflies.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,7 @@ pub struct NeonF32Butterfly3<T> {
twiddle: float32x4_t,
twiddle1re: float32x4_t,
twiddle1im: float32x4_t,
twiddle2im: float32x4_t,
}

boilerplate_fft_neon_f32_butterfly!(NeonF32Butterfly3, 3, |this: &NeonF32Butterfly3<_>| this
Expand All @@ -442,13 +443,15 @@ impl<T: FftNum> NeonF32Butterfly3<T> {
let twiddle = unsafe { vld1q_f32([tw1.re, tw1.re, -tw1.im, -tw1.im].as_ptr()) };
let twiddle1re = unsafe { vmovq_n_f32(tw1.re) };
let twiddle1im = unsafe { vmovq_n_f32(tw1.im) };
let twiddle2im = unsafe { vmovq_n_f32(-tw1.im) };
Self {
direction,
_phantom: std::marker::PhantomData,
rotate,
twiddle,
twiddle1re,
twiddle1im,
twiddle2im,
}
}
#[inline(always)]
Expand Down Expand Up @@ -498,8 +501,7 @@ impl<T: FftNum> NeonF32Butterfly3<T> {
// This is a Neon translation of the scalar 3-point butterfly
let rev12 = reverse_complex_and_negate_hi_f32(value12);
let temp12pn = self.rotate.rotate_hi(vaddq_f32(value12, rev12));
let twiddled = vmulq_f32(temp12pn, self.twiddle);
let temp = vaddq_f32(value0x, twiddled);
let temp = vfmaq_f32(value0x, temp12pn, self.twiddle);

let out12 = solo_fft2_f32(temp);
let out0x = vaddq_f32(value0x, temp12pn);
Expand All @@ -518,17 +520,14 @@ impl<T: FftNum> NeonF32Butterfly3<T> {
// This is a Neon translation of the scalar 3-point butterfly
let x12p = vaddq_f32(value1, value2);
let x12n = vsubq_f32(value1, value2);
let sum = vaddq_f32(value0, x12p);

let temp_a = vmulq_f32(self.twiddle1re, x12p);
let temp_a = vaddq_f32(temp_a, value0);

let temp = vfmaq_f32(value0, self.twiddle1re, x12p);
let n_rot = self.rotate.rotate_both(x12n);
let temp_b = vmulq_f32(self.twiddle1im, n_rot);

let x1 = vaddq_f32(temp_a, temp_b);
let x2 = vsubq_f32(temp_a, temp_b);
[sum, x1, x2]
let x0 = vaddq_f32(value0, x12p);
let x1 = vfmaq_f32(temp, self.twiddle1im, n_rot);
let x2 = vfmaq_f32(temp, self.twiddle2im, n_rot);
[x0, x1, x2]
}
}

Expand All @@ -545,6 +544,7 @@ pub struct NeonF64Butterfly3<T> {
rotate: Rotate90F64,
twiddle1re: float64x2_t,
twiddle1im: float64x2_t,
twiddle2im: float64x2_t,
}

boilerplate_fft_neon_f64_butterfly!(NeonF64Butterfly3, 3, |this: &NeonF64Butterfly3<_>| this
Expand All @@ -559,13 +559,15 @@ impl<T: FftNum> NeonF64Butterfly3<T> {
let tw1: Complex<f64> = twiddles::compute_twiddle(1, 3, direction);
let twiddle1re = unsafe { vmovq_n_f64(tw1.re) };
let twiddle1im = unsafe { vmovq_n_f64(tw1.im) };
let twiddle2im = unsafe { vmovq_n_f64(-tw1.im) };

Self {
direction,
_phantom: std::marker::PhantomData,
rotate,
twiddle1re,
twiddle1im,
twiddle2im,
}
}

Expand Down Expand Up @@ -594,16 +596,14 @@ impl<T: FftNum> NeonF64Butterfly3<T> {
// This is a Neon translation of the scalar 3-point butterfly
let x12p = vaddq_f64(value1, value2);
let x12n = vsubq_f64(value1, value2);
let sum = vaddq_f64(value0, x12p);

let temp_a = vfmaq_f64(value0, self.twiddle1re, x12p);

let temp = vfmaq_f64(value0, self.twiddle1re, x12p);
let n_rot = self.rotate.rotate(x12n);
let temp_b = vmulq_f64(self.twiddle1im, n_rot);

let x1 = vaddq_f64(temp_a, temp_b);
let x2 = vsubq_f64(temp_a, temp_b);
[sum, x1, x2]
let x0 = vaddq_f64(value0, x12p);
let x1 = vfmaq_f64(temp, self.twiddle1im, n_rot);
let x2 = vfmaq_f64(temp, self.twiddle2im, n_rot);
[x0, x1, x2]
}
}

Expand Down
Loading