sparse_ir_core/gemm.rs
1//! Column-major GEMM with a pluggable BLAS backend.
2//!
3//! All matrices are column-major (Fortran/BLAS convention) and every call is
4//! expressed in BLAS terms: `C <- alpha * op(A) * op(B) + beta * C` with
5//! explicit leading dimensions, so strided sub-blocks are addressed without
6//! copies.
7//!
8//! # Design
9//! - **Default**: pure Rust faer backend (sequential), or system BLAS when
10//! the `system-blas` feature is enabled.
11//! - **Optional**: external LP64/ILP64 BLAS injected through function
12//! pointers (used by the C API).
13//! - **Thread-safe**: the process-wide default is protected by an `RwLock`;
14//! per-call backends are passed as [`GemmBackendHandle`].
15//!
16//! The safe entry point is [`gemm`], which validates buffer lengths and
17//! leading dimensions before dispatching; [`matmul`] multiplies two
18//! [`Matrix`](crate::Matrix) values.
19//!
20//! # Example
21//! ```
22//! use sparse_ir::Matrix;
23//! use sparse_ir::gemm::{GemmBackendHandle, matmul};
24//!
25//! // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
26//! let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
27//! let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
28//!
29//! // `None` uses the global dispatcher: the Faer backend by default (system
30//! // BLAS with the `system-blas` feature, or a BLAS injected at runtime)
31//! let c = matmul(None, &a, &b).unwrap();
32//! assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
33//!
34//! // Or pass an explicit backend handle instead of relying on global state
35//! let faer = GemmBackendHandle::default();
36//! assert_eq!(
37//! matmul(Some(&faer), &a, &b).unwrap().host_data().unwrap(),
38//! c.host_data().unwrap()
39//! );
40//! ```
41//!
42//! A custom BLAS (e.g. from the C API) is injected at runtime with
43//! [`set_blas_backend`] (LP64) or [`set_ilp64_backend`] (ILP64); see their
44//! examples. [`clear_blas_backend`] restores the Faer backend.
45
46use num_complex::Complex;
47use num_traits::{One, Zero};
48use once_cell::sync::Lazy;
49use std::sync::{Arc, RwLock};
50
51//==============================================================================
52// BLAS Function Pointer Types
53//==============================================================================
54
55/// BLAS dgemm function pointer type (LP64: 32-bit integers)
56///
57/// Signature matches Fortran BLAS dgemm:
58/// ```c
59/// void dgemm_(char *transa, char *transb, int *m, int *n, int *k,
60/// double *alpha, double *a, int *lda, double *b, int *ldb,
61/// double *beta, double *c, int *ldc);
62/// ```
63/// Note: All parameters are passed by reference (pointers).
64/// Transpose options: 'N' (no transpose), 'T' (transpose), 'C' (conjugate transpose).
65pub type DgemmFnPtr = unsafe extern "C" fn(
66 transa: *const libc::c_char,
67 transb: *const libc::c_char,
68 m: *const libc::c_int,
69 n: *const libc::c_int,
70 k: *const libc::c_int,
71 alpha: *const libc::c_double,
72 a: *const libc::c_double,
73 lda: *const libc::c_int,
74 b: *const libc::c_double,
75 ldb: *const libc::c_int,
76 beta: *const libc::c_double,
77 c: *mut libc::c_double,
78 ldc: *const libc::c_int,
79);
80
81/// BLAS zgemm function pointer type (LP64: 32-bit integers)
82///
83/// Signature matches Fortran BLAS zgemm:
84/// ```c
85/// void zgemm_(char *transa, char *transb, int *m, int *n, int *k,
86/// void *alpha, void *a, int *lda, void *b, int *ldb,
87/// void *beta, void *c, int *ldc);
88/// ```
89/// Note: All parameters are passed by reference (pointers).
90/// Complex numbers are passed as void* (typically complex<double>*).
91/// Transpose options: 'N' (no transpose), 'T' (transpose), 'C' (conjugate transpose).
92pub type ZgemmFnPtr = unsafe extern "C" fn(
93 transa: *const libc::c_char,
94 transb: *const libc::c_char,
95 m: *const libc::c_int,
96 n: *const libc::c_int,
97 k: *const libc::c_int,
98 alpha: *const num_complex::Complex<f64>,
99 a: *const num_complex::Complex<f64>,
100 lda: *const libc::c_int,
101 b: *const num_complex::Complex<f64>,
102 ldb: *const libc::c_int,
103 beta: *const num_complex::Complex<f64>,
104 c: *mut num_complex::Complex<f64>,
105 ldc: *const libc::c_int,
106);
107
108// When using system BLAS via `blas-sys`, we need a small wrapper to adapt
109// `blas_sys::zgemm_` (which uses `c_double_complex = [f64; 2]`) to the
110// `ZgemmFnPtr` signature that takes `num_complex::Complex<f64>`.
111#[cfg(feature = "system-blas")]
112unsafe extern "C" fn zgemm_wrapper(
113 transa: *const libc::c_char,
114 transb: *const libc::c_char,
115 m: *const libc::c_int,
116 n: *const libc::c_int,
117 k: *const libc::c_int,
118 alpha: *const num_complex::Complex<f64>,
119 a: *const num_complex::Complex<f64>,
120 lda: *const libc::c_int,
121 b: *const num_complex::Complex<f64>,
122 ldb: *const libc::c_int,
123 beta: *const num_complex::Complex<f64>,
124 c: *mut num_complex::Complex<f64>,
125 ldc: *const libc::c_int,
126) {
127 // Safety: `blas_sys::c_double_complex` is defined as `[f64; 2]` and is
128 // layout-compatible with `num_complex::Complex<f64>` in memory, so we can
129 // cast between the two pointer types here.
130 unsafe {
131 blas_sys::zgemm_(
132 transa,
133 transb,
134 m,
135 n,
136 k,
137 alpha as *const _ as *const blas_sys::c_double_complex,
138 a as *const _ as *const blas_sys::c_double_complex,
139 lda,
140 b as *const _ as *const blas_sys::c_double_complex,
141 ldb,
142 beta as *const _ as *const blas_sys::c_double_complex,
143 c as *mut _ as *mut blas_sys::c_double_complex,
144 ldc,
145 );
146 }
147}
148
149/// BLAS dgemm function pointer type (ILP64: 64-bit integers)
150///
151/// Signature matches Fortran BLAS dgemm (ILP64):
152/// ```c
153/// void dgemm_(char *transa, char *transb, long long *m, long long *n, long long *k,
154/// double *alpha, double *a, long long *lda, double *b, long long *ldb,
155/// double *beta, double *c, long long *ldc);
156/// ```
157pub type Dgemm64FnPtr = unsafe extern "C" fn(
158 transa: *const libc::c_char,
159 transb: *const libc::c_char,
160 m: *const i64,
161 n: *const i64,
162 k: *const i64,
163 alpha: *const libc::c_double,
164 a: *const libc::c_double,
165 lda: *const i64,
166 b: *const libc::c_double,
167 ldb: *const i64,
168 beta: *const libc::c_double,
169 c: *mut libc::c_double,
170 ldc: *const i64,
171);
172
173/// BLAS zgemm function pointer type (ILP64: 64-bit integers)
174///
175/// Signature matches Fortran BLAS zgemm (ILP64):
176/// ```c
177/// void zgemm_(char *transa, char *transb, long long *m, long long *n, long long *k,
178/// void *alpha, void *a, long long *lda, void *b, long long *ldb,
179/// void *beta, void *c, long long *ldc);
180/// ```
181pub type Zgemm64FnPtr = unsafe extern "C" fn(
182 transa: *const libc::c_char,
183 transb: *const libc::c_char,
184 m: *const i64,
185 n: *const i64,
186 k: *const i64,
187 alpha: *const num_complex::Complex<f64>,
188 a: *const num_complex::Complex<f64>,
189 lda: *const i64,
190 b: *const num_complex::Complex<f64>,
191 ldb: *const i64,
192 beta: *const num_complex::Complex<f64>,
193 c: *mut num_complex::Complex<f64>,
194 ldc: *const i64,
195);
196
197//==============================================================================
198// Operation descriptors and errors
199//==============================================================================
200
201/// Operation applied to a GEMM operand.
202#[derive(Clone, Copy, Debug, PartialEq, Eq)]
203pub enum Transpose {
204 /// `op(X) = X`
205 N,
206 /// `op(X) = X^T`
207 T,
208 /// `op(X) = X^H`
209 C,
210}
211
212impl Transpose {
213 fn as_blas_char(self) -> libc::c_char {
214 (match self {
215 Transpose::N => b'N',
216 Transpose::T => b'T',
217 Transpose::C => b'C',
218 }) as libc::c_char
219 }
220}
221
222/// Errors reported by GEMM dispatch.
223#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
224pub enum GemmError {
225 /// A dimension or leading dimension does not fit the backend integer type.
226 #[error("GEMM argument {name}={value} exceeds the {abi} integer range")]
227 DimensionOverflow {
228 name: &'static str,
229 value: usize,
230 abi: &'static str,
231 },
232 /// A buffer is too short or a leading dimension is too small.
233 #[error("invalid GEMM argument: {0}")]
234 InvalidArgument(String),
235}
236
237//==============================================================================
238// GemmBackend Trait
239//==============================================================================
240
241/// GEMM backend trait for runtime dispatch.
242///
243/// Implementations compute `C <- alpha * op(A) * op(B) + beta * C` on
244/// column-major operands, with BLAS semantics: `op(A)` is `m x k`, `op(B)`
245/// is `k x n`, `C` is `m x n`, and `C` is not read when `beta == 0`.
246pub trait GemmBackend: Send + Sync {
247 /// Real double-precision GEMM.
248 ///
249 /// # Safety
250 /// The pointers must address column-major matrices of the stated shapes
251 /// and leading dimensions (`lda >= max(1, rows(A))`, and likewise for `B`
252 /// and `C`); `c` must not alias `a` or `b`.
253 #[allow(clippy::too_many_arguments)]
254 unsafe fn dgemm(
255 &self,
256 transa: Transpose,
257 transb: Transpose,
258 m: usize,
259 n: usize,
260 k: usize,
261 alpha: f64,
262 a: *const f64,
263 lda: usize,
264 b: *const f64,
265 ldb: usize,
266 beta: f64,
267 c: *mut f64,
268 ldc: usize,
269 ) -> Result<(), GemmError>;
270
271 /// Complex double-precision GEMM.
272 ///
273 /// # Safety
274 /// Same contract as [`GemmBackend::dgemm`].
275 #[allow(clippy::too_many_arguments)]
276 unsafe fn zgemm(
277 &self,
278 transa: Transpose,
279 transb: Transpose,
280 m: usize,
281 n: usize,
282 k: usize,
283 alpha: Complex<f64>,
284 a: *const Complex<f64>,
285 lda: usize,
286 b: *const Complex<f64>,
287 ldb: usize,
288 beta: Complex<f64>,
289 c: *mut Complex<f64>,
290 ldc: usize,
291 ) -> Result<(), GemmError>;
292
293 /// Returns true if this backend uses 64-bit integers (ILP64).
294 fn is_ilp64(&self) -> bool {
295 false
296 }
297
298 /// Backend name for diagnostics.
299 fn name(&self) -> &'static str;
300}
301
302//==============================================================================
303// Faer Backend (Default, Pure Rust, Zero-Copy)
304//==============================================================================
305
306/// Pure Rust faer backend (sequential).
307struct FaerBackend;
308
309/// Shared faer implementation for `f64` and `Complex<f64>`.
310///
311/// # Safety
312/// Same contract as [`GemmBackend::dgemm`].
313#[allow(clippy::too_many_arguments)]
314unsafe fn faer_gemm<T>(
315 transa: Transpose,
316 transb: Transpose,
317 m: usize,
318 n: usize,
319 k: usize,
320 alpha: T,
321 a: *const T,
322 lda: usize,
323 b: *const T,
324 ldb: usize,
325 beta: T,
326 c: *mut T,
327 ldc: usize,
328) where
329 T: faer_traits::ComplexField + Copy + PartialEq + Zero + One + std::ops::MulAssign,
330{
331 use faer::linalg::matmul::matmul_with_conj;
332 use faer::mat::{MatMut, MatRef};
333 use faer::{Accum, Conj, Par};
334
335 if m == 0 || n == 0 {
336 return;
337 }
338 // SAFETY: the caller guarantees `c` addresses an m x n column-major
339 // matrix with leading dimension ldc >= m; `c` is non-null because m, n > 0.
340 let mut dst = unsafe { MatMut::from_raw_parts_mut(c, m, n, 1isize, ldc as isize) };
341
342 // faer accumulates with beta in {0, 1}; other values scale C first.
343 let accum = if beta == T::zero() {
344 Accum::Replace
345 } else {
346 if beta != T::one() {
347 for col in dst.as_mut().col_iter_mut() {
348 for x in col.iter_mut() {
349 *x *= beta;
350 }
351 }
352 }
353 Accum::Add
354 };
355 if k == 0 {
356 if accum == Accum::Replace {
357 dst.fill(T::zero());
358 }
359 return;
360 }
361
362 let operand = |ptr: *const T, op: Transpose, rows: usize, cols: usize, ld: usize| {
363 // SAFETY: the caller guarantees `ptr` addresses the stored (untransposed)
364 // `rows x cols` operand with leading dimension ld; rows, cols > 0.
365 let stored = match op {
366 Transpose::N => unsafe { MatRef::from_raw_parts(ptr, rows, cols, 1isize, ld as isize) },
367 Transpose::T | Transpose::C => {
368 unsafe { MatRef::from_raw_parts(ptr, cols, rows, 1isize, ld as isize) }.transpose()
369 }
370 };
371 let conj = if op == Transpose::C {
372 Conj::Yes
373 } else {
374 Conj::No
375 };
376 (stored, conj)
377 };
378 let (lhs, conj_lhs) = operand(a, transa, m, k, lda);
379 let (rhs, conj_rhs) = operand(b, transb, k, n, ldb);
380 matmul_with_conj(dst, accum, lhs, conj_lhs, rhs, conj_rhs, alpha, Par::Seq);
381}
382
383impl GemmBackend for FaerBackend {
384 unsafe fn dgemm(
385 &self,
386 transa: Transpose,
387 transb: Transpose,
388 m: usize,
389 n: usize,
390 k: usize,
391 alpha: f64,
392 a: *const f64,
393 lda: usize,
394 b: *const f64,
395 ldb: usize,
396 beta: f64,
397 c: *mut f64,
398 ldc: usize,
399 ) -> Result<(), GemmError> {
400 // SAFETY: forwarded caller contract.
401 unsafe { faer_gemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) };
402 Ok(())
403 }
404
405 unsafe fn zgemm(
406 &self,
407 transa: Transpose,
408 transb: Transpose,
409 m: usize,
410 n: usize,
411 k: usize,
412 alpha: Complex<f64>,
413 a: *const Complex<f64>,
414 lda: usize,
415 b: *const Complex<f64>,
416 ldb: usize,
417 beta: Complex<f64>,
418 c: *mut Complex<f64>,
419 ldc: usize,
420 ) -> Result<(), GemmError> {
421 // SAFETY: forwarded caller contract.
422 unsafe { faer_gemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) };
423 Ok(())
424 }
425
426 fn name(&self) -> &'static str {
427 "Faer (Pure Rust)"
428 }
429}
430
431//==============================================================================
432// External BLAS Backends (LP64 and ILP64)
433//==============================================================================
434
435/// Checked conversion of the six BLAS integer arguments.
436fn blas_ints<I: TryFrom<usize>>(
437 abi: &'static str,
438 m: usize,
439 n: usize,
440 k: usize,
441 lda: usize,
442 ldb: usize,
443 ldc: usize,
444) -> Result<[I; 6], GemmError> {
445 let conv = |name: &'static str, value: usize| {
446 I::try_from(value).map_err(|_| GemmError::DimensionOverflow { name, value, abi })
447 };
448 Ok([
449 conv("m", m)?,
450 conv("n", n)?,
451 conv("k", k)?,
452 conv("lda", lda)?,
453 conv("ldb", ldb)?,
454 conv("ldc", ldc)?,
455 ])
456}
457
458/// External BLAS backend (LP64: 32-bit integers).
459pub struct ExternalBlasBackend {
460 dgemm: DgemmFnPtr,
461 zgemm: ZgemmFnPtr,
462}
463
464impl ExternalBlasBackend {
465 /// Wrap LP64 Fortran `dgemm_`/`zgemm_` function pointers.
466 pub fn new(dgemm: DgemmFnPtr, zgemm: ZgemmFnPtr) -> Self {
467 Self { dgemm, zgemm }
468 }
469}
470
471/// External BLAS backend (ILP64: 64-bit integers).
472pub struct ExternalBlas64Backend {
473 dgemm64: Dgemm64FnPtr,
474 zgemm64: Zgemm64FnPtr,
475}
476
477impl ExternalBlas64Backend {
478 /// Wrap ILP64 Fortran `dgemm_`/`zgemm_` function pointers.
479 pub fn new(dgemm64: Dgemm64FnPtr, zgemm64: Zgemm64FnPtr) -> Self {
480 Self { dgemm64, zgemm64 }
481 }
482}
483
484macro_rules! impl_external_backend {
485 ($ty:ty, $int:ty, $abi:literal, $dfield:ident, $zfield:ident, $ilp64:expr, $name:literal) => {
486 impl GemmBackend for $ty {
487 unsafe fn dgemm(
488 &self,
489 transa: Transpose,
490 transb: Transpose,
491 m: usize,
492 n: usize,
493 k: usize,
494 alpha: f64,
495 a: *const f64,
496 lda: usize,
497 b: *const f64,
498 ldb: usize,
499 beta: f64,
500 c: *mut f64,
501 ldc: usize,
502 ) -> Result<(), GemmError> {
503 if m == 0 || n == 0 {
504 return Ok(());
505 }
506 let [m, n, k, lda, ldb, ldc] = blas_ints::<$int>($abi, m, n, k, lda, ldb, ldc)?;
507 let (ta, tb) = (transa.as_blas_char(), transb.as_blas_char());
508 // SAFETY: arguments follow the Fortran BLAS reference-passing
509 // convention; the caller guarantees the operand contract.
510 unsafe {
511 (self.$dfield)(
512 &ta, &tb, &m, &n, &k, &alpha, a, &lda, b, &ldb, &beta, c, &ldc,
513 )
514 };
515 Ok(())
516 }
517
518 unsafe fn zgemm(
519 &self,
520 transa: Transpose,
521 transb: Transpose,
522 m: usize,
523 n: usize,
524 k: usize,
525 alpha: Complex<f64>,
526 a: *const Complex<f64>,
527 lda: usize,
528 b: *const Complex<f64>,
529 ldb: usize,
530 beta: Complex<f64>,
531 c: *mut Complex<f64>,
532 ldc: usize,
533 ) -> Result<(), GemmError> {
534 if m == 0 || n == 0 {
535 return Ok(());
536 }
537 let [m, n, k, lda, ldb, ldc] = blas_ints::<$int>($abi, m, n, k, lda, ldb, ldc)?;
538 let (ta, tb) = (transa.as_blas_char(), transb.as_blas_char());
539 // SAFETY: as for dgemm; Complex<f64> is layout-compatible with
540 // Fortran COMPLEX*16.
541 unsafe {
542 (self.$zfield)(
543 &ta, &tb, &m, &n, &k, &alpha, a, &lda, b, &ldb, &beta, c, &ldc,
544 )
545 };
546 Ok(())
547 }
548
549 fn is_ilp64(&self) -> bool {
550 $ilp64
551 }
552
553 fn name(&self) -> &'static str {
554 $name
555 }
556 }
557 };
558}
559
560impl_external_backend!(
561 ExternalBlasBackend,
562 libc::c_int,
563 "LP64",
564 dgemm,
565 zgemm,
566 false,
567 "External BLAS (LP64)"
568);
569impl_external_backend!(
570 ExternalBlas64Backend,
571 i64,
572 "ILP64",
573 dgemm64,
574 zgemm64,
575 true,
576 "External BLAS (ILP64)"
577);
578
579//==============================================================================
580// Backend Handle
581//==============================================================================
582
583/// Shared, cloneable GEMM backend selection.
584///
585/// # Example
586/// ```
587/// use sparse_ir::Matrix;
588/// use sparse_ir::gemm::{GemmBackendHandle, matmul};
589///
590/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
591/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
592/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
593/// let backend = GemmBackendHandle::default();
594/// let result = matmul(Some(&backend), &a, &b).unwrap();
595/// assert_eq!(result.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
596/// ```
597#[derive(Clone)]
598pub struct GemmBackendHandle {
599 inner: Arc<dyn GemmBackend>,
600}
601
602impl GemmBackendHandle {
603 /// Wrap a backend.
604 pub fn new(backend: Box<dyn GemmBackend>) -> Self {
605 Self {
606 inner: Arc::from(backend),
607 }
608 }
609
610 /// Pure Rust faer backend.
611 #[allow(clippy::should_implement_trait)]
612 pub fn default() -> Self {
613 Self {
614 inner: Arc::new(FaerBackend),
615 }
616 }
617
618 pub(crate) fn as_ref(&self) -> &dyn GemmBackend {
619 self.inner.as_ref()
620 }
621}
622
623//==============================================================================
624// Global Dispatcher
625//==============================================================================
626
627static BLAS_DISPATCHER: Lazy<RwLock<Box<dyn GemmBackend>>> = Lazy::new(|| {
628 #[cfg(feature = "system-blas")]
629 {
630 // Use system BLAS (LP64) by default via `blas-sys`.
631 let backend =
632 ExternalBlasBackend::new(blas_sys::dgemm_ as DgemmFnPtr, zgemm_wrapper as ZgemmFnPtr);
633 RwLock::new(Box::new(backend) as Box<dyn GemmBackend>)
634 }
635 #[cfg(not(feature = "system-blas"))]
636 {
637 RwLock::new(Box::new(FaerBackend) as Box<dyn GemmBackend>)
638 }
639});
640
641/// Set the process-wide default to an LP64 BLAS.
642///
643/// # Safety
644/// The function pointers must be valid Fortran-convention LP64 `dgemm_` and
645/// `zgemm_` implementations that stay valid for the rest of the process.
646///
647/// # Example
648/// ```
649/// use num_complex::Complex;
650/// use sparse_ir::Matrix;
651/// use sparse_ir::gemm::{clear_blas_backend, get_backend_info, matmul, set_blas_backend};
652/// # use std::ffi::{c_char, c_int};
653/// # use std::ops::{Add, Mul};
654/// #
655/// # // Naive column-major C = alpha * A * B + beta * C for transa = transb = 'N'
656/// # // (what sparse-ir passes), standing in for a BLAS library.
657/// # unsafe fn gemm_nn<T>(mnk: [usize; 3], alpha: T, a: *const T, lda: usize,
658/// # b: *const T, ldb: usize, beta: T, c: *mut T, ldc: usize)
659/// # where
660/// # T: Copy + Default + PartialEq + Add<Output = T> + Mul<Output = T>,
661/// # {
662/// # let [m, n, k] = mnk;
663/// # for j in 0..n {
664/// # for i in 0..m {
665/// # let mut acc = T::default();
666/// # for p in 0..k {
667/// # acc = acc + unsafe { *a.add(i + p * lda) * *b.add(p + j * ldb) };
668/// # }
669/// # let cij = unsafe { &mut *c.add(i + j * ldc) };
670/// # *cij = if beta == T::default() { alpha * acc } else { alpha * acc + beta * *cij };
671/// # }
672/// # }
673/// # }
674/// # unsafe extern "C" fn my_dgemm(
675/// # _transa: *const c_char, _transb: *const c_char,
676/// # m: *const c_int, n: *const c_int, k: *const c_int,
677/// # alpha: *const f64, a: *const f64, lda: *const c_int,
678/// # b: *const f64, ldb: *const c_int,
679/// # beta: *const f64, c: *mut f64, ldc: *const c_int,
680/// # ) {
681/// # unsafe {
682/// # let mnk = [*m as usize, *n as usize, *k as usize];
683/// # gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
684/// # }
685/// # }
686/// # unsafe extern "C" fn my_zgemm(
687/// # _transa: *const c_char, _transb: *const c_char,
688/// # m: *const c_int, n: *const c_int, k: *const c_int,
689/// # alpha: *const Complex<f64>, a: *const Complex<f64>, lda: *const c_int,
690/// # b: *const Complex<f64>, ldb: *const c_int,
691/// # beta: *const Complex<f64>, c: *mut Complex<f64>, ldc: *const c_int,
692/// # ) {
693/// # unsafe {
694/// # let mnk = [*m as usize, *n as usize, *k as usize];
695/// # gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
696/// # }
697/// # }
698///
699/// // `my_dgemm` and `my_zgemm` are the LP64 Fortran BLAS `dgemm_` and `zgemm_`
700/// // to use, e.g. from OpenBLAS. (This example defines naive stand-ins in
701/// // hidden lines so that it runs without a BLAS library.)
702/// unsafe {
703/// set_blas_backend(my_dgemm, my_zgemm);
704/// }
705/// assert_eq!(get_backend_info(), ("External BLAS (LP64)", true, false));
706///
707/// // Calls without an explicit backend handle now go through the injected BLAS
708/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
709/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
710/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
711/// let c = matmul(None, &a, &b).unwrap();
712/// assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
713///
714/// // Complex matrices use `zgemm`: (i A) B = i (A B)
715/// let to_complex = |m: &Matrix<f64>, imaginary: bool| {
716/// let data = m.host_data().unwrap().iter().map(|&x| {
717/// if imaginary { Complex::new(0.0, x) } else { Complex::new(x, 0.0) }
718/// });
719/// Matrix::from_vec_col_major([2, 2], data.collect()).unwrap()
720/// };
721/// let ic = matmul(None, &to_complex(&a, true), &to_complex(&b, false)).unwrap();
722/// assert_eq!(ic.host_data().unwrap(), to_complex(&c, true).host_data().unwrap());
723///
724/// clear_blas_backend(); // back to the pure-Rust Faer backend
725/// assert!(!get_backend_info().1);
726/// ```
727pub unsafe fn set_blas_backend(dgemm: DgemmFnPtr, zgemm: ZgemmFnPtr) {
728 let mut dispatcher = BLAS_DISPATCHER.write().unwrap_or_else(|e| e.into_inner());
729 *dispatcher = Box::new(ExternalBlasBackend::new(dgemm, zgemm));
730}
731
732/// Set the process-wide default to an ILP64 BLAS.
733///
734/// # Safety
735/// As for [`set_blas_backend`], with 64-bit integer arguments.
736///
737/// # Example
738/// ```
739/// use sparse_ir::Matrix;
740/// use sparse_ir::gemm::{clear_blas_backend, get_backend_info, matmul, set_ilp64_backend};
741/// # use num_complex::Complex;
742/// # use std::ffi::c_char;
743/// # use std::ops::{Add, Mul};
744/// #
745/// # // Naive column-major C = alpha * A * B + beta * C for transa = transb = 'N'
746/// # // (what sparse-ir passes), standing in for a BLAS library.
747/// # unsafe fn gemm_nn<T>(mnk: [usize; 3], alpha: T, a: *const T, lda: usize,
748/// # b: *const T, ldb: usize, beta: T, c: *mut T, ldc: usize)
749/// # where
750/// # T: Copy + Default + PartialEq + Add<Output = T> + Mul<Output = T>,
751/// # {
752/// # let [m, n, k] = mnk;
753/// # for j in 0..n {
754/// # for i in 0..m {
755/// # let mut acc = T::default();
756/// # for p in 0..k {
757/// # acc = acc + unsafe { *a.add(i + p * lda) * *b.add(p + j * ldb) };
758/// # }
759/// # let cij = unsafe { &mut *c.add(i + j * ldc) };
760/// # *cij = if beta == T::default() { alpha * acc } else { alpha * acc + beta * *cij };
761/// # }
762/// # }
763/// # }
764/// # unsafe extern "C" fn my_dgemm64(
765/// # _transa: *const c_char, _transb: *const c_char,
766/// # m: *const i64, n: *const i64, k: *const i64,
767/// # alpha: *const f64, a: *const f64, lda: *const i64,
768/// # b: *const f64, ldb: *const i64,
769/// # beta: *const f64, c: *mut f64, ldc: *const i64,
770/// # ) {
771/// # unsafe {
772/// # let mnk = [*m as usize, *n as usize, *k as usize];
773/// # gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
774/// # }
775/// # }
776/// # unsafe extern "C" fn my_zgemm64(
777/// # _transa: *const c_char, _transb: *const c_char,
778/// # m: *const i64, n: *const i64, k: *const i64,
779/// # alpha: *const Complex<f64>, a: *const Complex<f64>, lda: *const i64,
780/// # b: *const Complex<f64>, ldb: *const i64,
781/// # beta: *const Complex<f64>, c: *mut Complex<f64>, ldc: *const i64,
782/// # ) {
783/// # unsafe {
784/// # let mnk = [*m as usize, *n as usize, *k as usize];
785/// # gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
786/// # }
787/// # }
788///
789/// // `my_dgemm64` and `my_zgemm64` are the ILP64 (64-bit integer) Fortran BLAS
790/// // `dgemm_` and `zgemm_` to use. (This example defines naive stand-ins in
791/// // hidden lines so that it runs without an ILP64 BLAS library.)
792/// unsafe {
793/// set_ilp64_backend(my_dgemm64, my_zgemm64);
794/// }
795/// assert_eq!(get_backend_info(), ("External BLAS (ILP64)", true, true));
796///
797/// // Calls without an explicit backend handle now go through the injected BLAS
798/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
799/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
800/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
801/// let c = matmul(None, &a, &b).unwrap();
802/// assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
803///
804/// clear_blas_backend(); // back to the pure-Rust Faer backend
805/// let (_, is_external, is_ilp64) = get_backend_info();
806/// assert!(!is_external && !is_ilp64);
807/// ```
808pub unsafe fn set_ilp64_backend(dgemm64: Dgemm64FnPtr, zgemm64: Zgemm64FnPtr) {
809 let mut dispatcher = BLAS_DISPATCHER.write().unwrap_or_else(|e| e.into_inner());
810 *dispatcher = Box::new(ExternalBlas64Backend::new(dgemm64, zgemm64));
811}
812
813/// Reset the process-wide default to the pure Rust faer backend.
814pub fn clear_blas_backend() {
815 let mut dispatcher = BLAS_DISPATCHER.write().unwrap_or_else(|e| e.into_inner());
816 *dispatcher = Box::new(FaerBackend);
817}
818
819/// Returns `(name, is_external, is_ilp64)` of the process-wide default.
820pub fn get_backend_info() -> (&'static str, bool, bool) {
821 let dispatcher = BLAS_DISPATCHER.read().unwrap_or_else(|e| e.into_inner());
822 let name = dispatcher.name();
823 (name, !name.contains("Faer"), dispatcher.is_ilp64())
824}
825
826/// Run `f` with the selected backend (explicit handle or process default).
827fn with_backend<R>(
828 backend: Option<&GemmBackendHandle>,
829 f: impl FnOnce(&dyn GemmBackend) -> R,
830) -> R {
831 match backend {
832 Some(handle) => f(handle.as_ref()),
833 None => {
834 let dispatcher = BLAS_DISPATCHER.read().unwrap_or_else(|e| e.into_inner());
835 f(dispatcher.as_ref())
836 }
837 }
838}
839
840//==============================================================================
841// Scalar dispatch and safe entry points
842//==============================================================================
843
844mod sealed {
845 pub trait Sealed {}
846 impl Sealed for f64 {}
847 impl Sealed for num_complex::Complex<f64> {}
848}
849
850/// Scalars supported by [`gemm`]: `f64` and `Complex<f64>`.
851pub trait GemmScalar:
852 sealed::Sealed
853 + tenferro_tensor::TensorScalar
854 + Copy
855 + Send
856 + Sync
857 + PartialEq
858 + Zero
859 + One
860 + std::ops::Mul<Output = Self>
861 + 'static
862{
863 /// Dispatch to the backend routine for this scalar.
864 ///
865 /// # Safety
866 /// Same contract as [`GemmBackend::dgemm`].
867 #[allow(clippy::too_many_arguments)]
868 unsafe fn gemm_raw(
869 backend: &dyn GemmBackend,
870 transa: Transpose,
871 transb: Transpose,
872 m: usize,
873 n: usize,
874 k: usize,
875 alpha: Self,
876 a: *const Self,
877 lda: usize,
878 b: *const Self,
879 ldb: usize,
880 beta: Self,
881 c: *mut Self,
882 ldc: usize,
883 ) -> Result<(), GemmError>;
884}
885
886impl GemmScalar for f64 {
887 unsafe fn gemm_raw(
888 backend: &dyn GemmBackend,
889 transa: Transpose,
890 transb: Transpose,
891 m: usize,
892 n: usize,
893 k: usize,
894 alpha: f64,
895 a: *const f64,
896 lda: usize,
897 b: *const f64,
898 ldb: usize,
899 beta: f64,
900 c: *mut f64,
901 ldc: usize,
902 ) -> Result<(), GemmError> {
903 // SAFETY: forwarded caller contract.
904 unsafe { backend.dgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) }
905 }
906}
907
908impl GemmScalar for Complex<f64> {
909 unsafe fn gemm_raw(
910 backend: &dyn GemmBackend,
911 transa: Transpose,
912 transb: Transpose,
913 m: usize,
914 n: usize,
915 k: usize,
916 alpha: Self,
917 a: *const Self,
918 lda: usize,
919 b: *const Self,
920 ldb: usize,
921 beta: Self,
922 c: *mut Self,
923 ldc: usize,
924 ) -> Result<(), GemmError> {
925 // SAFETY: forwarded caller contract.
926 unsafe { backend.zgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) }
927 }
928}
929
930/// Validate one column-major operand: `rows x cols` stored with leading
931/// dimension `ld` in a buffer of `len` elements.
932fn check_operand(
933 name: &str,
934 rows: usize,
935 cols: usize,
936 ld: usize,
937 len: usize,
938) -> Result<(), GemmError> {
939 if ld < rows.max(1) {
940 return Err(GemmError::InvalidArgument(format!(
941 "leading dimension of {name} is {ld}, expected at least {}",
942 rows.max(1)
943 )));
944 }
945 if rows == 0 || cols == 0 {
946 return Ok(());
947 }
948 let required = ld
949 .checked_mul(cols - 1)
950 .and_then(|x| x.checked_add(rows))
951 .ok_or_else(|| GemmError::InvalidArgument(format!("{name} extent overflows usize")))?;
952 if len < required {
953 return Err(GemmError::InvalidArgument(format!(
954 "{name} buffer has {len} elements, {rows}x{cols} with ld={ld} needs {required}"
955 )));
956 }
957 Ok(())
958}
959
960/// Safe column-major GEMM: `C <- alpha * op(A) * op(B) + beta * C`.
961///
962/// `op(A)` is `m x k`, `op(B)` is `k x n`, and `C` is `m x n`; the stored
963/// operands are addressed through their leading dimensions.
964///
965/// # Errors
966/// Returns [`GemmError::InvalidArgument`] when a buffer is too short or a
967/// leading dimension is too small, and [`GemmError::DimensionOverflow`] when
968/// an LP64 backend cannot represent a dimension.
969#[allow(clippy::too_many_arguments)]
970pub fn gemm<T: GemmScalar>(
971 backend: Option<&GemmBackendHandle>,
972 transa: Transpose,
973 transb: Transpose,
974 m: usize,
975 n: usize,
976 k: usize,
977 alpha: T,
978 a: &[T],
979 lda: usize,
980 b: &[T],
981 ldb: usize,
982 beta: T,
983 c: &mut [T],
984 ldc: usize,
985) -> Result<(), GemmError> {
986 let (ar, ac) = if transa == Transpose::N {
987 (m, k)
988 } else {
989 (k, m)
990 };
991 let (br, bc) = if transb == Transpose::N {
992 (k, n)
993 } else {
994 (n, k)
995 };
996 check_operand("A", ar, ac, lda, a.len())?;
997 check_operand("B", br, bc, ldb, b.len())?;
998 check_operand("C", m, n, ldc, c.len())?;
999 // SAFETY: extents validated above; `c` is a unique borrow so it cannot
1000 // alias `a` or `b`.
1001 with_backend(backend, |be| unsafe {
1002 T::gemm_raw(
1003 be,
1004 transa,
1005 transb,
1006 m,
1007 n,
1008 k,
1009 alpha,
1010 a.as_ptr(),
1011 lda,
1012 b.as_ptr(),
1013 ldb,
1014 beta,
1015 c.as_mut_ptr(),
1016 ldc,
1017 )
1018 })
1019}
1020
1021/// Below this `pre` extent, a middle-axis contraction is packed into one
1022/// GEMM instead of looping over `post` slices.
1023const SLICE_LOOP_MIN_PRE: usize = 32;
1024
1025/// Matrix-vector products up to this size bypass the default backend.
1026const SMALL_MATVEC_MAX_ELEMS: usize = 128 * 128;
1027const SMALL_MATVEC_MAX_POST: usize = 4;
1028
1029/// Apply a contiguous column-major `m x n` matrix `a` along the middle axis
1030/// of a column-major `[pre, n, post]` array `x`, writing `[pre, m, post]`
1031/// into `y` (overwritten).
1032///
1033/// # Errors
1034/// Returns [`GemmError::InvalidArgument`] when a buffer length does not match
1035/// its extents, or a backend error.
1036#[allow(clippy::too_many_arguments)]
1037pub(crate) fn apply_along_axis<T: GemmScalar>(
1038 backend: Option<&GemmBackendHandle>,
1039 a: &[T],
1040 m: usize,
1041 n: usize,
1042 x: &[T],
1043 pre: usize,
1044 post: usize,
1045 y: &mut [T],
1046) -> Result<(), GemmError> {
1047 let len_ok = |len: usize, rows: usize| {
1048 pre.checked_mul(rows)
1049 .and_then(|v| v.checked_mul(post))
1050 .is_some_and(|v| v == len)
1051 };
1052 if a.len() != m * n || !len_ok(x.len(), n) || !len_ok(y.len(), m) {
1053 return Err(GemmError::InvalidArgument(format!(
1054 "apply_along_axis: A {}, X {}, Y {} elements for m={m}, n={n}, pre={pre}, post={post}",
1055 a.len(),
1056 x.len(),
1057 y.len()
1058 )));
1059 }
1060 if y.is_empty() {
1061 return Ok(());
1062 }
1063 let (one, zero) = (T::one(), T::zero());
1064 let lda = m.max(1);
1065 if pre == 1
1066 && post <= SMALL_MATVEC_MAX_POST
1067 && m * n <= SMALL_MATVEC_MAX_ELEMS
1068 && backend.is_none()
1069 {
1070 // Small matrix-vector products: the default backend's dispatch
1071 // overhead dominates, so accumulate columns of A directly.
1072 for q in 0..post {
1073 let yq = &mut y[q * m..(q + 1) * m];
1074 yq.fill(zero);
1075 for (j, col) in a.chunks_exact(m).enumerate() {
1076 let xj = x[q * n + j];
1077 for (yi, &aij) in yq.iter_mut().zip(col) {
1078 *yi = *yi + aij * xj;
1079 }
1080 }
1081 }
1082 return Ok(());
1083 }
1084 if pre == 1 {
1085 // Y(m x post) = A(m x n) X(n x post)
1086 return gemm(
1087 backend,
1088 Transpose::N,
1089 Transpose::N,
1090 m,
1091 post,
1092 n,
1093 one,
1094 a,
1095 lda,
1096 x,
1097 n.max(1),
1098 zero,
1099 y,
1100 m.max(1),
1101 );
1102 }
1103 if post == 1 || pre >= SLICE_LOOP_MIN_PRE {
1104 // Y_q(pre x m) = X_q(pre x n) A^T for each post index q.
1105 let (xs, ys) = (pre * n, pre * m);
1106 for q in 0..post {
1107 gemm(
1108 backend,
1109 Transpose::N,
1110 Transpose::T,
1111 pre,
1112 m,
1113 n,
1114 one,
1115 &x[q * xs..(q + 1) * xs],
1116 pre,
1117 a,
1118 lda,
1119 zero,
1120 &mut y[q * ys..(q + 1) * ys],
1121 pre,
1122 )?;
1123 }
1124 return Ok(());
1125 }
1126 // Small pre, many post slices: pack X into [n, pre*post], one GEMM, unpack.
1127 let cols = pre * post;
1128 let mut xp = Vec::with_capacity(n * cols);
1129 for q in 0..post {
1130 for p in 0..pre {
1131 let base = p + pre * n * q;
1132 xp.extend((0..n).map(|j| x[base + pre * j]));
1133 }
1134 }
1135 let mut yp = vec![zero; m * cols];
1136 gemm(
1137 backend,
1138 Transpose::N,
1139 Transpose::N,
1140 m,
1141 cols,
1142 n,
1143 one,
1144 a,
1145 lda,
1146 &xp,
1147 n.max(1),
1148 zero,
1149 &mut yp,
1150 m.max(1),
1151 )?;
1152 for q in 0..post {
1153 for p in 0..pre {
1154 let src = &yp[m * (p + pre * q)..m * (p + pre * q + 1)];
1155 let base = p + pre * m * q;
1156 for (i, &v) in src.iter().enumerate() {
1157 y[base + pre * i] = v;
1158 }
1159 }
1160 }
1161 Ok(())
1162}
1163
1164/// Matrix product `A * B` of two contiguous column-major matrices.
1165///
1166/// # Errors
1167/// Returns [`crate::Error::ShapeMismatch`] when the inner dimensions differ,
1168/// [`crate::Error::Tensor`] when an operand is not host-resident compact
1169/// column-major storage, or a backend error.
1170///
1171/// # Example
1172/// ```
1173/// use sparse_ir::Matrix;
1174/// use sparse_ir::gemm::{GemmBackendHandle, matmul};
1175///
1176/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
1177/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
1178/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
1179/// let backend = GemmBackendHandle::default();
1180/// let c = matmul(Some(&backend), &a, &b).unwrap();
1181/// assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
1182///
1183/// // A 2x3 times 3x1 product
1184/// let a = Matrix::<f64>::from_vec_col_major([2, 3], vec![1.0, 4.0, 2.0, 5.0, 3.0, 6.0]).unwrap();
1185/// let b = Matrix::<f64>::from_vec_col_major([3, 1], vec![7.0, 8.0, 9.0]).unwrap();
1186/// assert_eq!(matmul(None, &a, &b).unwrap().host_data().unwrap(), &[50.0, 122.0]);
1187/// ```
1188pub fn matmul<T: GemmScalar>(
1189 backend: Option<&GemmBackendHandle>,
1190 a: &crate::Matrix<T>,
1191 b: &crate::Matrix<T>,
1192) -> crate::Result<crate::Matrix<T>> {
1193 let (av, bv) = (a.host_col_major_view()?, b.host_col_major_view()?);
1194 let [m, k] = *av.shape();
1195 let [k2, n] = *bv.shape();
1196 if k != k2 {
1197 return Err(crate::Error::ShapeMismatch {
1198 which: crate::ArrayRole::Input,
1199 expected: vec![k, n],
1200 actual: vec![k2, n],
1201 });
1202 }
1203 let mut c = vec![T::zero(); m * n];
1204 gemm(
1205 backend,
1206 Transpose::N,
1207 Transpose::N,
1208 m,
1209 n,
1210 k,
1211 T::one(),
1212 av.as_slice(),
1213 m.max(1),
1214 bv.as_slice(),
1215 k.max(1),
1216 T::zero(),
1217 &mut c,
1218 m.max(1),
1219 )?;
1220 Ok(crate::Matrix::from_vec_col_major([m, n], c)?)
1221}
1222
1223#[cfg(test)]
1224#[path = "gemm_tests.rs"]
1225mod tests;