Skip to main content

sparse_ir_core/
gemm.rs

1//! Column-major GEMM with a pluggable BLAS backend.
2//!
3//! All matrices are column-major (Fortran/BLAS convention) and every call is
4//! expressed in BLAS terms: `C <- alpha * op(A) * op(B) + beta * C` with
5//! explicit leading dimensions, so strided sub-blocks are addressed without
6//! copies.
7//!
8//! # Design
9//! - **Default**: pure Rust faer backend (sequential), or system BLAS when
10//!   the `system-blas` feature is enabled.
11//! - **Optional**: external LP64/ILP64 BLAS injected through function
12//!   pointers (used by the C API).
13//! - **Thread-safe**: the process-wide default is protected by an `RwLock`;
14//!   per-call backends are passed as [`GemmBackendHandle`].
15//!
16//! The safe entry point is [`gemm`], which validates buffer lengths and
17//! leading dimensions before dispatching; [`matmul`] multiplies two
18//! [`Matrix`](crate::Matrix) values.
19//!
20//! # Example
21//! ```
22//! use sparse_ir::Matrix;
23//! use sparse_ir::gemm::{GemmBackendHandle, matmul};
24//!
25//! // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
26//! let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
27//! let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
28//!
29//! // `None` uses the global dispatcher: the Faer backend by default (system
30//! // BLAS with the `system-blas` feature, or a BLAS injected at runtime)
31//! let c = matmul(None, &a, &b).unwrap();
32//! assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
33//!
34//! // Or pass an explicit backend handle instead of relying on global state
35//! let faer = GemmBackendHandle::default();
36//! assert_eq!(
37//!     matmul(Some(&faer), &a, &b).unwrap().host_data().unwrap(),
38//!     c.host_data().unwrap()
39//! );
40//! ```
41//!
42//! A custom BLAS (e.g. from the C API) is injected at runtime with
43//! [`set_blas_backend`] (LP64) or [`set_ilp64_backend`] (ILP64); see their
44//! examples. [`clear_blas_backend`] restores the Faer backend.
45
46use num_complex::Complex;
47use num_traits::{One, Zero};
48use once_cell::sync::Lazy;
49use std::sync::{Arc, RwLock};
50
51//==============================================================================
52// BLAS Function Pointer Types
53//==============================================================================
54
55/// BLAS dgemm function pointer type (LP64: 32-bit integers)
56///
57/// Signature matches Fortran BLAS dgemm:
58/// ```c
59/// void dgemm_(char *transa, char *transb, int *m, int *n, int *k,
60///             double *alpha, double *a, int *lda, double *b, int *ldb,
61///             double *beta, double *c, int *ldc);
62/// ```
63/// Note: All parameters are passed by reference (pointers).
64/// Transpose options: 'N' (no transpose), 'T' (transpose), 'C' (conjugate transpose).
65pub type DgemmFnPtr = unsafe extern "C" fn(
66    transa: *const libc::c_char,
67    transb: *const libc::c_char,
68    m: *const libc::c_int,
69    n: *const libc::c_int,
70    k: *const libc::c_int,
71    alpha: *const libc::c_double,
72    a: *const libc::c_double,
73    lda: *const libc::c_int,
74    b: *const libc::c_double,
75    ldb: *const libc::c_int,
76    beta: *const libc::c_double,
77    c: *mut libc::c_double,
78    ldc: *const libc::c_int,
79);
80
81/// BLAS zgemm function pointer type (LP64: 32-bit integers)
82///
83/// Signature matches Fortran BLAS zgemm:
84/// ```c
85/// void zgemm_(char *transa, char *transb, int *m, int *n, int *k,
86///             void *alpha, void *a, int *lda, void *b, int *ldb,
87///             void *beta, void *c, int *ldc);
88/// ```
89/// Note: All parameters are passed by reference (pointers).
90/// Complex numbers are passed as void* (typically complex<double>*).
91/// Transpose options: 'N' (no transpose), 'T' (transpose), 'C' (conjugate transpose).
92pub type ZgemmFnPtr = unsafe extern "C" fn(
93    transa: *const libc::c_char,
94    transb: *const libc::c_char,
95    m: *const libc::c_int,
96    n: *const libc::c_int,
97    k: *const libc::c_int,
98    alpha: *const num_complex::Complex<f64>,
99    a: *const num_complex::Complex<f64>,
100    lda: *const libc::c_int,
101    b: *const num_complex::Complex<f64>,
102    ldb: *const libc::c_int,
103    beta: *const num_complex::Complex<f64>,
104    c: *mut num_complex::Complex<f64>,
105    ldc: *const libc::c_int,
106);
107
108// When using system BLAS via `blas-sys`, we need a small wrapper to adapt
109// `blas_sys::zgemm_` (which uses `c_double_complex = [f64; 2]`) to the
110// `ZgemmFnPtr` signature that takes `num_complex::Complex<f64>`.
111#[cfg(feature = "system-blas")]
112unsafe extern "C" fn zgemm_wrapper(
113    transa: *const libc::c_char,
114    transb: *const libc::c_char,
115    m: *const libc::c_int,
116    n: *const libc::c_int,
117    k: *const libc::c_int,
118    alpha: *const num_complex::Complex<f64>,
119    a: *const num_complex::Complex<f64>,
120    lda: *const libc::c_int,
121    b: *const num_complex::Complex<f64>,
122    ldb: *const libc::c_int,
123    beta: *const num_complex::Complex<f64>,
124    c: *mut num_complex::Complex<f64>,
125    ldc: *const libc::c_int,
126) {
127    // Safety: `blas_sys::c_double_complex` is defined as `[f64; 2]` and is
128    // layout-compatible with `num_complex::Complex<f64>` in memory, so we can
129    // cast between the two pointer types here.
130    unsafe {
131        blas_sys::zgemm_(
132            transa,
133            transb,
134            m,
135            n,
136            k,
137            alpha as *const _ as *const blas_sys::c_double_complex,
138            a as *const _ as *const blas_sys::c_double_complex,
139            lda,
140            b as *const _ as *const blas_sys::c_double_complex,
141            ldb,
142            beta as *const _ as *const blas_sys::c_double_complex,
143            c as *mut _ as *mut blas_sys::c_double_complex,
144            ldc,
145        );
146    }
147}
148
149/// BLAS dgemm function pointer type (ILP64: 64-bit integers)
150///
151/// Signature matches Fortran BLAS dgemm (ILP64):
152/// ```c
153/// void dgemm_(char *transa, char *transb, long long *m, long long *n, long long *k,
154///             double *alpha, double *a, long long *lda, double *b, long long *ldb,
155///             double *beta, double *c, long long *ldc);
156/// ```
157pub type Dgemm64FnPtr = unsafe extern "C" fn(
158    transa: *const libc::c_char,
159    transb: *const libc::c_char,
160    m: *const i64,
161    n: *const i64,
162    k: *const i64,
163    alpha: *const libc::c_double,
164    a: *const libc::c_double,
165    lda: *const i64,
166    b: *const libc::c_double,
167    ldb: *const i64,
168    beta: *const libc::c_double,
169    c: *mut libc::c_double,
170    ldc: *const i64,
171);
172
173/// BLAS zgemm function pointer type (ILP64: 64-bit integers)
174///
175/// Signature matches Fortran BLAS zgemm (ILP64):
176/// ```c
177/// void zgemm_(char *transa, char *transb, long long *m, long long *n, long long *k,
178///             void *alpha, void *a, long long *lda, void *b, long long *ldb,
179///             void *beta, void *c, long long *ldc);
180/// ```
181pub type Zgemm64FnPtr = unsafe extern "C" fn(
182    transa: *const libc::c_char,
183    transb: *const libc::c_char,
184    m: *const i64,
185    n: *const i64,
186    k: *const i64,
187    alpha: *const num_complex::Complex<f64>,
188    a: *const num_complex::Complex<f64>,
189    lda: *const i64,
190    b: *const num_complex::Complex<f64>,
191    ldb: *const i64,
192    beta: *const num_complex::Complex<f64>,
193    c: *mut num_complex::Complex<f64>,
194    ldc: *const i64,
195);
196
197//==============================================================================
198// Operation descriptors and errors
199//==============================================================================
200
201/// Operation applied to a GEMM operand.
202#[derive(Clone, Copy, Debug, PartialEq, Eq)]
203pub enum Transpose {
204    /// `op(X) = X`
205    N,
206    /// `op(X) = X^T`
207    T,
208    /// `op(X) = X^H`
209    C,
210}
211
212impl Transpose {
213    fn as_blas_char(self) -> libc::c_char {
214        (match self {
215            Transpose::N => b'N',
216            Transpose::T => b'T',
217            Transpose::C => b'C',
218        }) as libc::c_char
219    }
220}
221
222/// Errors reported by GEMM dispatch.
223#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
224pub enum GemmError {
225    /// A dimension or leading dimension does not fit the backend integer type.
226    #[error("GEMM argument {name}={value} exceeds the {abi} integer range")]
227    DimensionOverflow {
228        name: &'static str,
229        value: usize,
230        abi: &'static str,
231    },
232    /// A buffer is too short or a leading dimension is too small.
233    #[error("invalid GEMM argument: {0}")]
234    InvalidArgument(String),
235}
236
237//==============================================================================
238// GemmBackend Trait
239//==============================================================================
240
241/// GEMM backend trait for runtime dispatch.
242///
243/// Implementations compute `C <- alpha * op(A) * op(B) + beta * C` on
244/// column-major operands, with BLAS semantics: `op(A)` is `m x k`, `op(B)`
245/// is `k x n`, `C` is `m x n`, and `C` is not read when `beta == 0`.
246pub trait GemmBackend: Send + Sync {
247    /// Real double-precision GEMM.
248    ///
249    /// # Safety
250    /// The pointers must address column-major matrices of the stated shapes
251    /// and leading dimensions (`lda >= max(1, rows(A))`, and likewise for `B`
252    /// and `C`); `c` must not alias `a` or `b`.
253    #[allow(clippy::too_many_arguments)]
254    unsafe fn dgemm(
255        &self,
256        transa: Transpose,
257        transb: Transpose,
258        m: usize,
259        n: usize,
260        k: usize,
261        alpha: f64,
262        a: *const f64,
263        lda: usize,
264        b: *const f64,
265        ldb: usize,
266        beta: f64,
267        c: *mut f64,
268        ldc: usize,
269    ) -> Result<(), GemmError>;
270
271    /// Complex double-precision GEMM.
272    ///
273    /// # Safety
274    /// Same contract as [`GemmBackend::dgemm`].
275    #[allow(clippy::too_many_arguments)]
276    unsafe fn zgemm(
277        &self,
278        transa: Transpose,
279        transb: Transpose,
280        m: usize,
281        n: usize,
282        k: usize,
283        alpha: Complex<f64>,
284        a: *const Complex<f64>,
285        lda: usize,
286        b: *const Complex<f64>,
287        ldb: usize,
288        beta: Complex<f64>,
289        c: *mut Complex<f64>,
290        ldc: usize,
291    ) -> Result<(), GemmError>;
292
293    /// Returns true if this backend uses 64-bit integers (ILP64).
294    fn is_ilp64(&self) -> bool {
295        false
296    }
297
298    /// Backend name for diagnostics.
299    fn name(&self) -> &'static str;
300}
301
302//==============================================================================
303// Faer Backend (Default, Pure Rust, Zero-Copy)
304//==============================================================================
305
306/// Pure Rust faer backend (sequential).
307struct FaerBackend;
308
309/// Shared faer implementation for `f64` and `Complex<f64>`.
310///
311/// # Safety
312/// Same contract as [`GemmBackend::dgemm`].
313#[allow(clippy::too_many_arguments)]
314unsafe fn faer_gemm<T>(
315    transa: Transpose,
316    transb: Transpose,
317    m: usize,
318    n: usize,
319    k: usize,
320    alpha: T,
321    a: *const T,
322    lda: usize,
323    b: *const T,
324    ldb: usize,
325    beta: T,
326    c: *mut T,
327    ldc: usize,
328) where
329    T: faer_traits::ComplexField + Copy + PartialEq + Zero + One + std::ops::MulAssign,
330{
331    use faer::linalg::matmul::matmul_with_conj;
332    use faer::mat::{MatMut, MatRef};
333    use faer::{Accum, Conj, Par};
334
335    if m == 0 || n == 0 {
336        return;
337    }
338    // SAFETY: the caller guarantees `c` addresses an m x n column-major
339    // matrix with leading dimension ldc >= m; `c` is non-null because m, n > 0.
340    let mut dst = unsafe { MatMut::from_raw_parts_mut(c, m, n, 1isize, ldc as isize) };
341
342    // faer accumulates with beta in {0, 1}; other values scale C first.
343    let accum = if beta == T::zero() {
344        Accum::Replace
345    } else {
346        if beta != T::one() {
347            for col in dst.as_mut().col_iter_mut() {
348                for x in col.iter_mut() {
349                    *x *= beta;
350                }
351            }
352        }
353        Accum::Add
354    };
355    if k == 0 {
356        if accum == Accum::Replace {
357            dst.fill(T::zero());
358        }
359        return;
360    }
361
362    let operand = |ptr: *const T, op: Transpose, rows: usize, cols: usize, ld: usize| {
363        // SAFETY: the caller guarantees `ptr` addresses the stored (untransposed)
364        // `rows x cols` operand with leading dimension ld; rows, cols > 0.
365        let stored = match op {
366            Transpose::N => unsafe { MatRef::from_raw_parts(ptr, rows, cols, 1isize, ld as isize) },
367            Transpose::T | Transpose::C => {
368                unsafe { MatRef::from_raw_parts(ptr, cols, rows, 1isize, ld as isize) }.transpose()
369            }
370        };
371        let conj = if op == Transpose::C {
372            Conj::Yes
373        } else {
374            Conj::No
375        };
376        (stored, conj)
377    };
378    let (lhs, conj_lhs) = operand(a, transa, m, k, lda);
379    let (rhs, conj_rhs) = operand(b, transb, k, n, ldb);
380    matmul_with_conj(dst, accum, lhs, conj_lhs, rhs, conj_rhs, alpha, Par::Seq);
381}
382
383impl GemmBackend for FaerBackend {
384    unsafe fn dgemm(
385        &self,
386        transa: Transpose,
387        transb: Transpose,
388        m: usize,
389        n: usize,
390        k: usize,
391        alpha: f64,
392        a: *const f64,
393        lda: usize,
394        b: *const f64,
395        ldb: usize,
396        beta: f64,
397        c: *mut f64,
398        ldc: usize,
399    ) -> Result<(), GemmError> {
400        // SAFETY: forwarded caller contract.
401        unsafe { faer_gemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) };
402        Ok(())
403    }
404
405    unsafe fn zgemm(
406        &self,
407        transa: Transpose,
408        transb: Transpose,
409        m: usize,
410        n: usize,
411        k: usize,
412        alpha: Complex<f64>,
413        a: *const Complex<f64>,
414        lda: usize,
415        b: *const Complex<f64>,
416        ldb: usize,
417        beta: Complex<f64>,
418        c: *mut Complex<f64>,
419        ldc: usize,
420    ) -> Result<(), GemmError> {
421        // SAFETY: forwarded caller contract.
422        unsafe { faer_gemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) };
423        Ok(())
424    }
425
426    fn name(&self) -> &'static str {
427        "Faer (Pure Rust)"
428    }
429}
430
431//==============================================================================
432// External BLAS Backends (LP64 and ILP64)
433//==============================================================================
434
435/// Checked conversion of the six BLAS integer arguments.
436fn blas_ints<I: TryFrom<usize>>(
437    abi: &'static str,
438    m: usize,
439    n: usize,
440    k: usize,
441    lda: usize,
442    ldb: usize,
443    ldc: usize,
444) -> Result<[I; 6], GemmError> {
445    let conv = |name: &'static str, value: usize| {
446        I::try_from(value).map_err(|_| GemmError::DimensionOverflow { name, value, abi })
447    };
448    Ok([
449        conv("m", m)?,
450        conv("n", n)?,
451        conv("k", k)?,
452        conv("lda", lda)?,
453        conv("ldb", ldb)?,
454        conv("ldc", ldc)?,
455    ])
456}
457
458/// External BLAS backend (LP64: 32-bit integers).
459pub struct ExternalBlasBackend {
460    dgemm: DgemmFnPtr,
461    zgemm: ZgemmFnPtr,
462}
463
464impl ExternalBlasBackend {
465    /// Wrap LP64 Fortran `dgemm_`/`zgemm_` function pointers.
466    pub fn new(dgemm: DgemmFnPtr, zgemm: ZgemmFnPtr) -> Self {
467        Self { dgemm, zgemm }
468    }
469}
470
471/// External BLAS backend (ILP64: 64-bit integers).
472pub struct ExternalBlas64Backend {
473    dgemm64: Dgemm64FnPtr,
474    zgemm64: Zgemm64FnPtr,
475}
476
477impl ExternalBlas64Backend {
478    /// Wrap ILP64 Fortran `dgemm_`/`zgemm_` function pointers.
479    pub fn new(dgemm64: Dgemm64FnPtr, zgemm64: Zgemm64FnPtr) -> Self {
480        Self { dgemm64, zgemm64 }
481    }
482}
483
484macro_rules! impl_external_backend {
485    ($ty:ty, $int:ty, $abi:literal, $dfield:ident, $zfield:ident, $ilp64:expr, $name:literal) => {
486        impl GemmBackend for $ty {
487            unsafe fn dgemm(
488                &self,
489                transa: Transpose,
490                transb: Transpose,
491                m: usize,
492                n: usize,
493                k: usize,
494                alpha: f64,
495                a: *const f64,
496                lda: usize,
497                b: *const f64,
498                ldb: usize,
499                beta: f64,
500                c: *mut f64,
501                ldc: usize,
502            ) -> Result<(), GemmError> {
503                if m == 0 || n == 0 {
504                    return Ok(());
505                }
506                let [m, n, k, lda, ldb, ldc] = blas_ints::<$int>($abi, m, n, k, lda, ldb, ldc)?;
507                let (ta, tb) = (transa.as_blas_char(), transb.as_blas_char());
508                // SAFETY: arguments follow the Fortran BLAS reference-passing
509                // convention; the caller guarantees the operand contract.
510                unsafe {
511                    (self.$dfield)(
512                        &ta, &tb, &m, &n, &k, &alpha, a, &lda, b, &ldb, &beta, c, &ldc,
513                    )
514                };
515                Ok(())
516            }
517
518            unsafe fn zgemm(
519                &self,
520                transa: Transpose,
521                transb: Transpose,
522                m: usize,
523                n: usize,
524                k: usize,
525                alpha: Complex<f64>,
526                a: *const Complex<f64>,
527                lda: usize,
528                b: *const Complex<f64>,
529                ldb: usize,
530                beta: Complex<f64>,
531                c: *mut Complex<f64>,
532                ldc: usize,
533            ) -> Result<(), GemmError> {
534                if m == 0 || n == 0 {
535                    return Ok(());
536                }
537                let [m, n, k, lda, ldb, ldc] = blas_ints::<$int>($abi, m, n, k, lda, ldb, ldc)?;
538                let (ta, tb) = (transa.as_blas_char(), transb.as_blas_char());
539                // SAFETY: as for dgemm; Complex<f64> is layout-compatible with
540                // Fortran COMPLEX*16.
541                unsafe {
542                    (self.$zfield)(
543                        &ta, &tb, &m, &n, &k, &alpha, a, &lda, b, &ldb, &beta, c, &ldc,
544                    )
545                };
546                Ok(())
547            }
548
549            fn is_ilp64(&self) -> bool {
550                $ilp64
551            }
552
553            fn name(&self) -> &'static str {
554                $name
555            }
556        }
557    };
558}
559
560impl_external_backend!(
561    ExternalBlasBackend,
562    libc::c_int,
563    "LP64",
564    dgemm,
565    zgemm,
566    false,
567    "External BLAS (LP64)"
568);
569impl_external_backend!(
570    ExternalBlas64Backend,
571    i64,
572    "ILP64",
573    dgemm64,
574    zgemm64,
575    true,
576    "External BLAS (ILP64)"
577);
578
579//==============================================================================
580// Backend Handle
581//==============================================================================
582
583/// Shared, cloneable GEMM backend selection.
584///
585/// # Example
586/// ```
587/// use sparse_ir::Matrix;
588/// use sparse_ir::gemm::{GemmBackendHandle, matmul};
589///
590/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
591/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
592/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
593/// let backend = GemmBackendHandle::default();
594/// let result = matmul(Some(&backend), &a, &b).unwrap();
595/// assert_eq!(result.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
596/// ```
597#[derive(Clone)]
598pub struct GemmBackendHandle {
599    inner: Arc<dyn GemmBackend>,
600}
601
602impl GemmBackendHandle {
603    /// Wrap a backend.
604    pub fn new(backend: Box<dyn GemmBackend>) -> Self {
605        Self {
606            inner: Arc::from(backend),
607        }
608    }
609
610    /// Pure Rust faer backend.
611    #[allow(clippy::should_implement_trait)]
612    pub fn default() -> Self {
613        Self {
614            inner: Arc::new(FaerBackend),
615        }
616    }
617
618    pub(crate) fn as_ref(&self) -> &dyn GemmBackend {
619        self.inner.as_ref()
620    }
621}
622
623//==============================================================================
624// Global Dispatcher
625//==============================================================================
626
627static BLAS_DISPATCHER: Lazy<RwLock<Box<dyn GemmBackend>>> = Lazy::new(|| {
628    #[cfg(feature = "system-blas")]
629    {
630        // Use system BLAS (LP64) by default via `blas-sys`.
631        let backend =
632            ExternalBlasBackend::new(blas_sys::dgemm_ as DgemmFnPtr, zgemm_wrapper as ZgemmFnPtr);
633        RwLock::new(Box::new(backend) as Box<dyn GemmBackend>)
634    }
635    #[cfg(not(feature = "system-blas"))]
636    {
637        RwLock::new(Box::new(FaerBackend) as Box<dyn GemmBackend>)
638    }
639});
640
641/// Set the process-wide default to an LP64 BLAS.
642///
643/// # Safety
644/// The function pointers must be valid Fortran-convention LP64 `dgemm_` and
645/// `zgemm_` implementations that stay valid for the rest of the process.
646///
647/// # Example
648/// ```
649/// use num_complex::Complex;
650/// use sparse_ir::Matrix;
651/// use sparse_ir::gemm::{clear_blas_backend, get_backend_info, matmul, set_blas_backend};
652/// # use std::ffi::{c_char, c_int};
653/// # use std::ops::{Add, Mul};
654/// #
655/// # // Naive column-major C = alpha * A * B + beta * C for transa = transb = 'N'
656/// # // (what sparse-ir passes), standing in for a BLAS library.
657/// # unsafe fn gemm_nn<T>(mnk: [usize; 3], alpha: T, a: *const T, lda: usize,
658/// #                    b: *const T, ldb: usize, beta: T, c: *mut T, ldc: usize)
659/// # where
660/// #     T: Copy + Default + PartialEq + Add<Output = T> + Mul<Output = T>,
661/// # {
662/// #     let [m, n, k] = mnk;
663/// #     for j in 0..n {
664/// #         for i in 0..m {
665/// #             let mut acc = T::default();
666/// #             for p in 0..k {
667/// #                 acc = acc + unsafe { *a.add(i + p * lda) * *b.add(p + j * ldb) };
668/// #             }
669/// #             let cij = unsafe { &mut *c.add(i + j * ldc) };
670/// #             *cij = if beta == T::default() { alpha * acc } else { alpha * acc + beta * *cij };
671/// #         }
672/// #     }
673/// # }
674/// # unsafe extern "C" fn my_dgemm(
675/// #     _transa: *const c_char, _transb: *const c_char,
676/// #     m: *const c_int, n: *const c_int, k: *const c_int,
677/// #     alpha: *const f64, a: *const f64, lda: *const c_int,
678/// #     b: *const f64, ldb: *const c_int,
679/// #     beta: *const f64, c: *mut f64, ldc: *const c_int,
680/// # ) {
681/// #     unsafe {
682/// #         let mnk = [*m as usize, *n as usize, *k as usize];
683/// #         gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
684/// #     }
685/// # }
686/// # unsafe extern "C" fn my_zgemm(
687/// #     _transa: *const c_char, _transb: *const c_char,
688/// #     m: *const c_int, n: *const c_int, k: *const c_int,
689/// #     alpha: *const Complex<f64>, a: *const Complex<f64>, lda: *const c_int,
690/// #     b: *const Complex<f64>, ldb: *const c_int,
691/// #     beta: *const Complex<f64>, c: *mut Complex<f64>, ldc: *const c_int,
692/// # ) {
693/// #     unsafe {
694/// #         let mnk = [*m as usize, *n as usize, *k as usize];
695/// #         gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
696/// #     }
697/// # }
698///
699/// // `my_dgemm` and `my_zgemm` are the LP64 Fortran BLAS `dgemm_` and `zgemm_`
700/// // to use, e.g. from OpenBLAS. (This example defines naive stand-ins in
701/// // hidden lines so that it runs without a BLAS library.)
702/// unsafe {
703///     set_blas_backend(my_dgemm, my_zgemm);
704/// }
705/// assert_eq!(get_backend_info(), ("External BLAS (LP64)", true, false));
706///
707/// // Calls without an explicit backend handle now go through the injected BLAS
708/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
709/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
710/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
711/// let c = matmul(None, &a, &b).unwrap();
712/// assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
713///
714/// // Complex matrices use `zgemm`: (i A) B = i (A B)
715/// let to_complex = |m: &Matrix<f64>, imaginary: bool| {
716///     let data = m.host_data().unwrap().iter().map(|&x| {
717///         if imaginary { Complex::new(0.0, x) } else { Complex::new(x, 0.0) }
718///     });
719///     Matrix::from_vec_col_major([2, 2], data.collect()).unwrap()
720/// };
721/// let ic = matmul(None, &to_complex(&a, true), &to_complex(&b, false)).unwrap();
722/// assert_eq!(ic.host_data().unwrap(), to_complex(&c, true).host_data().unwrap());
723///
724/// clear_blas_backend(); // back to the pure-Rust Faer backend
725/// assert!(!get_backend_info().1);
726/// ```
727pub unsafe fn set_blas_backend(dgemm: DgemmFnPtr, zgemm: ZgemmFnPtr) {
728    let mut dispatcher = BLAS_DISPATCHER.write().unwrap_or_else(|e| e.into_inner());
729    *dispatcher = Box::new(ExternalBlasBackend::new(dgemm, zgemm));
730}
731
732/// Set the process-wide default to an ILP64 BLAS.
733///
734/// # Safety
735/// As for [`set_blas_backend`], with 64-bit integer arguments.
736///
737/// # Example
738/// ```
739/// use sparse_ir::Matrix;
740/// use sparse_ir::gemm::{clear_blas_backend, get_backend_info, matmul, set_ilp64_backend};
741/// # use num_complex::Complex;
742/// # use std::ffi::c_char;
743/// # use std::ops::{Add, Mul};
744/// #
745/// # // Naive column-major C = alpha * A * B + beta * C for transa = transb = 'N'
746/// # // (what sparse-ir passes), standing in for a BLAS library.
747/// # unsafe fn gemm_nn<T>(mnk: [usize; 3], alpha: T, a: *const T, lda: usize,
748/// #                    b: *const T, ldb: usize, beta: T, c: *mut T, ldc: usize)
749/// # where
750/// #     T: Copy + Default + PartialEq + Add<Output = T> + Mul<Output = T>,
751/// # {
752/// #     let [m, n, k] = mnk;
753/// #     for j in 0..n {
754/// #         for i in 0..m {
755/// #             let mut acc = T::default();
756/// #             for p in 0..k {
757/// #                 acc = acc + unsafe { *a.add(i + p * lda) * *b.add(p + j * ldb) };
758/// #             }
759/// #             let cij = unsafe { &mut *c.add(i + j * ldc) };
760/// #             *cij = if beta == T::default() { alpha * acc } else { alpha * acc + beta * *cij };
761/// #         }
762/// #     }
763/// # }
764/// # unsafe extern "C" fn my_dgemm64(
765/// #     _transa: *const c_char, _transb: *const c_char,
766/// #     m: *const i64, n: *const i64, k: *const i64,
767/// #     alpha: *const f64, a: *const f64, lda: *const i64,
768/// #     b: *const f64, ldb: *const i64,
769/// #     beta: *const f64, c: *mut f64, ldc: *const i64,
770/// # ) {
771/// #     unsafe {
772/// #         let mnk = [*m as usize, *n as usize, *k as usize];
773/// #         gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
774/// #     }
775/// # }
776/// # unsafe extern "C" fn my_zgemm64(
777/// #     _transa: *const c_char, _transb: *const c_char,
778/// #     m: *const i64, n: *const i64, k: *const i64,
779/// #     alpha: *const Complex<f64>, a: *const Complex<f64>, lda: *const i64,
780/// #     b: *const Complex<f64>, ldb: *const i64,
781/// #     beta: *const Complex<f64>, c: *mut Complex<f64>, ldc: *const i64,
782/// # ) {
783/// #     unsafe {
784/// #         let mnk = [*m as usize, *n as usize, *k as usize];
785/// #         gemm_nn(mnk, *alpha, a, *lda as usize, b, *ldb as usize, *beta, c, *ldc as usize);
786/// #     }
787/// # }
788///
789/// // `my_dgemm64` and `my_zgemm64` are the ILP64 (64-bit integer) Fortran BLAS
790/// // `dgemm_` and `zgemm_` to use. (This example defines naive stand-ins in
791/// // hidden lines so that it runs without an ILP64 BLAS library.)
792/// unsafe {
793///     set_ilp64_backend(my_dgemm64, my_zgemm64);
794/// }
795/// assert_eq!(get_backend_info(), ("External BLAS (ILP64)", true, true));
796///
797/// // Calls without an explicit backend handle now go through the injected BLAS
798/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
799/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
800/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
801/// let c = matmul(None, &a, &b).unwrap();
802/// assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
803///
804/// clear_blas_backend(); // back to the pure-Rust Faer backend
805/// let (_, is_external, is_ilp64) = get_backend_info();
806/// assert!(!is_external && !is_ilp64);
807/// ```
808pub unsafe fn set_ilp64_backend(dgemm64: Dgemm64FnPtr, zgemm64: Zgemm64FnPtr) {
809    let mut dispatcher = BLAS_DISPATCHER.write().unwrap_or_else(|e| e.into_inner());
810    *dispatcher = Box::new(ExternalBlas64Backend::new(dgemm64, zgemm64));
811}
812
813/// Reset the process-wide default to the pure Rust faer backend.
814pub fn clear_blas_backend() {
815    let mut dispatcher = BLAS_DISPATCHER.write().unwrap_or_else(|e| e.into_inner());
816    *dispatcher = Box::new(FaerBackend);
817}
818
819/// Returns `(name, is_external, is_ilp64)` of the process-wide default.
820pub fn get_backend_info() -> (&'static str, bool, bool) {
821    let dispatcher = BLAS_DISPATCHER.read().unwrap_or_else(|e| e.into_inner());
822    let name = dispatcher.name();
823    (name, !name.contains("Faer"), dispatcher.is_ilp64())
824}
825
826/// Run `f` with the selected backend (explicit handle or process default).
827fn with_backend<R>(
828    backend: Option<&GemmBackendHandle>,
829    f: impl FnOnce(&dyn GemmBackend) -> R,
830) -> R {
831    match backend {
832        Some(handle) => f(handle.as_ref()),
833        None => {
834            let dispatcher = BLAS_DISPATCHER.read().unwrap_or_else(|e| e.into_inner());
835            f(dispatcher.as_ref())
836        }
837    }
838}
839
840//==============================================================================
841// Scalar dispatch and safe entry points
842//==============================================================================
843
844mod sealed {
845    pub trait Sealed {}
846    impl Sealed for f64 {}
847    impl Sealed for num_complex::Complex<f64> {}
848}
849
850/// Scalars supported by [`gemm`]: `f64` and `Complex<f64>`.
851pub trait GemmScalar:
852    sealed::Sealed
853    + tenferro_tensor::TensorScalar
854    + Copy
855    + Send
856    + Sync
857    + PartialEq
858    + Zero
859    + One
860    + std::ops::Mul<Output = Self>
861    + 'static
862{
863    /// Dispatch to the backend routine for this scalar.
864    ///
865    /// # Safety
866    /// Same contract as [`GemmBackend::dgemm`].
867    #[allow(clippy::too_many_arguments)]
868    unsafe fn gemm_raw(
869        backend: &dyn GemmBackend,
870        transa: Transpose,
871        transb: Transpose,
872        m: usize,
873        n: usize,
874        k: usize,
875        alpha: Self,
876        a: *const Self,
877        lda: usize,
878        b: *const Self,
879        ldb: usize,
880        beta: Self,
881        c: *mut Self,
882        ldc: usize,
883    ) -> Result<(), GemmError>;
884}
885
886impl GemmScalar for f64 {
887    unsafe fn gemm_raw(
888        backend: &dyn GemmBackend,
889        transa: Transpose,
890        transb: Transpose,
891        m: usize,
892        n: usize,
893        k: usize,
894        alpha: f64,
895        a: *const f64,
896        lda: usize,
897        b: *const f64,
898        ldb: usize,
899        beta: f64,
900        c: *mut f64,
901        ldc: usize,
902    ) -> Result<(), GemmError> {
903        // SAFETY: forwarded caller contract.
904        unsafe { backend.dgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) }
905    }
906}
907
908impl GemmScalar for Complex<f64> {
909    unsafe fn gemm_raw(
910        backend: &dyn GemmBackend,
911        transa: Transpose,
912        transb: Transpose,
913        m: usize,
914        n: usize,
915        k: usize,
916        alpha: Self,
917        a: *const Self,
918        lda: usize,
919        b: *const Self,
920        ldb: usize,
921        beta: Self,
922        c: *mut Self,
923        ldc: usize,
924    ) -> Result<(), GemmError> {
925        // SAFETY: forwarded caller contract.
926        unsafe { backend.zgemm(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc) }
927    }
928}
929
930/// Validate one column-major operand: `rows x cols` stored with leading
931/// dimension `ld` in a buffer of `len` elements.
932fn check_operand(
933    name: &str,
934    rows: usize,
935    cols: usize,
936    ld: usize,
937    len: usize,
938) -> Result<(), GemmError> {
939    if ld < rows.max(1) {
940        return Err(GemmError::InvalidArgument(format!(
941            "leading dimension of {name} is {ld}, expected at least {}",
942            rows.max(1)
943        )));
944    }
945    if rows == 0 || cols == 0 {
946        return Ok(());
947    }
948    let required = ld
949        .checked_mul(cols - 1)
950        .and_then(|x| x.checked_add(rows))
951        .ok_or_else(|| GemmError::InvalidArgument(format!("{name} extent overflows usize")))?;
952    if len < required {
953        return Err(GemmError::InvalidArgument(format!(
954            "{name} buffer has {len} elements, {rows}x{cols} with ld={ld} needs {required}"
955        )));
956    }
957    Ok(())
958}
959
960/// Safe column-major GEMM: `C <- alpha * op(A) * op(B) + beta * C`.
961///
962/// `op(A)` is `m x k`, `op(B)` is `k x n`, and `C` is `m x n`; the stored
963/// operands are addressed through their leading dimensions.
964///
965/// # Errors
966/// Returns [`GemmError::InvalidArgument`] when a buffer is too short or a
967/// leading dimension is too small, and [`GemmError::DimensionOverflow`] when
968/// an LP64 backend cannot represent a dimension.
969#[allow(clippy::too_many_arguments)]
970pub fn gemm<T: GemmScalar>(
971    backend: Option<&GemmBackendHandle>,
972    transa: Transpose,
973    transb: Transpose,
974    m: usize,
975    n: usize,
976    k: usize,
977    alpha: T,
978    a: &[T],
979    lda: usize,
980    b: &[T],
981    ldb: usize,
982    beta: T,
983    c: &mut [T],
984    ldc: usize,
985) -> Result<(), GemmError> {
986    let (ar, ac) = if transa == Transpose::N {
987        (m, k)
988    } else {
989        (k, m)
990    };
991    let (br, bc) = if transb == Transpose::N {
992        (k, n)
993    } else {
994        (n, k)
995    };
996    check_operand("A", ar, ac, lda, a.len())?;
997    check_operand("B", br, bc, ldb, b.len())?;
998    check_operand("C", m, n, ldc, c.len())?;
999    // SAFETY: extents validated above; `c` is a unique borrow so it cannot
1000    // alias `a` or `b`.
1001    with_backend(backend, |be| unsafe {
1002        T::gemm_raw(
1003            be,
1004            transa,
1005            transb,
1006            m,
1007            n,
1008            k,
1009            alpha,
1010            a.as_ptr(),
1011            lda,
1012            b.as_ptr(),
1013            ldb,
1014            beta,
1015            c.as_mut_ptr(),
1016            ldc,
1017        )
1018    })
1019}
1020
1021/// Below this `pre` extent, a middle-axis contraction is packed into one
1022/// GEMM instead of looping over `post` slices.
1023const SLICE_LOOP_MIN_PRE: usize = 32;
1024
1025/// Matrix-vector products up to this size bypass the default backend.
1026const SMALL_MATVEC_MAX_ELEMS: usize = 128 * 128;
1027const SMALL_MATVEC_MAX_POST: usize = 4;
1028
1029/// Apply a contiguous column-major `m x n` matrix `a` along the middle axis
1030/// of a column-major `[pre, n, post]` array `x`, writing `[pre, m, post]`
1031/// into `y` (overwritten).
1032///
1033/// # Errors
1034/// Returns [`GemmError::InvalidArgument`] when a buffer length does not match
1035/// its extents, or a backend error.
1036#[allow(clippy::too_many_arguments)]
1037pub(crate) fn apply_along_axis<T: GemmScalar>(
1038    backend: Option<&GemmBackendHandle>,
1039    a: &[T],
1040    m: usize,
1041    n: usize,
1042    x: &[T],
1043    pre: usize,
1044    post: usize,
1045    y: &mut [T],
1046) -> Result<(), GemmError> {
1047    let len_ok = |len: usize, rows: usize| {
1048        pre.checked_mul(rows)
1049            .and_then(|v| v.checked_mul(post))
1050            .is_some_and(|v| v == len)
1051    };
1052    if a.len() != m * n || !len_ok(x.len(), n) || !len_ok(y.len(), m) {
1053        return Err(GemmError::InvalidArgument(format!(
1054            "apply_along_axis: A {}, X {}, Y {} elements for m={m}, n={n}, pre={pre}, post={post}",
1055            a.len(),
1056            x.len(),
1057            y.len()
1058        )));
1059    }
1060    if y.is_empty() {
1061        return Ok(());
1062    }
1063    let (one, zero) = (T::one(), T::zero());
1064    let lda = m.max(1);
1065    if pre == 1
1066        && post <= SMALL_MATVEC_MAX_POST
1067        && m * n <= SMALL_MATVEC_MAX_ELEMS
1068        && backend.is_none()
1069    {
1070        // Small matrix-vector products: the default backend's dispatch
1071        // overhead dominates, so accumulate columns of A directly.
1072        for q in 0..post {
1073            let yq = &mut y[q * m..(q + 1) * m];
1074            yq.fill(zero);
1075            for (j, col) in a.chunks_exact(m).enumerate() {
1076                let xj = x[q * n + j];
1077                for (yi, &aij) in yq.iter_mut().zip(col) {
1078                    *yi = *yi + aij * xj;
1079                }
1080            }
1081        }
1082        return Ok(());
1083    }
1084    if pre == 1 {
1085        // Y(m x post) = A(m x n) X(n x post)
1086        return gemm(
1087            backend,
1088            Transpose::N,
1089            Transpose::N,
1090            m,
1091            post,
1092            n,
1093            one,
1094            a,
1095            lda,
1096            x,
1097            n.max(1),
1098            zero,
1099            y,
1100            m.max(1),
1101        );
1102    }
1103    if post == 1 || pre >= SLICE_LOOP_MIN_PRE {
1104        // Y_q(pre x m) = X_q(pre x n) A^T for each post index q.
1105        let (xs, ys) = (pre * n, pre * m);
1106        for q in 0..post {
1107            gemm(
1108                backend,
1109                Transpose::N,
1110                Transpose::T,
1111                pre,
1112                m,
1113                n,
1114                one,
1115                &x[q * xs..(q + 1) * xs],
1116                pre,
1117                a,
1118                lda,
1119                zero,
1120                &mut y[q * ys..(q + 1) * ys],
1121                pre,
1122            )?;
1123        }
1124        return Ok(());
1125    }
1126    // Small pre, many post slices: pack X into [n, pre*post], one GEMM, unpack.
1127    let cols = pre * post;
1128    let mut xp = Vec::with_capacity(n * cols);
1129    for q in 0..post {
1130        for p in 0..pre {
1131            let base = p + pre * n * q;
1132            xp.extend((0..n).map(|j| x[base + pre * j]));
1133        }
1134    }
1135    let mut yp = vec![zero; m * cols];
1136    gemm(
1137        backend,
1138        Transpose::N,
1139        Transpose::N,
1140        m,
1141        cols,
1142        n,
1143        one,
1144        a,
1145        lda,
1146        &xp,
1147        n.max(1),
1148        zero,
1149        &mut yp,
1150        m.max(1),
1151    )?;
1152    for q in 0..post {
1153        for p in 0..pre {
1154            let src = &yp[m * (p + pre * q)..m * (p + pre * q + 1)];
1155            let base = p + pre * m * q;
1156            for (i, &v) in src.iter().enumerate() {
1157                y[base + pre * i] = v;
1158            }
1159        }
1160    }
1161    Ok(())
1162}
1163
1164/// Matrix product `A * B` of two contiguous column-major matrices.
1165///
1166/// # Errors
1167/// Returns [`crate::Error::ShapeMismatch`] when the inner dimensions differ,
1168/// [`crate::Error::Tensor`] when an operand is not host-resident compact
1169/// column-major storage, or a backend error.
1170///
1171/// # Example
1172/// ```
1173/// use sparse_ir::Matrix;
1174/// use sparse_ir::gemm::{GemmBackendHandle, matmul};
1175///
1176/// // Column-major data: [[1, 2], [3, 4]] and [[5, 6], [7, 8]]
1177/// let a = Matrix::<f64>::from_vec_col_major([2, 2], vec![1.0, 3.0, 2.0, 4.0]).unwrap();
1178/// let b = Matrix::<f64>::from_vec_col_major([2, 2], vec![5.0, 7.0, 6.0, 8.0]).unwrap();
1179/// let backend = GemmBackendHandle::default();
1180/// let c = matmul(Some(&backend), &a, &b).unwrap();
1181/// assert_eq!(c.host_data().unwrap(), &[19.0, 43.0, 22.0, 50.0]); // [[19, 22], [43, 50]]
1182///
1183/// // A 2x3 times 3x1 product
1184/// let a = Matrix::<f64>::from_vec_col_major([2, 3], vec![1.0, 4.0, 2.0, 5.0, 3.0, 6.0]).unwrap();
1185/// let b = Matrix::<f64>::from_vec_col_major([3, 1], vec![7.0, 8.0, 9.0]).unwrap();
1186/// assert_eq!(matmul(None, &a, &b).unwrap().host_data().unwrap(), &[50.0, 122.0]);
1187/// ```
1188pub fn matmul<T: GemmScalar>(
1189    backend: Option<&GemmBackendHandle>,
1190    a: &crate::Matrix<T>,
1191    b: &crate::Matrix<T>,
1192) -> crate::Result<crate::Matrix<T>> {
1193    let (av, bv) = (a.host_col_major_view()?, b.host_col_major_view()?);
1194    let [m, k] = *av.shape();
1195    let [k2, n] = *bv.shape();
1196    if k != k2 {
1197        return Err(crate::Error::ShapeMismatch {
1198            which: crate::ArrayRole::Input,
1199            expected: vec![k, n],
1200            actual: vec![k2, n],
1201        });
1202    }
1203    let mut c = vec![T::zero(); m * n];
1204    gemm(
1205        backend,
1206        Transpose::N,
1207        Transpose::N,
1208        m,
1209        n,
1210        k,
1211        T::one(),
1212        av.as_slice(),
1213        m.max(1),
1214        bv.as_slice(),
1215        k.max(1),
1216        T::zero(),
1217        &mut c,
1218        m.max(1),
1219    )?;
1220    Ok(crate::Matrix::from_vec_col_major([m, n], c)?)
1221}
1222
1223#[cfg(test)]
1224#[path = "gemm_tests.rs"]
1225mod tests;