From f5ae6e5b83d6c230096d7c94c6745d4f40780a64 Mon Sep 17 00:00:00 2001 From: Hugo Meiland Date: Sun, 27 Sep 2026 15:59:36 +0200 Subject: [PATCH 1/2] Add RISCV64_ZVL1024B target for wide-VLEN RVV (SpacemiT A100) Introduce GEMM/TRMM kernels generated for VLEN>=1024 (d/s 16x8, c 8x8, z 8x4), DYNAMIC_ARCH dispatch on vlenb>=128, and CMake/docs wiring. Also make stock ZVL256B GEMM vget splits VLEN-portable so the same DYNAMIC_ARCH build remains correct on wider vector units. Co-authored-by: Cursor --- Makefile.prebuild | 3 + Makefile.riscv64 | 4 + Makefile.system | 1 + README.md | 10 +- TargetList.txt | 1 + cmake/arch.cmake | 2 +- cmake/cc.cmake | 5 +- cmake/prebuild.cmake | 18 + cmake/system.cmake | 5 +- common_riscv64.h | 4 +- cpuid_riscv64.c | 19 +- docs/install.md | 5 +- driver/others/dynamic_riscv64.c | 29 +- getarch.c | 13 + kernel/riscv64/KERNEL.RISCV64_ZVL1024B | 293 +++ kernel/riscv64/cgemm_kernel_8x8_zvl1024b.c | 1828 ++++++++++++++++++ kernel/riscv64/ctrmm_kernel_8x8_zvl1024b.c | 1901 +++++++++++++++++++ kernel/riscv64/dgemm_kernel_16x8_zvl1024b.c | 998 ++++++++++ kernel/riscv64/dgemm_kernel_8x8_zvl256b.c | 144 +- kernel/riscv64/dtrmm_kernel_16x8_zvl1024b.c | 1262 ++++++++++++ kernel/riscv64/sgemm_kernel_16x8_zvl1024b.c | 998 ++++++++++ kernel/riscv64/sgemm_kernel_16x8_zvl256b.c | 80 +- kernel/riscv64/strmm_kernel_16x8_zvl1024b.c | 1262 ++++++++++++ kernel/riscv64/zgemm_kernel_8x4_zvl1024b.c | 1018 ++++++++++ kernel/riscv64/ztrmm_kernel_8x4_zvl1024b.c | 1125 +++++++++++ param.h | 78 + 26 files changed, 10975 insertions(+), 131 deletions(-) create mode 100644 kernel/riscv64/KERNEL.RISCV64_ZVL1024B create mode 100644 kernel/riscv64/cgemm_kernel_8x8_zvl1024b.c create mode 100644 kernel/riscv64/ctrmm_kernel_8x8_zvl1024b.c create mode 100644 kernel/riscv64/dgemm_kernel_16x8_zvl1024b.c create mode 100644 kernel/riscv64/dtrmm_kernel_16x8_zvl1024b.c create mode 100644 kernel/riscv64/sgemm_kernel_16x8_zvl1024b.c create mode 100644 kernel/riscv64/strmm_kernel_16x8_zvl1024b.c create mode 100644 kernel/riscv64/zgemm_kernel_8x4_zvl1024b.c create mode 100644 kernel/riscv64/ztrmm_kernel_8x4_zvl1024b.c diff --git a/Makefile.prebuild b/Makefile.prebuild index 6b2567cb90..4214703c70 100644 --- a/Makefile.prebuild +++ b/Makefile.prebuild @@ -63,6 +63,9 @@ ifeq ($(TARGET), x280) TARGET_FLAGS = -march=rv64imafdcv_zba_zbb_zfh -mabi=lp64d endif +ifeq ($(TARGET), RISCV64_ZVL1024B) +TARGET_FLAGS = -march=rv64imafdcv -mabi=lp64d +endif ifeq ($(TARGET), RISCV64_ZVL256B) TARGET_FLAGS = -march=rv64imafdcv -mabi=lp64d endif diff --git a/Makefile.riscv64 b/Makefile.riscv64 index 7ee92a46f0..4c1c7e82a4 100644 --- a/Makefile.riscv64 +++ b/Makefile.riscv64 @@ -13,6 +13,10 @@ endif ifeq ($(BUILD_BFLOAT16), 1) RISCV64_OPT := $(RISCV64_OPT)_zvfbfwma endif +ifeq ($(CORE), RISCV64_ZVL1024B) +CCOMMON_OPT += -march=$(RISCV64_OPT)_zvl1024b -mabi=lp64d +FCOMMON_OPT += -march=$(RISCV64_OPT) -mabi=lp64d +endif ifeq ($(CORE), RISCV64_ZVL256B) CCOMMON_OPT += -march=$(RISCV64_OPT)_zvl256b -mabi=lp64d FCOMMON_OPT += -march=$(RISCV64_OPT) -mabi=lp64d diff --git a/Makefile.system b/Makefile.system index e8e732e9c8..f394498bdc 100644 --- a/Makefile.system +++ b/Makefile.system @@ -783,6 +783,7 @@ ifeq ($(ARCH), riscv64) DYNAMIC_CORE = RISCV64_GENERIC DYNAMIC_CORE += RISCV64_ZVL128B DYNAMIC_CORE += RISCV64_ZVL256B +DYNAMIC_CORE += RISCV64_ZVL1024B ifdef DYNAMIC_LIST override DYNAMIC_CORE = RISCV64_GENERIC $(DYNAMIC_LIST) XCCOMMON_OPT = -DDYNAMIC_LIST -DDYN_RISCV64_GENERIC diff --git a/README.md b/README.md index d651c42b36..73961fd5da 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,7 @@ Please read `GotoBLAS_01Readme.txt` for older CPU models already supported by th (For RVV 1.0 target selection and compiler requirements, see the [RISC-V section in docs/install.md](docs/install.md#risc-v).) -- **ZVL???B**: Level-3 BLAS and Level-1,2 including vectorised kernels targeting generic RISCV cores with vector support with registers of at least the corresponding width; ZVL128B and ZVL256B are available. +- **ZVL???B**: Level-3 BLAS and Level-1,2 including vectorised kernels targeting generic RISCV cores with vector support with registers of at least the corresponding width; ZVL128B, ZVL256B, and ZVL1024B are available. e.g.: ```sh make TARGET=RISCV64_ZVL256B CFLAGS="-DTARGET=RISCV64_ZVL256B" \ @@ -233,6 +233,14 @@ e.g.: HOSTCC=gcc HOSTFC=gfortran -j ``` + ```sh + make TARGET=RISCV64_ZVL1024B CFLAGS="-DTARGET=RISCV64_ZVL1024B" \ + BINARY=64 ARCH=riscv64 CC='clang -target riscv64-unknown-linux-gnu' \ + AR=riscv64-unknown-linux-gnu-ar AS=riscv64-unknown-linux-gnu-gcc \ + LD=riscv64-unknown-linux-gnu-gcc FC=riscv64-unknown-linux-gnu-gfortran \ + HOSTCC=gcc HOSTFC=gfortran -j + ``` + #### LOONGARCH64 - **LA64_GENERIC**: Optimized Level-3, Level-2 and Level-1 BLAS with scalar instruction diff --git a/TargetList.txt b/TargetList.txt index 377ecc7d0d..995b765e3e 100644 --- a/TargetList.txt +++ b/TargetList.txt @@ -128,6 +128,7 @@ RISCV64_ZVL128B C910V x280 RISCV64_ZVL256B +RISCV64_ZVL1024B U74 (e.g. SiFive U74 / StarFive JH7110 / VisionFive 2) 11.LOONGARCH64: diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 12d7de54e7..0515987609 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -80,7 +80,7 @@ if (DYNAMIC_ARCH) endif () if (RISCV64) - set(DYNAMIC_CORE RISCV64_GENERIC RISCV64_ZVL128B RISCV64_ZVL256B) + set(DYNAMIC_CORE RISCV64_GENERIC RISCV64_ZVL128B RISCV64_ZVL256B RISCV64_ZVL1024B) endif () if (X86) diff --git a/cmake/cc.cmake b/cmake/cc.cmake index 4d8735490a..43dd20a0de 100644 --- a/cmake/cc.cmake +++ b/cmake/cc.cmake @@ -410,7 +410,7 @@ if (${CORE} STREQUAL PPCG4) endif () -if ((${CORE} STREQUAL RISCV64_ZVL128B) OR (${CORE} STREQUAL RISCV64_ZVL256B)) +if ((${CORE} STREQUAL RISCV64_ZVL128B) OR (${CORE} STREQUAL RISCV64_ZVL256B) OR (${CORE} STREQUAL RISCV64_ZVL1024B)) set (RISCV64_OPT "rv64imafdcv") if (BUILD_BFLOAT16) set (RISCV64_OPT "${RISCV64_OPT}_zvfbfwma") @@ -418,6 +418,9 @@ if ((${CORE} STREQUAL RISCV64_ZVL128B) OR (${CORE} STREQUAL RISCV64_ZVL256B)) if (BUILD_HFLOAT16) set (RISCV64_OPT "${RISCV64_OPT}_zvfh_zfh") endif() + if (${CORE} STREQUAL RISCV64_ZVL1024B) + set (CCOMMON_OPT "${CCOMMON_OPT} -march=${RISCV64_OPT}_zvl1024b -mabi=lp64d") + endif() if (${CORE} STREQUAL RISCV64_ZVL256B) set (CCOMMON_OPT "${CCOMMON_OPT} -march=${RISCV64_OPT}_zvl256b -mabi=lp64d") endif() diff --git a/cmake/prebuild.cmake b/cmake/prebuild.cmake index cb02556e3d..652fb5098e 100644 --- a/cmake/prebuild.cmake +++ b/cmake/prebuild.cmake @@ -1468,6 +1468,24 @@ endif () set(ZGEMM_UNROLL_M 4) set(ZGEMM_UNROLL_N 4) set(SYMV_P 16) + elseif ("${TCORE}" STREQUAL "RISCV64_ZVL1024B") + file(APPEND ${TARGET_CONF_TEMP} + "#define L1_DATA_SIZE 65536\n" + "#define L1_DATA_LINESIZE 32\n" + "#define L2_SIZE 2097152\n" + "#define L2_LINESIZE 32 \n" + "#define DTB_DEFAULT_ENTRIES 128\n" + "#define DTB_SIZE 4096\n" + "#define L2_ASSOCIATIVE 4\n") + set(SGEMM_UNROLL_M 16) + set(SGEMM_UNROLL_N 8) + set(DGEMM_UNROLL_M 16) + set(DGEMM_UNROLL_N 8) + set(CGEMM_UNROLL_M 8) + set(CGEMM_UNROLL_N 8) + set(ZGEMM_UNROLL_M 8) + set(ZGEMM_UNROLL_N 4) + set(SYMV_P 16) elseif ("${TCORE}" STREQUAL "RISCV64_ZVL256B") file(APPEND ${TARGET_CONF_TEMP} "#define L1_DATA_SIZE 65536\n" diff --git a/cmake/system.cmake b/cmake/system.cmake index 436f2109aa..87b5e516d2 100644 --- a/cmake/system.cmake +++ b/cmake/system.cmake @@ -392,7 +392,7 @@ if (${TARGET} STREQUAL NEOVERSEV1) endif() endif() - if ((${TARGET} STREQUAL RISCV64_ZVL128B) OR (${TARGET} STREQUAL RISCV64_ZVL256B)) + if ((${TARGET} STREQUAL RISCV64_ZVL128B) OR (${TARGET} STREQUAL RISCV64_ZVL256B) OR (${TARGET} STREQUAL RISCV64_ZVL1024B)) set (RISCV64_OPT "rv64imafdcv") if (BUILD_BFLOAT16) set (RISCV64_OPT "${RISCV64_OPT}_zvfbfwma") @@ -400,6 +400,9 @@ if (${TARGET} STREQUAL NEOVERSEV1) if (BUILD_HFLOAT16) set (RISCV64_OPT "${RISCV64_OPT}_zvfh_zfh") endif() + if (${TARGET} STREQUAL RISCV64_ZVL1024B) + set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=${RISCV64_OPT}_zvl1024b -mabi=lp64d") + endif() if (${TARGET} STREQUAL RISCV64_ZVL256B) set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -march=${RISCV64_OPT}_zvl256b -mabi=lp64d") endif() diff --git a/common_riscv64.h b/common_riscv64.h index ba638e8be5..2eab02fead 100644 --- a/common_riscv64.h +++ b/common_riscv64.h @@ -89,7 +89,7 @@ static inline int blas_quickdivide(blasint x, blasint y){ #define BUFFER_SIZE ( 32 << 20) #define SEEK_ADDRESS -#if defined(C910V) || defined(RISCV64_ZVL256B) || defined(RISCV64_ZVL128B) || defined(x280) +#if defined(C910V) || defined(RISCV64_ZVL1024B) || defined(RISCV64_ZVL256B) || defined(RISCV64_ZVL128B) || defined(x280) # include #endif @@ -101,7 +101,7 @@ static inline int blas_quickdivide(blasint x, blasint y){ #define RISCV_RVV(x) __riscv_ ## x #endif -#if defined(C910V) || defined(RISCV64_ZVL256B) +#if defined(C910V) || defined(RISCV64_ZVL1024B) || defined(RISCV64_ZVL256B) # if !defined(DOUBLE) # define EXTRACT_FLOAT(v) RISCV_RVV(vfmv_f_s_f32m1_f32)(v) # else diff --git a/cpuid_riscv64.c b/cpuid_riscv64.c index 859da986eb..c3fe1dfa6b 100644 --- a/cpuid_riscv64.c +++ b/cpuid_riscv64.c @@ -70,19 +70,21 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. /* or implied, of The University of Texas at Austin. */ /*********************************************************************/ -#define CPU_GENERIC 0 -#define CPU_C910V 1 -#define CPU_x280 2 -#define CPU_RISCV64_ZVL256B 3 -#define CPU_RISCV64_ZVL128B 4 -#define CPU_U74 5 +#define CPU_GENERIC 0 +#define CPU_C910V 1 +#define CPU_x280 2 +#define CPU_RISCV64_ZVL1024B 3 +#define CPU_RISCV64_ZVL256B 4 +#define CPU_RISCV64_ZVL128B 5 +#define CPU_U74 6 static char *cpuname[] = { "RISCV64_GENERIC", "C910V", "x280", - "CPU_RISCV64_ZVL256B", - "CPU_RISCV64_ZVL128B", + "RISCV64_ZVL1024B", + "RISCV64_ZVL256B", + "RISCV64_ZVL128B", "U74" }; @@ -90,6 +92,7 @@ static char *cpuname_lower[] = { "riscv64_generic", "c910v", "x280", + "riscv64_zvl1024b", "riscv64_zvl256b", "riscv64_zvl128b", "u74" diff --git a/docs/install.md b/docs/install.md index edd539918f..77f8383452 100644 --- a/docs/install.md +++ b/docs/install.md @@ -810,6 +810,7 @@ the resulting build is vectorized. | `RISCV64_GENERIC` | None — scalar reference path | Non-vector cores, or as a baseline reference build | | `RISCV64_ZVL128B` | RVV 1.0, VLEN >= 128 bits | Generic RVV-1.0 hardware with 128-bit vector registers | | `RISCV64_ZVL256B` | RVV 1.0, VLEN >= 256 bits | Generic RVV-1.0 hardware with 256-bit vector registers | +| `RISCV64_ZVL1024B` | RVV 1.0, VLEN >= 1024 bits | Wide-VLEN RVV cores (e.g. SpacemiT K3 A100) | | `C910V` | RVV 0.7.1 (legacy) | T-Head C910 (Allwinner D1, etc.) | | `x280` | RVV 1.0, SiFive-tuned | SiFive x280 cores | @@ -820,7 +821,7 @@ scalar `-march` override for this target that takes precedence over any user-supplied `-march=rv64gcv` flag, so passing the V extension on the command line will not produce a vectorized build under this target. -For RVV 1.0 vectorized builds, use `RISCV64_ZVL128B` or `RISCV64_ZVL256B`. +For RVV 1.0 vectorized builds, use `RISCV64_ZVL128B`, `RISCV64_ZVL256B`, or `RISCV64_ZVL1024B`. These targets route all three BLAS levels including DGEMM to the `_rvv.c` kernel set introduced in 2022; see [issue #3808](https://github.com/OpenMathLib/OpenBLAS/issues/3808) for the @@ -830,7 +831,7 @@ codebase separation. #### Compiler requirements for ZVL targets GCC 14 or later is required on current OpenBLAS releases when building the -`RISCV64_ZVL128B` or `RISCV64_ZVL256B` targets. GCC 13 does not implement +`RISCV64_ZVL128B`, `RISCV64_ZVL256B`, or `RISCV64_ZVL1024B` targets. GCC 13 does not implement the segmented load/store intrinsics (`__riscv_vsseg*`) used by the `_rvv.c` kernels; under GCC 13 the build still completes and produces a library, but the affected routines fall back to scalar code paths. diff --git a/driver/others/dynamic_riscv64.c b/driver/others/dynamic_riscv64.c index ec7fe8f55e..15a0190cc9 100644 --- a/driver/others/dynamic_riscv64.c +++ b/driver/others/dynamic_riscv64.c @@ -113,6 +113,9 @@ unsigned detect_riscv64_get_vlenb(void); uint64_t detect_riscv64_rvv100(void); extern gotoblas_t gotoblas_RISCV64_GENERIC; +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL1024B) +extern gotoblas_t gotoblas_RISCV64_ZVL1024B; +#endif #if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) extern gotoblas_t gotoblas_RISCV64_ZVL256B; #endif @@ -120,12 +123,14 @@ extern gotoblas_t gotoblas_RISCV64_ZVL256B; extern gotoblas_t gotoblas_RISCV64_ZVL128B; #endif -#define CPU_GENERIC 0 -#define CPU_RISCV64_ZVL256B 1 -#define CPU_RISCV64_ZVL128B 2 +#define CPU_GENERIC 0 +#define CPU_RISCV64_ZVL1024B 1 +#define CPU_RISCV64_ZVL256B 2 +#define CPU_RISCV64_ZVL128B 3 static char *cpuname[] = { "riscv64_generic", + "riscv64_zvl1024b", "riscv64_zvl256b", "riscv64_zvl128b" }; @@ -135,6 +140,10 @@ extern int openblas_verbose(void); extern void openblas_warning(int verbose, const char* msg); char* gotoblas_corename(void) { +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL1024B) + if (gotoblas == &gotoblas_RISCV64_ZVL1024B) + return cpuname[CPU_RISCV64_ZVL1024B]; +#endif #if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) if (gotoblas == &gotoblas_RISCV64_ZVL256B) return cpuname[CPU_RISCV64_ZVL256B]; @@ -206,6 +215,10 @@ static gotoblas_t* get_coretype(void) { if (vlenb < 16) return NULL; +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL1024B) + if (vlenb >= 128) + return &gotoblas_RISCV64_ZVL1024B; +#endif #if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) if (vlenb >= 32) return &gotoblas_RISCV64_ZVL256B; @@ -229,6 +242,16 @@ static gotoblas_t* force_coretype(char* coretype) { if (i == CPU_GENERIC) return &gotoblas_RISCV64_GENERIC; + if (i == CPU_RISCV64_ZVL1024B) { +#if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL1024B) + return &gotoblas_RISCV64_ZVL1024B; +#else + openblas_warning(1, + "riscv64_zvl1024b support not compiled in\n"); + return NULL; +#endif + } + if (i == CPU_RISCV64_ZVL256B) { #if !defined(DYNAMIC_LIST) || defined(DYN_RISCV64_ZVL256B) return &gotoblas_RISCV64_ZVL256B; diff --git a/getarch.c b/getarch.c index ea0c52c368..8d7df51980 100644 --- a/getarch.c +++ b/getarch.c @@ -1840,6 +1840,19 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #else #endif +#ifdef FORCE_RISCV64_ZVL1024B +#define FORCE +#define ARCHITECTURE "RISCV64" +#define SUBARCHITECTURE "RISCV64_ZVL1024B" +#define SUBDIRNAME "riscv64" +#define ARCHCONFIG "-DRISCV64_ZVL1024B " \ + "-DL1_DATA_SIZE=64536 -DL1_DATA_LINESIZE=32 " \ + "-DL2_SIZE=262144 -DL2_LINESIZE=32 " \ + "-DDTB_DEFAULT_ENTRIES=128 -DDTB_SIZE=4096 -DL2_ASSOCIATIVE=4 " +#define LIBNAME "riscv64_zvl1024b" +#define CORENAME "RISCV64_ZVL1024B" +#endif + #ifdef FORCE_RISCV64_ZVL256B #define FORCE #define ARCHITECTURE "RISCV64" diff --git a/kernel/riscv64/KERNEL.RISCV64_ZVL1024B b/kernel/riscv64/KERNEL.RISCV64_ZVL1024B new file mode 100644 index 0000000000..9819188dce --- /dev/null +++ b/kernel/riscv64/KERNEL.RISCV64_ZVL1024B @@ -0,0 +1,293 @@ +SAMAXKERNEL = amax_vector.c +DAMAXKERNEL = amax_vector.c +CAMAXKERNEL = zamax_vector.c +ZAMAXKERNEL = zamax_vector.c + +SAMINKERNEL = amin_vector.c +DAMINKERNEL = amin_vector.c +CAMINKERNEL = zamin_vector.c +ZAMINKERNEL = zamin_vector.c + +SMAXKERNEL = max_vector.c +DMAXKERNEL = max_vector.c + +SMINKERNEL = min_vector.c +DMINKERNEL = min_vector.c + +ISAMAXKERNEL = iamax_vector.c +IDAMAXKERNEL = iamax_vector.c +ICAMAXKERNEL = izamax_vector.c +IZAMAXKERNEL = izamax_vector.c + +ISAMINKERNEL = iamin_vector.c +IDAMINKERNEL = iamin_vector.c +ICAMINKERNEL = izamin_vector.c +IZAMINKERNEL = izamin_vector.c + +ISMAXKERNEL = imax_vector.c +IDMAXKERNEL = imax_vector.c + +ISMINKERNEL = imin_vector.c +IDMINKERNEL = imin_vector.c + +SASUMKERNEL = asum_vector.c +DASUMKERNEL = asum_vector.c +CASUMKERNEL = zasum_vector.c +ZASUMKERNEL = zasum_vector.c + +SSUMKERNEL = sum_vector.c +DSUMKERNEL = sum_vector.c +CSUMKERNEL = zsum_vector.c +ZSUMKERNEL = zsum_vector.c + +SAXPYKERNEL = axpy_vector.c +DAXPYKERNEL = axpy_vector.c +CAXPYKERNEL = zaxpy_vector.c +ZAXPYKERNEL = zaxpy_vector.c + +SCOPYKERNEL = copy_vector.c +DCOPYKERNEL = copy_vector.c +CCOPYKERNEL = zcopy_vector.c +ZCOPYKERNEL = zcopy_vector.c + +SDOTKERNEL = dot_vector.c +DDOTKERNEL = dot_vector.c +CDOTKERNEL = zdot_vector.c +ZDOTKERNEL = zdot_vector.c +DSDOTKERNEL = ../generic/dot.c + +SNRM2KERNEL = nrm2_vector.c +DNRM2KERNEL = nrm2_vector.c +CNRM2KERNEL = znrm2_vector.c +ZNRM2KERNEL = znrm2_vector.c + +SROTKERNEL = rot_vector.c +DROTKERNEL = rot_vector.c +CROTKERNEL = zrot_vector.c +ZROTKERNEL = zrot_vector.c + +SROTMKERNEL = rotm_rvv.c +DROTMKERNEL = rotm_rvv.c +QROTMKERNEL = ../generic/rotm.c + +SSCALKERNEL = scal_vector.c +DSCALKERNEL = scal_vector.c +CSCALKERNEL = zscal_vector.c +ZSCALKERNEL = zscal_vector.c + +SSWAPKERNEL = swap_vector.c +DSWAPKERNEL = swap_vector.c +CSWAPKERNEL = zswap_vector.c +ZSWAPKERNEL = zswap_vector.c + +SGEMVNKERNEL = gemv_n_vector.c +DGEMVNKERNEL = gemv_n_vector.c +CGEMVNKERNEL = zgemv_n_vector.c +ZGEMVNKERNEL = zgemv_n_vector.c + +SGEMVTKERNEL = gemv_t_vector.c +DGEMVTKERNEL = gemv_t_vector.c +CGEMVTKERNEL = zgemv_t_vector.c +ZGEMVTKERNEL = zgemv_t_vector.c + +STRMMKERNEL = strmm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N)_zvl1024b.c +DTRMMKERNEL = dtrmm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N)_zvl1024b.c +CTRMMKERNEL = ctrmm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N)_zvl1024b.c +ZTRMMKERNEL = ztrmm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N)_zvl1024b.c + +SGEMMKERNEL = sgemm_kernel_$(SGEMM_UNROLL_M)x$(SGEMM_UNROLL_N)_zvl1024b.c +ifneq ($(filter $(SGEMM_UNROLL_N),4 8 16),) +SGEMMONCOPY = gemm_ncopy_$(SGEMM_UNROLL_N)_rvv.c +SGEMMOTCOPY = gemm_tcopy_$(SGEMM_UNROLL_N)_rvv.c +else +SGEMMONCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_N).c +SGEMMOTCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_N).c +endif +SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) +SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) +ifneq ($(SGEMM_UNROLL_M), $(SGEMM_UNROLL_N)) +ifneq ($(filter $(SGEMM_UNROLL_M),4 8 16),) +SGEMMINCOPY = gemm_ncopy_$(SGEMM_UNROLL_M)_rvv.c +SGEMMITCOPY = gemm_tcopy_$(SGEMM_UNROLL_M)_rvv.c +else +SGEMMINCOPY = ../generic/gemm_ncopy_$(SGEMM_UNROLL_M).c +SGEMMITCOPY = ../generic/gemm_tcopy_$(SGEMM_UNROLL_M).c +endif +SGEMMINCOPYOBJ = sgemm_incopy$(TSUFFIX).$(SUFFIX) +SGEMMITCOPYOBJ = sgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif + +DGEMMKERNEL = dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N)_zvl1024b.c +DGEMMONCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c +ifneq ($(filter $(DGEMM_UNROLL_N),4 8 16),) +DGEMMOTCOPY = gemm_tcopy_$(DGEMM_UNROLL_N)_rvv.c +else +DGEMMOTCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c +endif +DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) +DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) +ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N)) +DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c +ifneq ($(filter $(DGEMM_UNROLL_M),4 8 16),) +DGEMMITCOPY = gemm_tcopy_$(DGEMM_UNROLL_M)_rvv.c +else +DGEMMITCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c +endif +DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) +DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif + +CGEMMKERNEL = cgemm_kernel_$(CGEMM_UNROLL_M)x$(CGEMM_UNROLL_N)_zvl1024b.c +CGEMMONCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_N).c +CGEMMOTCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_N).c +CGEMMONCOPYOBJ = cgemm_oncopy$(TSUFFIX).$(SUFFIX) +CGEMMOTCOPYOBJ = cgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ifneq ($(CGEMM_UNROLL_M), $(CGEMM_UNROLL_N)) +CGEMMINCOPY = ../generic/zgemm_ncopy_$(CGEMM_UNROLL_M).c +CGEMMITCOPY = ../generic/zgemm_tcopy_$(CGEMM_UNROLL_M).c +CGEMMINCOPYOBJ = cgemm_incopy$(TSUFFIX).$(SUFFIX) +CGEMMITCOPYOBJ = cgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif + +ZGEMMKERNEL = zgemm_kernel_$(ZGEMM_UNROLL_M)x$(ZGEMM_UNROLL_N)_zvl1024b.c +ZGEMMONCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_N).c +ZGEMMOTCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c +ZGEMMONCOPYOBJ = zgemm_oncopy$(TSUFFIX).$(SUFFIX) +ZGEMMOTCOPYOBJ = zgemm_otcopy$(TSUFFIX).$(SUFFIX) + +ifneq ($(ZGEMM_UNROLL_M), $(ZGEMM_UNROLL_N)) +ZGEMMINCOPY = ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_M).c +ZGEMMITCOPY = ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_M).c +ZGEMMINCOPYOBJ = zgemm_incopy$(TSUFFIX).$(SUFFIX) +ZGEMMITCOPYOBJ = zgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif + +STRSMKERNEL_LN = trsm_kernel_LN_rvv.c +STRSMKERNEL_LT = trsm_kernel_LT_rvv.c +STRSMKERNEL_RN = trsm_kernel_RN_rvv.c +STRSMKERNEL_RT = trsm_kernel_RT_rvv.c + +DTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +DTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +DTRSMKERNEL_RN = trsm_kernel_RN_rvv.c +DTRSMKERNEL_RT = trsm_kernel_RT_rvv.c + +CTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +CTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +CTRSMKERNEL_RN = trsm_kernel_RN_rvv.c +CTRSMKERNEL_RT = trsm_kernel_RT_rvv.c + +ZTRSMKERNEL_LN = ../generic/trsm_kernel_LN.c +ZTRSMKERNEL_LT = ../generic/trsm_kernel_LT.c +ZTRSMKERNEL_RN = ../generic/trsm_kernel_RN.c +ZTRSMKERNEL_RT = trsm_kernel_RT_rvv.c + +SSYMV_U_KERNEL = symv_U_vector.c +SSYMV_L_KERNEL = symv_L_vector.c +DSYMV_U_KERNEL = symv_U_vector.c +DSYMV_L_KERNEL = symv_L_vector.c + +CSYMV_U_KERNEL = ../generic/zsymv_k.c +CSYMV_L_KERNEL = ../generic/zsymv_k.c +ZSYMV_U_KERNEL = ../generic/zsymv_k.c +ZSYMV_L_KERNEL = ../generic/zsymv_k.c + +CHEMV_L_KERNEL = zhemv_LM_vector.c +CHEMV_M_KERNEL = zhemv_LM_vector.c +CHEMV_U_KERNEL = zhemv_UV_vector.c +CHEMV_V_KERNEL = zhemv_UV_vector.c +ZHEMV_L_KERNEL = zhemv_LM_vector.c +ZHEMV_M_KERNEL = zhemv_LM_vector.c +ZHEMV_U_KERNEL = zhemv_UV_vector.c +ZHEMV_V_KERNEL = zhemv_UV_vector.c + +LSAME_KERNEL = ../generic/lsame.c + +SCABS_KERNEL = ../generic/cabs.c +DCABS_KERNEL = ../generic/cabs.c +QCABS_KERNEL = ../generic/cabs.c + +ifndef SGEMM_BETA +SGEMM_BETA = ../generic/gemm_beta.c +endif +ifndef DGEMM_BETA +DGEMM_BETA = ../generic/gemm_beta.c +endif +ifndef CGEMM_BETA +CGEMM_BETA = ../generic/zgemm_beta.c +endif +ifndef ZGEMM_BETA +ZGEMM_BETA = ../generic/zgemm_beta.c +endif + +ZOMATCOPY_CN = zomatcopy_cn_vector.c +COMATCOPY_CN = zomatcopy_cn_vector.c + +DOMATCOPY_CN = omatcopy_cn_vector.c +SOMATCOPY_CN = omatcopy_cn_vector.c + +DOMATCOPY_CT = omatcopy_ct_rvv.c +SOMATCOPY_CT = omatcopy_ct_rvv.c + + +ifeq ($(BUILD_HFLOAT16), 1) +SHGEMMKERNEL = shgemm_kernel_$(SHGEMM_UNROLL_M)x$(SHGEMM_UNROLL_N)_zvl256b.c +ifneq ($(SHGEMM_UNROLL_M), $(SHGEMM_UNROLL_N)) +ifneq ($(filter $(SHGEMM_UNROLL_M),8 16),) +SHGEMMINCOPY = gemm_ncopy_$(SHGEMM_UNROLL_M)fp_rvv.c +SHGEMMITCOPY = gemm_tcopy_$(SHGEMM_UNROLL_M)fp_rvv.c +else +SHGEMMINCOPY = ../generic/gemm_ncopy_$(SHGEMM_UNROLL_M).c +SHGEMMITCOPY = ../generic/gemm_tcopy_$(SHGEMM_UNROLL_M).c +endif +SHGEMMINCOPYOBJ = shgemm_incopy$(TSUFFIX).$(SUFFIX) +SHGEMMITCOPYOBJ = shgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif +ifneq ($(filter $(SHGEMM_UNROLL_N),8 16),) +SHGEMMONCOPY = gemm_ncopy_$(SHGEMM_UNROLL_N)fp_rvv.c +SHGEMMOTCOPY = gemm_tcopy_$(SHGEMM_UNROLL_N)fp_rvv.c +else +SHGEMMONCOPY = ../generic/gemm_ncopy_$(SHGEMM_UNROLL_N).c +SHGEMMOTCOPY = ../generic/gemm_tcopy_$(SHGEMM_UNROLL_N).c +endif +SHGEMMONCOPYOBJ = shgemm_oncopy$(TSUFFIX).$(SUFFIX) +SHGEMMOTCOPYOBJ = shgemm_otcopy$(TSUFFIX).$(SUFFIX) +ifndef SHGEMM_BETA +SHGEMM_BETA = gemm_beta_rvv.c +endif +SHGEMVNKERNEL = sbgemv_n_vector.c +SHGEMVTKERNEL = sbgemv_t_vector.c +endif + +ifeq ($(BUILD_BFLOAT16), 1) +SBGEMMKERNEL = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_zvl256b.c +ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N)) +ifneq ($(filter $(SBGEMM_UNROLL_M),8 16),) +SBGEMMINCOPY = gemm_ncopy_$(SBGEMM_UNROLL_M)fp_rvv.c +SBGEMMITCOPY = gemm_tcopy_$(SBGEMM_UNROLL_M)fp_rvv.c +else +SBGEMMINCOPY = ../generic/gemm_ncopy_$(SBGEMM_UNROLL_M).c +SBGEMMITCOPY = ../generic/gemm_tcopy_$(SBGEMM_UNROLL_M).c +endif +SBGEMMINCOPYOBJ = sbgemm_incopy$(TSUFFIX).$(SUFFIX) +SBGEMMITCOPYOBJ = sbgemm_itcopy$(TSUFFIX).$(SUFFIX) +endif +ifneq ($(filter $(SBGEMM_UNROLL_N),8 16),) +SBGEMMONCOPY = gemm_ncopy_$(SBGEMM_UNROLL_N)fp_rvv.c +SBGEMMOTCOPY = gemm_tcopy_$(SBGEMM_UNROLL_N)fp_rvv.c +else +SBGEMMONCOPY = ../generic/gemm_ncopy_$(SBGEMM_UNROLL_N).c +SBGEMMOTCOPY = ../generic/gemm_tcopy_$(SBGEMM_UNROLL_N).c +endif +SBGEMMONCOPYOBJ = sbgemm_oncopy$(TSUFFIX).$(SUFFIX) +SBGEMMOTCOPYOBJ = sbgemm_otcopy$(TSUFFIX).$(SUFFIX) +ifndef SBGEMM_BETA +SBGEMM_BETA = gemm_beta_rvv.c +endif +SBGEMVNKERNEL = sbgemv_n_vector.c +SBGEMVTKERNEL = sbgemv_t_vector.c +endif + +SAXPBYKERNEL = axpby_vector_v2.c +DAXPBYKERNEL = axpby_vector_v2.c diff --git a/kernel/riscv64/cgemm_kernel_8x8_zvl1024b.c b/kernel/riscv64/cgemm_kernel_8x8_zvl1024b.c new file mode 100644 index 0000000000..cc05485286 --- /dev/null +++ b/kernel/riscv64/cgemm_kernel_8x8_zvl1024b.c @@ -0,0 +1,1828 @@ +/* + +AUTOGENERATED KERNEL +Settings: + LMUL=1 + M=8 + M_tail_scalar_from=2 + N=8 + __riscv_='__riscv_' + complex=True + conjugate=False + cpu='zvl1024b' + force_acc_double=False + index_type='BLASLONG' + op='gemm' + param_precision='float' + reg_width_bits=1024 + tail_policy='' + trace=False + +Derived: + ELEN_ACC=32 + ELEN_PARAM=32 + LMUL_ACC=1 + VFMACC='__riscv_vfmacc_vf_f32m1' + VFMUL='__riscv_vfmul_vf_f32m1' + VLEV='__riscv_vle32_v_f32m1' + VLSEV='__riscv_vlse32_v_f32m1' + VMACC_TO_ACC='__riscv_vfmacc_vf_f32m1' + VMUL_TO_ACC='__riscv_vfmul_vf_f32m1' + VSETVL='__riscv_vsetvl_e32m1' + VSEV='__riscv_vse32_v_f32m1' + VSSEV='__riscv_vsse32_v_f32m1' + acc_vector_t='vfloat32m1_t' + output='cgemm_kernel_8x8_zvl1024b.c' + param_scalar_t='float' + param_vector_t='vfloat32m1_t' + +*/ + +#include "common.h" + + + +#if defined(NN) || defined(NT) || defined(TN) || defined(TT) + #define S0 1 + #define S1 -1 + #define S2 1 + #define S3 1 + #define VFMACC_RR __riscv_vfmsac + #define VFMACC_RI __riscv_vfmacc +#endif +#if defined(NR) || defined(NC) || defined(TR) || defined(TC) + #define S0 1 + #define S1 1 + #define S2 1 + #define S3 -1 + #define VFMACC_RR __riscv_vfmacc + #define VFMACC_RI __riscv_vfmsac +#endif +#if defined(RN) || defined(RT) || defined(CN) || defined(CT) + #define S0 1 + #define S1 1 + #define S2 -1 + #define S3 1 + #define VFMACC_RR __riscv_vfmacc + #define VFMACC_RI __riscv_vfnmsac +#endif +#if defined(RR) || defined(RC) || defined(CR) || defined(CC) + #define S0 1 + #define S1 -1 + #define S2 -1 + #define S3 -1 + #define VFMACC_RR __riscv_vfmsac + #define VFMACC_RI __riscv_vfnmacc +#endif + +int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alphar, FLOAT alphai, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc) + +{ + BLASLONG gvl = 0; + BLASLONG m_top = 0; + BLASLONG n_top = 0; + + + // -- MAIN PASS + + for (BLASLONG j=0; j= 8) { - vfloat64m8_t B00 = __riscv_vle64_v_f64m8(B, N * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); + B4 = __riscv_vle64_v_f64m2((B) + (N) * 2, N); + B6 = __riscv_vle64_v_f64m2((B) + (N) * 3, N); B += (N * 4); - B0 = __riscv_vget_v_f64m8_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m8_f64m2(B00, 1); - B4 = __riscv_vget_v_f64m8_f64m2(B00, 2); - B6 = __riscv_vget_v_f64m8_f64m2(B00, 3); #ifdef GEMM_NEW_PACKING A00 = A0[0 + (1 * 0)]; @@ -103,12 +103,12 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON result2 = __riscv_vfmul_vf_f64m2(B4, A02, 8); result4 = __riscv_vfmul_vf_f64m2(B6, A03, 8); - B00 = __riscv_vle64_v_f64m8(B, N * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); + B4 = __riscv_vle64_v_f64m2((B) + (N) * 2, N); + B6 = __riscv_vle64_v_f64m2((B) + (N) * 3, N); B += (N * 4); - B0 = __riscv_vget_v_f64m8_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m8_f64m2(B00, 1); - B4 = __riscv_vget_v_f64m8_f64m2(B00, 2); - B6 = __riscv_vget_v_f64m8_f64m2(B00, 3); result6 = __riscv_vfmul_vf_f64m2(B0, A04, 8); result8 = __riscv_vfmul_vf_f64m2(B2, A05, 8); @@ -118,12 +118,12 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON BLASLONG k = (K / 8); K &= 7; while (--k) { - B00 = __riscv_vle64_v_f64m8(B, N * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); + B4 = __riscv_vle64_v_f64m2((B) + (N) * 2, N); + B6 = __riscv_vle64_v_f64m2((B) + (N) * 3, N); B += (N * 4); - B0 = __riscv_vget_v_f64m8_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m8_f64m2(B00, 1); - B4 = __riscv_vget_v_f64m8_f64m2(B00, 2); - B6 = __riscv_vget_v_f64m8_f64m2(B00, 3); #ifdef GEMM_NEW_PACKING A00 = A0[0 + (1 * 0)]; @@ -152,12 +152,12 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON result2 = __riscv_vfmacc_vf_f64m2(result2, A02, B4, 8); result4 = __riscv_vfmacc_vf_f64m2(result4, A03, B6, 8); - B00 = __riscv_vle64_v_f64m8(B, N * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); + B4 = __riscv_vle64_v_f64m2((B) + (N) * 2, N); + B6 = __riscv_vle64_v_f64m2((B) + (N) * 3, N); B += (N * 4); - B0 = __riscv_vget_v_f64m8_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m8_f64m2(B00, 1); - B4 = __riscv_vget_v_f64m8_f64m2(B00, 2); - B6 = __riscv_vget_v_f64m8_f64m2(B00, 3); result6 = __riscv_vfmacc_vf_f64m2(result6, A04, B0, 8); result8 = __riscv_vfmacc_vf_f64m2(result8, A05, B2, 8); @@ -177,12 +177,12 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON } } else if (M == 2) { if (K >= 4) { - vfloat64m8_t B00 = __riscv_vle64_v_f64m8(B, N * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); + B4 = __riscv_vle64_v_f64m2((B) + (N) * 2, N); + B6 = __riscv_vle64_v_f64m2((B) + (N) * 3, N); B += (N * 4); - B0 = __riscv_vget_v_f64m8_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m8_f64m2(B00, 1); - B4 = __riscv_vget_v_f64m8_f64m2(B00, 2); - B6 = __riscv_vget_v_f64m8_f64m2(B00, 3); #ifdef GEMM_NEW_PACKING A00 = A0[0 + (2 * 0)]; @@ -218,12 +218,12 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON BLASLONG k = (K / 4); K &= 3; while (--k) { - B00 = __riscv_vle64_v_f64m8(B, N * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); + B4 = __riscv_vle64_v_f64m2((B) + (N) * 2, N); + B6 = __riscv_vle64_v_f64m2((B) + (N) * 3, N); B += (N * 4); - B0 = __riscv_vget_v_f64m8_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m8_f64m2(B00, 1); - B4 = __riscv_vget_v_f64m8_f64m2(B00, 2); - B6 = __riscv_vget_v_f64m8_f64m2(B00, 3); #ifdef GEMM_NEW_PACKING A00 = A0[0 + (2 * 0)]; @@ -269,10 +269,10 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON } } else if (M <= 4) { if (K >= 2) { - vfloat64m4_t B00 = __riscv_vle64_v_f64m4(B, N * 2); + /* VLEN-portable: split load (avoid m4+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); B += (N * 2); - B0 = __riscv_vget_v_f64m4_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m4_f64m2(B00, 1); if (M & 4) { A00 = A0[0]; @@ -336,10 +336,10 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON BLASLONG k = (K / 2); K &= 1; while (--k) { - B00 = __riscv_vle64_v_f64m4(B, N * 2); + /* VLEN-portable: split load (avoid m4+vget) */ + B0 = __riscv_vle64_v_f64m2((B) + (N) * 0, N); + B2 = __riscv_vle64_v_f64m2((B) + (N) * 1, N); B += (N * 2); - B0 = __riscv_vget_v_f64m4_f64m2(B00, 0); - B2 = __riscv_vget_v_f64m4_f64m2(B00, 1); if (M & 4) { A00 = A0[0]; @@ -1051,12 +1051,12 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO if (N == 1) { if (K3) { - vfloat64m8_t A01 = __riscv_vle64_v_f64m8(*A, 8 * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (8) * 0, 8); + A2 = __riscv_vle64_v_f64m2((*A) + (8) * 1, 8); + A4 = __riscv_vle64_v_f64m2((*A) + (8) * 2, 8); + A6 = __riscv_vle64_v_f64m2((*A) + (8) * 3, 8); *A += (8 * 4); - A0 = __riscv_vget_v_f64m8_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m8_f64m2(A01, 1); - A4 = __riscv_vget_v_f64m8_f64m2(A01, 2); - A6 = __riscv_vget_v_f64m8_f64m2(A01, 3); #ifdef GEMM_NEW_PACKING B0 = B00[0]; @@ -1085,11 +1085,11 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO result2 = __riscv_vfmul_vf_f64m2(A4, B2, 8); result4 = __riscv_vfmul_vf_f64m2(A6, B3, 8); - A01 = __riscv_vle64_v_f64m8(*A, 8 * 4); - A0 = __riscv_vget_v_f64m8_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m8_f64m2(A01, 1); - A4 = __riscv_vget_v_f64m8_f64m2(A01, 2); - A6 = __riscv_vget_v_f64m8_f64m2(A01, 3); + /* VLEN-portable: split load (avoid m8+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (8) * 0, 8); + A2 = __riscv_vle64_v_f64m2((*A) + (8) * 1, 8); + A4 = __riscv_vle64_v_f64m2((*A) + (8) * 2, 8); + A6 = __riscv_vle64_v_f64m2((*A) + (8) * 3, 8); *A += (8 * 4); result6 = __riscv_vfmul_vf_f64m2(A0, B4, 8); @@ -1098,12 +1098,12 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO resultE = __riscv_vfmul_vf_f64m2(A6, B7, 8); for (BLASLONG k = K3; --k; ) { - A01 = __riscv_vle64_v_f64m8(*A, 8 * 4); + /* VLEN-portable: split load (avoid m8+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (8) * 0, 8); + A2 = __riscv_vle64_v_f64m2((*A) + (8) * 1, 8); + A4 = __riscv_vle64_v_f64m2((*A) + (8) * 2, 8); + A6 = __riscv_vle64_v_f64m2((*A) + (8) * 3, 8); *A += (8 * 4); - A0 = __riscv_vget_v_f64m8_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m8_f64m2(A01, 1); - A4 = __riscv_vget_v_f64m8_f64m2(A01, 2); - A6 = __riscv_vget_v_f64m8_f64m2(A01, 3); #ifdef GEMM_NEW_PACKING B0 = B00[0]; @@ -1132,11 +1132,11 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO result2 = __riscv_vfmacc_vf_f64m2(result2, B2, A4, 8); result4 = __riscv_vfmacc_vf_f64m2(result4, B3, A6, 8); - A01 = __riscv_vle64_v_f64m8(*A, 8 * 4); - A0 = __riscv_vget_v_f64m8_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m8_f64m2(A01, 1); - A4 = __riscv_vget_v_f64m8_f64m2(A01, 2); - A6 = __riscv_vget_v_f64m8_f64m2(A01, 3); + /* VLEN-portable: split load (avoid m8+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (8) * 0, 8); + A2 = __riscv_vle64_v_f64m2((*A) + (8) * 1, 8); + A4 = __riscv_vle64_v_f64m2((*A) + (8) * 2, 8); + A6 = __riscv_vle64_v_f64m2((*A) + (8) * 3, 8); *A += (8 * 4); result6 = __riscv_vfmacc_vf_f64m2(result6, B4, A0, 8); @@ -1157,11 +1157,11 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO } } else if (N == 2) { if (K3) { - vfloat64m8_t A01 = __riscv_vle64_v_f64m8(*A, 8 * 4); - A0 = __riscv_vget_v_f64m8_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m8_f64m2(A01, 1); - A4 = __riscv_vget_v_f64m8_f64m2(A01, 2); - A6 = __riscv_vget_v_f64m8_f64m2(A01, 3); + /* VLEN-portable: split load (avoid m8+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (8) * 0, 8); + A2 = __riscv_vle64_v_f64m2((*A) + (8) * 1, 8); + A4 = __riscv_vle64_v_f64m2((*A) + (8) * 2, 8); + A6 = __riscv_vle64_v_f64m2((*A) + (8) * 3, 8); *A += (8 * 4); #ifdef GEMM_NEW_PACKING @@ -1196,11 +1196,11 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO resultE = __riscv_vfmul_vf_f64m2(A6, B7, 8); for (BLASLONG k = K3; --k; ) { - A01 = __riscv_vle64_v_f64m8(*A, 8 * 4); - A0 = __riscv_vget_v_f64m8_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m8_f64m2(A01, 1); - A4 = __riscv_vget_v_f64m8_f64m2(A01, 2); - A6 = __riscv_vget_v_f64m8_f64m2(A01, 3); + /* VLEN-portable: split load (avoid m8+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (8) * 0, 8); + A2 = __riscv_vle64_v_f64m2((*A) + (8) * 1, 8); + A4 = __riscv_vle64_v_f64m2((*A) + (8) * 2, 8); + A6 = __riscv_vle64_v_f64m2((*A) + (8) * 3, 8); *A += (8 * 4); #ifdef GEMM_NEW_PACKING @@ -1247,9 +1247,9 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO } } else if (N <= 4) { if (K3) { - vfloat64m4_t A01 = __riscv_vle64_v_f64m4(*A, 4 * 4); - A0 = __riscv_vget_v_f64m4_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m4_f64m2(A01, 1); + /* VLEN-portable: split load (avoid m4+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (((4 * 4)/2)) * 0, ((4 * 4)/2)); + A2 = __riscv_vle64_v_f64m2((*A) + (((4 * 4)/2)) * 1, ((4 * 4)/2)); *A += (4 * 4); if (N == 4) { @@ -1312,9 +1312,9 @@ static void FORCEINLINE N_TAIL_ONE(BLASLONG K, BLASLONG M, const BLASLONG N, FLO } for (BLASLONG k = K3; --k; ) { - A01 = __riscv_vle64_v_f64m4(*A, 4 * 4); - A0 = __riscv_vget_v_f64m4_f64m2(A01, 0); - A2 = __riscv_vget_v_f64m4_f64m2(A01, 1); + /* VLEN-portable: split load (avoid m4+vget) */ + A0 = __riscv_vle64_v_f64m2((*A) + (((4 * 4)/2)) * 0, ((4 * 4)/2)); + A2 = __riscv_vle64_v_f64m2((*A) + (((4 * 4)/2)) * 1, ((4 * 4)/2)); *A += (4 * 4); if (N == 4) { diff --git a/kernel/riscv64/dtrmm_kernel_16x8_zvl1024b.c b/kernel/riscv64/dtrmm_kernel_16x8_zvl1024b.c new file mode 100644 index 0000000000..0c82bbe09d --- /dev/null +++ b/kernel/riscv64/dtrmm_kernel_16x8_zvl1024b.c @@ -0,0 +1,1262 @@ +/* + +AUTOGENERATED KERNEL +Settings: + LMUL=1 + M=16 + M_tail_scalar_from=2 + N=8 + __riscv_='__riscv_' + complex=False + conjugate=False + cpu='zvl1024b' + force_acc_double=False + index_type='BLASLONG' + op='trmm' + param_precision='double' + reg_width_bits=1024 + tail_policy='' + trace=False + +Derived: + ELEN_ACC=64 + ELEN_PARAM=64 + LMUL_ACC=1 + VFMACC='__riscv_vfmacc_vf_f64m1' + VFMUL='__riscv_vfmul_vf_f64m1' + VLEV='__riscv_vle64_v_f64m1' + VLSEV='__riscv_vlse64_v_f64m1' + VMACC_TO_ACC='__riscv_vfmacc_vf_f64m1' + VMUL_TO_ACC='__riscv_vfmul_vf_f64m1' + VSETVL='__riscv_vsetvl_e64m1' + VSEV='__riscv_vse64_v_f64m1' + VSSEV='__riscv_vsse64_v_f64m1' + acc_vector_t='vfloat64m1_t' + output='dtrmm_kernel_16x8_zvl1024b.c' + param_scalar_t='double' + param_vector_t='vfloat64m1_t' + +*/ + +#include "common.h" + + + +#if defined(LEFT) != defined(TRANSA) + #define BACKWARDS +#endif + +int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc, BLASLONG offset) + +{ + BLASLONG gvl = 0; + BLASLONG m_top = 0; + BLASLONG n_top = 0; + + + // -- MAIN PASS + + for (BLASLONG j=0; j= 8) { - vfloat32m8_t B00 = __riscv_vle32_v_f32m8(B, N * 8); - B0 = __riscv_vget_v_f32m8_f32m1(B00, 0); - B1 = __riscv_vget_v_f32m8_f32m1(B00, 1); - B2 = __riscv_vget_v_f32m8_f32m1(B00, 2); - B3 = __riscv_vget_v_f32m8_f32m1(B00, 3); - B4 = __riscv_vget_v_f32m8_f32m1(B00, 4); - B5 = __riscv_vget_v_f32m8_f32m1(B00, 5); - B6 = __riscv_vget_v_f32m8_f32m1(B00, 6); - B7 = __riscv_vget_v_f32m8_f32m1(B00, 7); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle32_v_f32m1((B) + (N) * 0, N); + B1 = __riscv_vle32_v_f32m1((B) + (N) * 1, N); + B2 = __riscv_vle32_v_f32m1((B) + (N) * 2, N); + B3 = __riscv_vle32_v_f32m1((B) + (N) * 3, N); + B4 = __riscv_vle32_v_f32m1((B) + (N) * 4, N); + B5 = __riscv_vle32_v_f32m1((B) + (N) * 5, N); + B6 = __riscv_vle32_v_f32m1((B) + (N) * 6, N); + B7 = __riscv_vle32_v_f32m1((B) + (N) * 7, N); B += (N * 8); #ifdef GEMM_NEW_PACKING @@ -144,15 +144,15 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON BLASLONG k = (K / 8); K &= 7; while (--k) { - B00 = __riscv_vle32_v_f32m8(B, N * 8); - B0 = __riscv_vget_v_f32m8_f32m1(B00, 0); - B1 = __riscv_vget_v_f32m8_f32m1(B00, 1); - B2 = __riscv_vget_v_f32m8_f32m1(B00, 2); - B3 = __riscv_vget_v_f32m8_f32m1(B00, 3); - B4 = __riscv_vget_v_f32m8_f32m1(B00, 4); - B5 = __riscv_vget_v_f32m8_f32m1(B00, 5); - B6 = __riscv_vget_v_f32m8_f32m1(B00, 6); - B7 = __riscv_vget_v_f32m8_f32m1(B00, 7); + /* VLEN-portable: split load (avoid m8+vget) */ + B0 = __riscv_vle32_v_f32m1((B) + (N) * 0, N); + B1 = __riscv_vle32_v_f32m1((B) + (N) * 1, N); + B2 = __riscv_vle32_v_f32m1((B) + (N) * 2, N); + B3 = __riscv_vle32_v_f32m1((B) + (N) * 3, N); + B4 = __riscv_vle32_v_f32m1((B) + (N) * 4, N); + B5 = __riscv_vle32_v_f32m1((B) + (N) * 5, N); + B6 = __riscv_vle32_v_f32m1((B) + (N) * 6, N); + B7 = __riscv_vle32_v_f32m1((B) + (N) * 7, N); B += (N * 8); #ifdef GEMM_NEW_PACKING @@ -251,11 +251,11 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON } } else if (M == 3) { if (K >= 4) { - vfloat32m4_t B00 = __riscv_vle32_v_f32m4(B, N * 4); - B0 = __riscv_vget_v_f32m4_f32m1(B00, 0); - B1 = __riscv_vget_v_f32m4_f32m1(B00, 1); - B2 = __riscv_vget_v_f32m4_f32m1(B00, 2); - B3 = __riscv_vget_v_f32m4_f32m1(B00, 3); + /* VLEN-portable: split load (avoid m4+vget) */ + B0 = __riscv_vle32_v_f32m1((B) + (N) * 0, N); + B1 = __riscv_vle32_v_f32m1((B) + (N) * 1, N); + B2 = __riscv_vle32_v_f32m1((B) + (N) * 2, N); + B3 = __riscv_vle32_v_f32m1((B) + (N) * 3, N); B += (N * 4); #ifdef GEMM_NEW_PACKING @@ -294,11 +294,11 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON BLASLONG k = (K / 4); K &= 3; while (--k) { - B00 = __riscv_vle32_v_f32m4(B, N * 4); - B0 = __riscv_vget_v_f32m4_f32m1(B00, 0); - B1 = __riscv_vget_v_f32m4_f32m1(B00, 1); - B2 = __riscv_vget_v_f32m4_f32m1(B00, 2); - B3 = __riscv_vget_v_f32m4_f32m1(B00, 3); + /* VLEN-portable: split load (avoid m4+vget) */ + B0 = __riscv_vle32_v_f32m1((B) + (N) * 0, N); + B1 = __riscv_vle32_v_f32m1((B) + (N) * 1, N); + B2 = __riscv_vle32_v_f32m1((B) + (N) * 2, N); + B3 = __riscv_vle32_v_f32m1((B) + (N) * 3, N); B += (N * 4); #ifdef GEMM_NEW_PACKING @@ -357,16 +357,16 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON vfloat32m1_t A5; if (!S2) { - B00 = __riscv_vle32_v_f32m2(B, N * 2); - B0 = __riscv_vget_v_f32m2_f32m1(B00, 0); - B1 = __riscv_vget_v_f32m2_f32m1(B00, 1); + /* VLEN-portable: split load (avoid m2+vget) */ + B0 = __riscv_vle32_v_f32m1((B) + (N) * 0, N); + B1 = __riscv_vle32_v_f32m1((B) + (N) * 1, N); } if (M == 8) { if (S2) { - A00 = __riscv_vle32_v_f32m2(A0, N * 2); - A4 = __riscv_vget_v_f32m2_f32m1(A00, 0); - A5 = __riscv_vget_v_f32m2_f32m1(A00, 1); + /* VLEN-portable: split load (avoid m2+vget) */ + A4 = __riscv_vle32_v_f32m1((A0) + (N) * 0, N); + A5 = __riscv_vle32_v_f32m1((A0) + (N) * 1, N); result0 = __riscv_vfmul_vf_f32m1(A4, B[0], N); result1 = __riscv_vfmul_vf_f32m1(A4, B[1], N); @@ -459,16 +459,16 @@ static FORCEINLINE FLOAT* M_TAIL_ONE(BLASLONG K, const BLASLONG M, const BLASLON K &= 1; while (--k) { if (!S2) { - B00 = __riscv_vle32_v_f32m2(B, N * 2); - B0 = __riscv_vget_v_f32m2_f32m1(B00, 0); - B1 = __riscv_vget_v_f32m2_f32m1(B00, 1); + /* VLEN-portable: split load (avoid m2+vget) */ + B0 = __riscv_vle32_v_f32m1((B) + (N) * 0, N); + B1 = __riscv_vle32_v_f32m1((B) + (N) * 1, N); } if (M == 8) { if (S2) { - A00 = __riscv_vle32_v_f32m2(A0, N * 2); - A4 = __riscv_vget_v_f32m2_f32m1(A00, 0); - A5 = __riscv_vget_v_f32m2_f32m1(A00, 1); + /* VLEN-portable: split load (avoid m2+vget) */ + A4 = __riscv_vle32_v_f32m1((A0) + (N) * 0, N); + A5 = __riscv_vle32_v_f32m1((A0) + (N) * 1, N); result0 = __riscv_vfmacc_vf_f32m1(result0, B[0], A4, N); result1 = __riscv_vfmacc_vf_f32m1(result1, B[1], A4, N); diff --git a/kernel/riscv64/strmm_kernel_16x8_zvl1024b.c b/kernel/riscv64/strmm_kernel_16x8_zvl1024b.c new file mode 100644 index 0000000000..af431f1301 --- /dev/null +++ b/kernel/riscv64/strmm_kernel_16x8_zvl1024b.c @@ -0,0 +1,1262 @@ +/* + +AUTOGENERATED KERNEL +Settings: + LMUL=1 + M=16 + M_tail_scalar_from=2 + N=8 + __riscv_='__riscv_' + complex=False + conjugate=False + cpu='zvl1024b' + force_acc_double=False + index_type='BLASLONG' + op='trmm' + param_precision='float' + reg_width_bits=1024 + tail_policy='' + trace=False + +Derived: + ELEN_ACC=32 + ELEN_PARAM=32 + LMUL_ACC=1 + VFMACC='__riscv_vfmacc_vf_f32m1' + VFMUL='__riscv_vfmul_vf_f32m1' + VLEV='__riscv_vle32_v_f32m1' + VLSEV='__riscv_vlse32_v_f32m1' + VMACC_TO_ACC='__riscv_vfmacc_vf_f32m1' + VMUL_TO_ACC='__riscv_vfmul_vf_f32m1' + VSETVL='__riscv_vsetvl_e32m1' + VSEV='__riscv_vse32_v_f32m1' + VSSEV='__riscv_vsse32_v_f32m1' + acc_vector_t='vfloat32m1_t' + output='strmm_kernel_16x8_zvl1024b.c' + param_scalar_t='float' + param_vector_t='vfloat32m1_t' + +*/ + +#include "common.h" + + + +#if defined(LEFT) != defined(TRANSA) + #define BACKWARDS +#endif + +int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc, BLASLONG offset) + +{ + BLASLONG gvl = 0; + BLASLONG m_top = 0; + BLASLONG n_top = 0; + + + // -- MAIN PASS + + for (BLASLONG j=0; j Date: Mon, 28 Sep 2026 14:44:23 +0200 Subject: [PATCH 2/2] Switch ZVL1024B GEMM to bitsliced MR=32/64 kernels Use dgemm 32x8 / sgemm 64x8 with power-of-two packers matching bitsliced A-panel tails; add TRSM MR=32/64 copies and SHIFT wiring. Default param MR stays 32/64; 16x8 remains available for A/B. Co-authored-by: Cursor --- kernel/generic/neg_tcopy_32.c | 26 + kernel/generic/neg_tcopy_64.c | 26 + kernel/generic/trsm_kernel_LN.c | 8 + kernel/generic/trsm_kernel_LT.c | 8 + kernel/generic/trsm_kernel_RN.c | 8 + kernel/generic/trsm_kernel_RT.c | 8 + kernel/generic/trsm_lncopy_32.c | 273 +++ kernel/generic/trsm_lncopy_64.c | 273 +++ kernel/generic/trsm_ltcopy_32.c | 230 +++ kernel/generic/trsm_ltcopy_64.c | 230 +++ kernel/generic/trsm_uncopy_32.c | 358 ++++ kernel/generic/trsm_uncopy_64.c | 574 ++++++ kernel/generic/trsm_utcopy_32.c | 227 +++ kernel/generic/trsm_utcopy_64.c | 227 +++ kernel/riscv64/KERNEL.RISCV64_ZVL1024B | 28 +- kernel/riscv64/dgemm_kernel_32x8_zvl1024b.c | 1303 +++++++++++++ kernel/riscv64/dtrmm_kernel_32x8_zvl1024b.c | 1593 ++++++++++++++++ kernel/riscv64/gemm_ncopy_32_rvv.c | 61 + kernel/riscv64/gemm_ncopy_64_rvv.c | 60 + kernel/riscv64/gemm_ncopy_contig_rvv.c | 37 + kernel/riscv64/gemm_tcopy_32_rvv.c | 62 + kernel/riscv64/gemm_tcopy_64_rvv.c | 60 + kernel/riscv64/gemm_tcopy_contig_rvv.c | 44 + kernel/riscv64/generate_kernel.py | 61 +- kernel/riscv64/sgemm_kernel_64x8_zvl1024b.c | 1524 +++++++++++++++ kernel/riscv64/strmm_kernel_64x8_zvl1024b.c | 1855 +++++++++++++++++++ kernel/riscv64/symm_lcopy_32_rvv.c | 75 + kernel/riscv64/symm_lcopy_64_rvv.c | 75 + kernel/riscv64/symm_ucopy_32_rvv.c | 75 + kernel/riscv64/symm_ucopy_64_rvv.c | 75 + kernel/riscv64/trmm_lncopy_32_rvv.c | 142 ++ kernel/riscv64/trmm_lncopy_64_rvv.c | 142 ++ kernel/riscv64/trmm_ltcopy_32_rvv.c | 138 ++ kernel/riscv64/trmm_ltcopy_64_rvv.c | 138 ++ kernel/riscv64/trmm_uncopy_32_rvv.c | 140 ++ kernel/riscv64/trmm_uncopy_64_rvv.c | 140 ++ kernel/riscv64/trmm_utcopy_32_rvv.c | 138 ++ kernel/riscv64/trmm_utcopy_64_rvv.c | 138 ++ param.h | 4 +- 39 files changed, 10567 insertions(+), 17 deletions(-) create mode 100644 kernel/generic/neg_tcopy_32.c create mode 100644 kernel/generic/neg_tcopy_64.c create mode 100644 kernel/generic/trsm_lncopy_32.c create mode 100644 kernel/generic/trsm_lncopy_64.c create mode 100644 kernel/generic/trsm_ltcopy_32.c create mode 100644 kernel/generic/trsm_ltcopy_64.c create mode 100644 kernel/generic/trsm_uncopy_32.c create mode 100644 kernel/generic/trsm_uncopy_64.c create mode 100644 kernel/generic/trsm_utcopy_32.c create mode 100644 kernel/generic/trsm_utcopy_64.c create mode 100644 kernel/riscv64/dgemm_kernel_32x8_zvl1024b.c create mode 100644 kernel/riscv64/dtrmm_kernel_32x8_zvl1024b.c create mode 100644 kernel/riscv64/gemm_ncopy_32_rvv.c create mode 100644 kernel/riscv64/gemm_ncopy_64_rvv.c create mode 100644 kernel/riscv64/gemm_ncopy_contig_rvv.c create mode 100644 kernel/riscv64/gemm_tcopy_32_rvv.c create mode 100644 kernel/riscv64/gemm_tcopy_64_rvv.c create mode 100644 kernel/riscv64/gemm_tcopy_contig_rvv.c create mode 100644 kernel/riscv64/sgemm_kernel_64x8_zvl1024b.c create mode 100644 kernel/riscv64/strmm_kernel_64x8_zvl1024b.c create mode 100644 kernel/riscv64/symm_lcopy_32_rvv.c create mode 100644 kernel/riscv64/symm_lcopy_64_rvv.c create mode 100644 kernel/riscv64/symm_ucopy_32_rvv.c create mode 100644 kernel/riscv64/symm_ucopy_64_rvv.c create mode 100644 kernel/riscv64/trmm_lncopy_32_rvv.c create mode 100644 kernel/riscv64/trmm_lncopy_64_rvv.c create mode 100644 kernel/riscv64/trmm_ltcopy_32_rvv.c create mode 100644 kernel/riscv64/trmm_ltcopy_64_rvv.c create mode 100644 kernel/riscv64/trmm_uncopy_32_rvv.c create mode 100644 kernel/riscv64/trmm_uncopy_64_rvv.c create mode 100644 kernel/riscv64/trmm_utcopy_32_rvv.c create mode 100644 kernel/riscv64/trmm_utcopy_64_rvv.c diff --git a/kernel/generic/neg_tcopy_32.c b/kernel/generic/neg_tcopy_32.c new file mode 100644 index 0000000000..ea92dc6cb9 --- /dev/null +++ b/kernel/generic/neg_tcopy_32.c @@ -0,0 +1,26 @@ +/*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ +/* Bitsliced neg_tcopy MR=32 — panels 32,16,8,4,2,1. */ +/*********************************************************************/ +#include "common.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){ + FLOAT *boffset = b; + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + BLASLONG col = 0; + int wi; + + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + BLASLONG k, r; + for (k = 0; k < m; k++) { + for (r = 0; r < w; r++) + *boffset++ = -a[(col + r) + k * lda]; + } + col += w; + n -= w; + } + } + return 0; +} diff --git a/kernel/generic/neg_tcopy_64.c b/kernel/generic/neg_tcopy_64.c new file mode 100644 index 0000000000..e1cad5f807 --- /dev/null +++ b/kernel/generic/neg_tcopy_64.c @@ -0,0 +1,26 @@ +/*********************************************************************/ +/* Copyright 2025 The OpenBLAS Project. */ +/* Bitsliced neg_tcopy MR=64 — panels 64,32,16,8,4,2,1. */ +/*********************************************************************/ +#include "common.h" + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b){ + FLOAT *boffset = b; + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + BLASLONG col = 0; + int wi; + + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + BLASLONG k, r; + for (k = 0; k < m; k++) { + for (r = 0; r < w; r++) + *boffset++ = -a[(col + r) + k * lda]; + } + col += w; + n -= w; + } + } + return 0; +} diff --git a/kernel/generic/trsm_kernel_LN.c b/kernel/generic/trsm_kernel_LN.c index c08f86e514..e5de1e53f8 100644 --- a/kernel/generic/trsm_kernel_LN.c +++ b/kernel/generic/trsm_kernel_LN.c @@ -70,6 +70,14 @@ static FLOAT dm1 = -1.; #define GEMM_UNROLL_M_SHIFT 4 #endif +#if GEMM_DEFAULT_UNROLL_M == 32 +#define GEMM_UNROLL_M_SHIFT 5 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 64 +#define GEMM_UNROLL_M_SHIFT 6 +#endif + #if GEMM_DEFAULT_UNROLL_N == 1 #define GEMM_UNROLL_N_SHIFT 0 #endif diff --git a/kernel/generic/trsm_kernel_LT.c b/kernel/generic/trsm_kernel_LT.c index 07b33467ea..7162a3659a 100644 --- a/kernel/generic/trsm_kernel_LT.c +++ b/kernel/generic/trsm_kernel_LT.c @@ -70,6 +70,14 @@ static FLOAT dm1 = -1.; #define GEMM_UNROLL_M_SHIFT 4 #endif +#if GEMM_DEFAULT_UNROLL_M == 32 +#define GEMM_UNROLL_M_SHIFT 5 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 64 +#define GEMM_UNROLL_M_SHIFT 6 +#endif + #if GEMM_DEFAULT_UNROLL_N == 1 #define GEMM_UNROLL_N_SHIFT 0 #endif diff --git a/kernel/generic/trsm_kernel_RN.c b/kernel/generic/trsm_kernel_RN.c index 07a4f3b406..e5b1c6cd3d 100644 --- a/kernel/generic/trsm_kernel_RN.c +++ b/kernel/generic/trsm_kernel_RN.c @@ -70,6 +70,14 @@ static FLOAT dm1 = -1.; #define GEMM_UNROLL_M_SHIFT 4 #endif +#if GEMM_DEFAULT_UNROLL_M == 32 +#define GEMM_UNROLL_M_SHIFT 5 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 64 +#define GEMM_UNROLL_M_SHIFT 6 +#endif + #if GEMM_DEFAULT_UNROLL_N == 1 #define GEMM_UNROLL_N_SHIFT 0 #endif diff --git a/kernel/generic/trsm_kernel_RT.c b/kernel/generic/trsm_kernel_RT.c index 0c4db335cf..295b7a887e 100644 --- a/kernel/generic/trsm_kernel_RT.c +++ b/kernel/generic/trsm_kernel_RT.c @@ -71,6 +71,14 @@ static FLOAT dm1 = -1.; #define GEMM_UNROLL_M_SHIFT 4 #endif +#if GEMM_DEFAULT_UNROLL_M == 32 +#define GEMM_UNROLL_M_SHIFT 5 +#endif + +#if GEMM_DEFAULT_UNROLL_M == 64 +#define GEMM_UNROLL_M_SHIFT 6 +#endif + #if GEMM_DEFAULT_UNROLL_N == 1 #define GEMM_UNROLL_N_SHIFT 0 #endif diff --git a/kernel/generic/trsm_lncopy_32.c b/kernel/generic/trsm_lncopy_32.c new file mode 100644 index 0000000000..4a0e8c35fe --- /dev/null +++ b/kernel/generic/trsm_lncopy_32.c @@ -0,0 +1,273 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" +/* NOTE: based on lncopy_16; MR=32 main panel not yet specialized. + * Correct for n < 32 (covers BLAT3). Full MR=32 panel TBD. */ + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8; + FLOAT *a9, *a10, *a11, *a12, *a13, *a14, *a15, *a16; + + + jj = offset; + + j = (n >> 4); + while (j > 0){ + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + + a += 16 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 16) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + b += 16; + ii ++; + } + + jj += 16; + j --; + } + + if (n & 8) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a += 8 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 8) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + b += 8; + ii ++; + } + + jj += 8; + } + + if (n & 4) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a += 4 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 4) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + b += 4; + ii ++; + } + + jj += 4; + } + + if (n & 2) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a += 2 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 2) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + } + + a1 ++; + a2 ++; + b += 2; + ii ++; + } + + jj += 2; + } + + if (n & 1) { + + a1 = a + 0 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 1) { + *(b + 0) = *(a1 + 0); + } + + a1 ++; + b += 1; + ii ++; + } + } + + return 0; +} diff --git a/kernel/generic/trsm_lncopy_64.c b/kernel/generic/trsm_lncopy_64.c new file mode 100644 index 0000000000..d842601836 --- /dev/null +++ b/kernel/generic/trsm_lncopy_64.c @@ -0,0 +1,273 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include "common.h" +/* NOTE: based on lncopy_16; MR=64 main panel not yet specialized. + * Correct for n < 32 (covers BLAT3). Full MR=64 panel TBD. */ + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8; + FLOAT *a9, *a10, *a11, *a12, *a13, *a14, *a15, *a16; + + + jj = offset; + + j = (n >> 4); + while (j > 0){ + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + + a += 16 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 16) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + b += 16; + ii ++; + } + + jj += 16; + j --; + } + + if (n & 8) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a += 8 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 8) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + b += 8; + ii ++; + } + + jj += 8; + } + + if (n & 4) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a += 4 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 4) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + b += 4; + ii ++; + } + + jj += 4; + } + + if (n & 2) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a += 2 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 2) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + } + + a1 ++; + a2 ++; + b += 2; + ii ++; + } + + jj += 2; + } + + if (n & 1) { + + a1 = a + 0 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k * lda); + } + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + } + + if (ii - jj >= 1) { + *(b + 0) = *(a1 + 0); + } + + a1 ++; + b += 1; + ii ++; + } + } + + return 0; +} diff --git a/kernel/generic/trsm_ltcopy_32.c b/kernel/generic/trsm_ltcopy_32.c new file mode 100644 index 0000000000..f04b2c11f0 --- /dev/null +++ b/kernel/generic/trsm_ltcopy_32.c @@ -0,0 +1,230 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" +/* NOTE: based on ltcopy_16; MR=32 main panel not yet specialized. + * Correct for n < 32 (covers BLAT3). Full MR=32 panel TBD. */ + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1; + + jj = offset; + + j = (n >> 4); + while (j > 0){ + + a1 = a; + a += 16; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 16; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + *(b + 8) = *(a1 + 8); + *(b + 9) = *(a1 + 9); + *(b + 10) = *(a1 + 10); + *(b + 11) = *(a1 + 11); + *(b + 12) = *(a1 + 12); + *(b + 13) = *(a1 + 13); + *(b + 14) = *(a1 + 14); + *(b + 15) = *(a1 + 15); + } + + b += 16; + a1 += lda; + ii ++; + } + + jj += 16; + j --; + } + + j = (n & 8); + if (j > 0) { + a1 = a; + a += 8; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 8; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + } + + b += 8; + a1 += lda; + ii ++; + } + + jj += 8; + } + + j = (n & 4); + if (j > 0) { + + a1 = a; + a += 4; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 4; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + } + + b += 4; + a1 += lda; + ii ++; + } + + jj += 4; + } + + j = (n & 2); + if (j > 0) { + + a1 = a; + a += 2; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 2; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + } + + b += 2; + a1 += lda; + ii ++; + } + + jj += 2; + } + + j = (n & 1); + if (j > 0) { + + a1 = a; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + } + + b += 1; + a1 += lda; + ii ++; + } + } + + return 0; +} diff --git a/kernel/generic/trsm_ltcopy_64.c b/kernel/generic/trsm_ltcopy_64.c new file mode 100644 index 0000000000..584b971321 --- /dev/null +++ b/kernel/generic/trsm_ltcopy_64.c @@ -0,0 +1,230 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" +/* NOTE: based on ltcopy_16; MR=64 main panel not yet specialized. + * Correct for n < 32 (covers BLAT3). Full MR=64 panel TBD. */ + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1; + + jj = offset; + + j = (n >> 4); + while (j > 0){ + + a1 = a; + a += 16; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 16; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + *(b + 8) = *(a1 + 8); + *(b + 9) = *(a1 + 9); + *(b + 10) = *(a1 + 10); + *(b + 11) = *(a1 + 11); + *(b + 12) = *(a1 + 12); + *(b + 13) = *(a1 + 13); + *(b + 14) = *(a1 + 14); + *(b + 15) = *(a1 + 15); + } + + b += 16; + a1 += lda; + ii ++; + } + + jj += 16; + j --; + } + + j = (n & 8); + if (j > 0) { + a1 = a; + a += 8; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 8; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + } + + b += 8; + a1 += lda; + ii ++; + } + + jj += 8; + } + + j = (n & 4); + if (j > 0) { + + a1 = a; + a += 4; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 4; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + } + + b += 4; + a1 += lda; + ii ++; + } + + jj += 4; + } + + j = (n & 2); + if (j > 0) { + + a1 = a; + a += 2; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + + for (k = ii - jj + 1; k < 2; k ++) { + *(b + k) = *(a1 + k); + } + + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + } + + b += 2; + a1 += lda; + ii ++; + } + + jj += 2; + } + + j = (n & 1); + if (j > 0) { + + a1 = a; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + } + + b += 1; + a1 += lda; + ii ++; + } + } + + return 0; +} diff --git a/kernel/generic/trsm_uncopy_32.c b/kernel/generic/trsm_uncopy_32.c new file mode 100644 index 0000000000..2d88852d07 --- /dev/null +++ b/kernel/generic/trsm_uncopy_32.c @@ -0,0 +1,358 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/*********************************************************************/ + +#include "common.h" + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, *a9, *a10, *a11, *a12, *a13, *a14, *a15, *a16, *a17, *a18, *a19, *a20, *a21, *a22, *a23, *a24, *a25, *a26, *a27, *a28, *a29, *a30, *a31, *a32; + + jj = offset; + + j = (n >> 5); + while (j > 0){ + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + a17 = a + 16 * lda; + a18 = a + 17 * lda; + a19 = a + 18 * lda; + a20 = a + 19 * lda; + a21 = a + 20 * lda; + a22 = a + 21 * lda; + a23 = a + 22 * lda; + a24 = a + 23 * lda; + a25 = a + 24 * lda; + a26 = a + 25 * lda; + a27 = a + 26 * lda; + a28 = a + 27 * lda; + a29 = a + 28 * lda; + a30 = a + 29 * lda; + a31 = a + 30 * lda; + a32 = a + 31 * lda; + a += 32 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 32)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 32; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + *(b + 16) = *(a17 + 0); + *(b + 17) = *(a18 + 0); + *(b + 18) = *(a19 + 0); + *(b + 19) = *(a20 + 0); + *(b + 20) = *(a21 + 0); + *(b + 21) = *(a22 + 0); + *(b + 22) = *(a23 + 0); + *(b + 23) = *(a24 + 0); + *(b + 24) = *(a25 + 0); + *(b + 25) = *(a26 + 0); + *(b + 26) = *(a27 + 0); + *(b + 27) = *(a28 + 0); + *(b + 28) = *(a29 + 0); + *(b + 29) = *(a30 + 0); + *(b + 30) = *(a31 + 0); + *(b + 31) = *(a32 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + a17 ++; + a18 ++; + a19 ++; + a20 ++; + a21 ++; + a22 ++; + a23 ++; + a24 ++; + a25 ++; + a26 ++; + a27 ++; + a28 ++; + a29 ++; + a30 ++; + a31 ++; + a32 ++; + b += 32; + ii ++; + } + + jj += 32; + j --; + } + + if (n & 16) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + a += 16 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 16; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + b += 16; + ii ++; + } + + jj += 16; + } + + if (n & 8) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a += 8 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 8; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + b += 8; + ii ++; + } + + jj += 8; + } + + if (n & 4) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a += 4 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 4; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + b += 4; + ii ++; + } + + jj += 4; + } + + if (n & 2) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a += 2 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 2; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + } + + a1 ++; + a2 ++; + b += 2; + ii ++; + } + + jj += 2; + } + + if (n & 1) { + + a1 = a + 0 * lda; + a += 1 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 1; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + } + + a1 ++; + b += 1; + ii ++; + } + + jj += 1; + } + + return 0; +} diff --git a/kernel/generic/trsm_uncopy_64.c b/kernel/generic/trsm_uncopy_64.c new file mode 100644 index 0000000000..ff4a9173a8 --- /dev/null +++ b/kernel/generic/trsm_uncopy_64.c @@ -0,0 +1,574 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* Copyright 2025 The OpenBLAS Project. */ +/*********************************************************************/ + +#include "common.h" + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1, *a2, *a3, *a4, *a5, *a6, *a7, *a8, *a9, *a10, *a11, *a12, *a13, *a14, *a15, *a16, *a17, *a18, *a19, *a20, *a21, *a22, *a23, *a24, *a25, *a26, *a27, *a28, *a29, *a30, *a31, *a32, *a33, *a34, *a35, *a36, *a37, *a38, *a39, *a40, *a41, *a42, *a43, *a44, *a45, *a46, *a47, *a48, *a49, *a50, *a51, *a52, *a53, *a54, *a55, *a56, *a57, *a58, *a59, *a60, *a61, *a62, *a63, *a64; + + jj = offset; + + j = (n >> 6); + while (j > 0){ + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + a17 = a + 16 * lda; + a18 = a + 17 * lda; + a19 = a + 18 * lda; + a20 = a + 19 * lda; + a21 = a + 20 * lda; + a22 = a + 21 * lda; + a23 = a + 22 * lda; + a24 = a + 23 * lda; + a25 = a + 24 * lda; + a26 = a + 25 * lda; + a27 = a + 26 * lda; + a28 = a + 27 * lda; + a29 = a + 28 * lda; + a30 = a + 29 * lda; + a31 = a + 30 * lda; + a32 = a + 31 * lda; + a33 = a + 32 * lda; + a34 = a + 33 * lda; + a35 = a + 34 * lda; + a36 = a + 35 * lda; + a37 = a + 36 * lda; + a38 = a + 37 * lda; + a39 = a + 38 * lda; + a40 = a + 39 * lda; + a41 = a + 40 * lda; + a42 = a + 41 * lda; + a43 = a + 42 * lda; + a44 = a + 43 * lda; + a45 = a + 44 * lda; + a46 = a + 45 * lda; + a47 = a + 46 * lda; + a48 = a + 47 * lda; + a49 = a + 48 * lda; + a50 = a + 49 * lda; + a51 = a + 50 * lda; + a52 = a + 51 * lda; + a53 = a + 52 * lda; + a54 = a + 53 * lda; + a55 = a + 54 * lda; + a56 = a + 55 * lda; + a57 = a + 56 * lda; + a58 = a + 57 * lda; + a59 = a + 58 * lda; + a60 = a + 59 * lda; + a61 = a + 60 * lda; + a62 = a + 61 * lda; + a63 = a + 62 * lda; + a64 = a + 63 * lda; + a += 64 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 64)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 64; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + *(b + 16) = *(a17 + 0); + *(b + 17) = *(a18 + 0); + *(b + 18) = *(a19 + 0); + *(b + 19) = *(a20 + 0); + *(b + 20) = *(a21 + 0); + *(b + 21) = *(a22 + 0); + *(b + 22) = *(a23 + 0); + *(b + 23) = *(a24 + 0); + *(b + 24) = *(a25 + 0); + *(b + 25) = *(a26 + 0); + *(b + 26) = *(a27 + 0); + *(b + 27) = *(a28 + 0); + *(b + 28) = *(a29 + 0); + *(b + 29) = *(a30 + 0); + *(b + 30) = *(a31 + 0); + *(b + 31) = *(a32 + 0); + *(b + 32) = *(a33 + 0); + *(b + 33) = *(a34 + 0); + *(b + 34) = *(a35 + 0); + *(b + 35) = *(a36 + 0); + *(b + 36) = *(a37 + 0); + *(b + 37) = *(a38 + 0); + *(b + 38) = *(a39 + 0); + *(b + 39) = *(a40 + 0); + *(b + 40) = *(a41 + 0); + *(b + 41) = *(a42 + 0); + *(b + 42) = *(a43 + 0); + *(b + 43) = *(a44 + 0); + *(b + 44) = *(a45 + 0); + *(b + 45) = *(a46 + 0); + *(b + 46) = *(a47 + 0); + *(b + 47) = *(a48 + 0); + *(b + 48) = *(a49 + 0); + *(b + 49) = *(a50 + 0); + *(b + 50) = *(a51 + 0); + *(b + 51) = *(a52 + 0); + *(b + 52) = *(a53 + 0); + *(b + 53) = *(a54 + 0); + *(b + 54) = *(a55 + 0); + *(b + 55) = *(a56 + 0); + *(b + 56) = *(a57 + 0); + *(b + 57) = *(a58 + 0); + *(b + 58) = *(a59 + 0); + *(b + 59) = *(a60 + 0); + *(b + 60) = *(a61 + 0); + *(b + 61) = *(a62 + 0); + *(b + 62) = *(a63 + 0); + *(b + 63) = *(a64 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + a17 ++; + a18 ++; + a19 ++; + a20 ++; + a21 ++; + a22 ++; + a23 ++; + a24 ++; + a25 ++; + a26 ++; + a27 ++; + a28 ++; + a29 ++; + a30 ++; + a31 ++; + a32 ++; + a33 ++; + a34 ++; + a35 ++; + a36 ++; + a37 ++; + a38 ++; + a39 ++; + a40 ++; + a41 ++; + a42 ++; + a43 ++; + a44 ++; + a45 ++; + a46 ++; + a47 ++; + a48 ++; + a49 ++; + a50 ++; + a51 ++; + a52 ++; + a53 ++; + a54 ++; + a55 ++; + a56 ++; + a57 ++; + a58 ++; + a59 ++; + a60 ++; + a61 ++; + a62 ++; + a63 ++; + a64 ++; + b += 64; + ii ++; + } + + jj += 64; + j --; + } + + if (n & 32) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + a17 = a + 16 * lda; + a18 = a + 17 * lda; + a19 = a + 18 * lda; + a20 = a + 19 * lda; + a21 = a + 20 * lda; + a22 = a + 21 * lda; + a23 = a + 22 * lda; + a24 = a + 23 * lda; + a25 = a + 24 * lda; + a26 = a + 25 * lda; + a27 = a + 26 * lda; + a28 = a + 27 * lda; + a29 = a + 28 * lda; + a30 = a + 29 * lda; + a31 = a + 30 * lda; + a32 = a + 31 * lda; + a += 32 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 32)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 32; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + *(b + 16) = *(a17 + 0); + *(b + 17) = *(a18 + 0); + *(b + 18) = *(a19 + 0); + *(b + 19) = *(a20 + 0); + *(b + 20) = *(a21 + 0); + *(b + 21) = *(a22 + 0); + *(b + 22) = *(a23 + 0); + *(b + 23) = *(a24 + 0); + *(b + 24) = *(a25 + 0); + *(b + 25) = *(a26 + 0); + *(b + 26) = *(a27 + 0); + *(b + 27) = *(a28 + 0); + *(b + 28) = *(a29 + 0); + *(b + 29) = *(a30 + 0); + *(b + 30) = *(a31 + 0); + *(b + 31) = *(a32 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + a17 ++; + a18 ++; + a19 ++; + a20 ++; + a21 ++; + a22 ++; + a23 ++; + a24 ++; + a25 ++; + a26 ++; + a27 ++; + a28 ++; + a29 ++; + a30 ++; + a31 ++; + a32 ++; + b += 32; + ii ++; + } + + jj += 32; + } + + if (n & 16) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a9 = a + 8 * lda; + a10 = a + 9 * lda; + a11 = a + 10 * lda; + a12 = a + 11 * lda; + a13 = a + 12 * lda; + a14 = a + 13 * lda; + a15 = a + 14 * lda; + a16 = a + 15 * lda; + a += 16 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 16; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + *(b + 8) = *(a9 + 0); + *(b + 9) = *(a10 + 0); + *(b + 10) = *(a11 + 0); + *(b + 11) = *(a12 + 0); + *(b + 12) = *(a13 + 0); + *(b + 13) = *(a14 + 0); + *(b + 14) = *(a15 + 0); + *(b + 15) = *(a16 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + a9 ++; + a10 ++; + a11 ++; + a12 ++; + a13 ++; + a14 ++; + a15 ++; + a16 ++; + b += 16; + ii ++; + } + + jj += 16; + } + + if (n & 8) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a5 = a + 4 * lda; + a6 = a + 5 * lda; + a7 = a + 6 * lda; + a8 = a + 7 * lda; + a += 8 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 8; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + *(b + 4) = *(a5 + 0); + *(b + 5) = *(a6 + 0); + *(b + 6) = *(a7 + 0); + *(b + 7) = *(a8 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + a5 ++; + a6 ++; + a7 ++; + a8 ++; + b += 8; + ii ++; + } + + jj += 8; + } + + if (n & 4) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a3 = a + 2 * lda; + a4 = a + 3 * lda; + a += 4 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 4; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + *(b + 2) = *(a3 + 0); + *(b + 3) = *(a4 + 0); + } + + a1 ++; + a2 ++; + a3 ++; + a4 ++; + b += 4; + ii ++; + } + + jj += 4; + } + + if (n & 2) { + + a1 = a + 0 * lda; + a2 = a + 1 * lda; + a += 2 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 2; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a2 + 0); + } + + a1 ++; + a2 ++; + b += 2; + ii ++; + } + + jj += 2; + } + + if (n & 1) { + + a1 = a + 0 * lda; + a += 1 * lda; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + *(b + ii - jj) = INV(*(a1 + (ii - jj) * lda)); + for (k = ii - jj + 1; k < 1; k ++) { + *(b + k) = *(a1 + k * lda); + } + } + + if (ii - jj < 0) { + *(b + 0) = *(a1 + 0); + } + + a1 ++; + b += 1; + ii ++; + } + + jj += 1; + } + + return 0; +} diff --git a/kernel/generic/trsm_utcopy_32.c b/kernel/generic/trsm_utcopy_32.c new file mode 100644 index 0000000000..2a0637ebdc --- /dev/null +++ b/kernel/generic/trsm_utcopy_32.c @@ -0,0 +1,227 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" +/* NOTE: based on utcopy_16; MR=32 main panel not yet specialized. + * Correct for n < 32 (covers BLAT3). Full MR=32 panel TBD. */ + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1; + + jj = offset; + + j = (n >> 4); + while (j > 0){ + + a1 = a; + a += 16; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 16) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + *(b + 8) = *(a1 + 8); + *(b + 9) = *(a1 + 9); + *(b + 10) = *(a1 + 10); + *(b + 11) = *(a1 + 11); + *(b + 12) = *(a1 + 12); + *(b + 13) = *(a1 + 13); + *(b + 14) = *(a1 + 14); + *(b + 15) = *(a1 + 15); + } + + b += 16; + a1 += lda; + ii ++; + } + + jj += 16; + j --; + } + + j = (n & 8); + if (j > 0) { + a1 = a; + a += 8; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 8) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + } + + b += 8; + a1 += lda; + ii ++; + } + + jj += 8; + } + + j = (n & 4); + if (j > 0) { + + a1 = a; + a += 4; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 4) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + } + + b += 4; + a1 += lda; + ii ++; + } + + jj += 4; + } + + j = (n & 2); + if (j > 0) { + + a1 = a; + a += 2; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 2) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + } + + b += 2; + a1 += lda; + ii ++; + } + + jj += 2; + } + + j = (n & 1); + if (j > 0) { + + a1 = a; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 1) { + *(b + 0) = *(a1 + 0); + } + + b += 1; + a1 += lda; + ii ++; + } + } + + return 0; +} diff --git a/kernel/generic/trsm_utcopy_64.c b/kernel/generic/trsm_utcopy_64.c new file mode 100644 index 0000000000..da5fcecd86 --- /dev/null +++ b/kernel/generic/trsm_utcopy_64.c @@ -0,0 +1,227 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/* All rights reserved. */ +/* */ +/* Redistribution and use in source and binary forms, with or */ +/* without modification, are permitted provided that the following */ +/* conditions are met: */ +/* */ +/* 1. Redistributions of source code must retain the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer. */ +/* */ +/* 2. Redistributions in binary form must reproduce the above */ +/* copyright notice, this list of conditions and the following */ +/* disclaimer in the documentation and/or other materials */ +/* provided with the distribution. */ +/* */ +/* THIS SOFTWARE IS PROVIDED BY THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, */ +/* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF */ +/* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE */ +/* DISCLAIMED. IN NO EVENT SHALL THE UNIVERSITY OF TEXAS AT */ +/* AUSTIN OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, */ +/* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES */ +/* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE */ +/* GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR */ +/* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF */ +/* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT */ +/* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT */ +/* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE */ +/* POSSIBILITY OF SUCH DAMAGE. */ +/* */ +/* The views and conclusions contained in the software and */ +/* documentation are those of the authors and should not be */ +/* interpreted as representing official policies, either expressed */ +/* or implied, of The University of Texas at Austin. */ +/*********************************************************************/ + +#include +#include "common.h" +/* NOTE: based on utcopy_16; MR=64 main panel not yet specialized. + * Correct for n < 32 (covers BLAT3). Full MR=64 panel TBD. */ + +#ifndef UNIT +#define INV(a) (ONE / (a)) +#else +#define INV(a) (ONE) +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG offset, FLOAT *b){ + + BLASLONG i, ii, j, jj, k; + + FLOAT *a1; + + jj = offset; + + j = (n >> 4); + while (j > 0){ + + a1 = a; + a += 16; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 16)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 16) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + *(b + 8) = *(a1 + 8); + *(b + 9) = *(a1 + 9); + *(b + 10) = *(a1 + 10); + *(b + 11) = *(a1 + 11); + *(b + 12) = *(a1 + 12); + *(b + 13) = *(a1 + 13); + *(b + 14) = *(a1 + 14); + *(b + 15) = *(a1 + 15); + } + + b += 16; + a1 += lda; + ii ++; + } + + jj += 16; + j --; + } + + j = (n & 8); + if (j > 0) { + a1 = a; + a += 8; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 8)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 8) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + *(b + 4) = *(a1 + 4); + *(b + 5) = *(a1 + 5); + *(b + 6) = *(a1 + 6); + *(b + 7) = *(a1 + 7); + } + + b += 8; + a1 += lda; + ii ++; + } + + jj += 8; + } + + j = (n & 4); + if (j > 0) { + + a1 = a; + a += 4; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 4)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 4) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + *(b + 2) = *(a1 + 2); + *(b + 3) = *(a1 + 3); + } + + b += 4; + a1 += lda; + ii ++; + } + + jj += 4; + } + + j = (n & 2); + if (j > 0) { + + a1 = a; + a += 2; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 2)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 2) { + *(b + 0) = *(a1 + 0); + *(b + 1) = *(a1 + 1); + } + + b += 2; + a1 += lda; + ii ++; + } + + jj += 2; + } + + j = (n & 1); + if (j > 0) { + + a1 = a; + ii = 0; + + for (i = 0; i < m; i++) { + + if ((ii >= jj ) && (ii - jj < 1)) { + for (k = 0; k < ii - jj; k ++) { + *(b + k) = *(a1 + k); + } + + *(b + ii - jj) = INV(*(a1 + ii - jj)); + } + + if (ii - jj >= 1) { + *(b + 0) = *(a1 + 0); + } + + b += 1; + a1 += lda; + ii ++; + } + } + + return 0; +} diff --git a/kernel/riscv64/KERNEL.RISCV64_ZVL1024B b/kernel/riscv64/KERNEL.RISCV64_ZVL1024B index 9819188dce..5fe04c1831 100644 --- a/kernel/riscv64/KERNEL.RISCV64_ZVL1024B +++ b/kernel/riscv64/KERNEL.RISCV64_ZVL1024B @@ -106,7 +106,7 @@ endif SGEMMONCOPYOBJ = sgemm_oncopy$(TSUFFIX).$(SUFFIX) SGEMMOTCOPYOBJ = sgemm_otcopy$(TSUFFIX).$(SUFFIX) ifneq ($(SGEMM_UNROLL_M), $(SGEMM_UNROLL_N)) -ifneq ($(filter $(SGEMM_UNROLL_M),4 8 16),) +ifneq ($(filter $(SGEMM_UNROLL_M),4 8 16 32 64),) SGEMMINCOPY = gemm_ncopy_$(SGEMM_UNROLL_M)_rvv.c SGEMMITCOPY = gemm_tcopy_$(SGEMM_UNROLL_M)_rvv.c else @@ -127,12 +127,19 @@ endif DGEMMONCOPYOBJ = dgemm_oncopy$(TSUFFIX).$(SUFFIX) DGEMMOTCOPYOBJ = dgemm_otcopy$(TSUFFIX).$(SUFFIX) ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N)) -DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c ifneq ($(filter $(DGEMM_UNROLL_M),4 8 16),) +DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c +DGEMMITCOPY = gemm_tcopy_$(DGEMM_UNROLL_M)_rvv.c +else +ifneq ($(filter $(DGEMM_UNROLL_M),32 64),) +# Fixed-MR bitsliced packers (not VLMAX/contig) for 32x8 / 64x8 tiles +DGEMMINCOPY = gemm_ncopy_$(DGEMM_UNROLL_M)_rvv.c DGEMMITCOPY = gemm_tcopy_$(DGEMM_UNROLL_M)_rvv.c else +DGEMMINCOPY = ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c DGEMMITCOPY = ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c endif +endif DGEMMINCOPYOBJ = dgemm_incopy$(TSUFFIX).$(SUFFIX) DGEMMITCOPYOBJ = dgemm_itcopy$(TSUFFIX).$(SUFFIX) endif @@ -291,3 +298,20 @@ endif SAXPBYKERNEL = axpby_vector_v2.c DAXPBYKERNEL = axpby_vector_v2.c + +# MR=32/64 bitsliced packers: power-of-two panels (…,16,8,4,2,1) matching +# generate_kernel a_pack=bitsliced tails (not VLMAX-chunked rvv_v1). +STRMMUNCOPY_M = trmm_uncopy_64_rvv.c +STRMMLNCOPY_M = trmm_lncopy_64_rvv.c +STRMMUTCOPY_M = trmm_utcopy_64_rvv.c +STRMMLTCOPY_M = trmm_ltcopy_64_rvv.c +SSYMMUCOPY_M = symm_ucopy_64_rvv.c +SSYMMLCOPY_M = symm_lcopy_64_rvv.c + +DTRMMUNCOPY_M = trmm_uncopy_32_rvv.c +DTRMMLNCOPY_M = trmm_lncopy_32_rvv.c +DTRMMUTCOPY_M = trmm_utcopy_32_rvv.c +DTRMMLTCOPY_M = trmm_ltcopy_32_rvv.c +DSYMMUCOPY_M = symm_ucopy_32_rvv.c +DSYMMLCOPY_M = symm_lcopy_32_rvv.c + diff --git a/kernel/riscv64/dgemm_kernel_32x8_zvl1024b.c b/kernel/riscv64/dgemm_kernel_32x8_zvl1024b.c new file mode 100644 index 0000000000..7cbe89356a --- /dev/null +++ b/kernel/riscv64/dgemm_kernel_32x8_zvl1024b.c @@ -0,0 +1,1303 @@ +/* + +AUTOGENERATED KERNEL +Settings: + LMUL=1 + M=32 + M_tail_scalar_from=2 + N=8 + __riscv_='__riscv_' + a_pack='bitsliced' + complex=False + conjugate=False + cpu='zvl1024b' + force_acc_double=False + index_type='BLASLONG' + op='gemm' + param_precision='double' + reg_width_bits=1024 + tail_policy='' + trace=False + +Derived: + ELEN_ACC=64 + ELEN_PARAM=64 + LMUL_ACC=1 + VFMACC='__riscv_vfmacc_vf_f64m1' + VFMUL='__riscv_vfmul_vf_f64m1' + VLEV='__riscv_vle64_v_f64m1' + VLSEV='__riscv_vlse64_v_f64m1' + VMACC_TO_ACC='__riscv_vfmacc_vf_f64m1' + VMUL_TO_ACC='__riscv_vfmul_vf_f64m1' + VSETVL='__riscv_vsetvl_e64m1' + VSEV='__riscv_vse64_v_f64m1' + VSSEV='__riscv_vsse64_v_f64m1' + acc_vector_t='vfloat64m1_t' + output='dgemm_kernel_32x8_zvl1024b.c' + param_scalar_t='double' + param_vector_t='vfloat64m1_t' + +*/ + +#include "common.h" + + +int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc) + +{ + BLASLONG gvl = 0; + BLASLONG m_top = 0; + BLASLONG n_top = 0; + + + // -- MAIN PASS + + for (BLASLONG j=0; j 0) { + vl = VSETVL((size_t)remain); + FLOAT_V_T v0 = VLSEV_FLOAT(src, lda * sizeof(FLOAT), vl); + VSEV_FLOAT(b_offset, v0, vl); + src += (BLASLONG)vl * lda; + b_offset += vl; + remain -= (BLASLONG)vl; + } + } + *b_offset_p = b_offset; +} + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b) +{ + /* m = K, n = M_rows. Gather rows via stride lda. */ + FLOAT *a_offset = a; + FLOAT *b_offset = b; + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + pack_panel(m, w, a_offset, lda, &b_offset); + a_offset += w * lda; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/gemm_ncopy_64_rvv.c b/kernel/riscv64/gemm_ncopy_64_rvv.c new file mode 100644 index 0000000000..b73134a798 --- /dev/null +++ b/kernel/riscv64/gemm_ncopy_64_rvv.c @@ -0,0 +1,60 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +/* Bitsliced ncopy for MR=64 — panels 64,32,16,8,4,2,1. */ + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#endif + +static void pack_panel(BLASLONG m, BLASLONG w, FLOAT *a_offset, BLASLONG lda, FLOAT **b_offset_p) +{ + FLOAT *b_offset = *b_offset_p; + BLASLONG k; + size_t vl; + + for (k = 0; k < m; k++) { + BLASLONG remain = w; + FLOAT *src = a_offset + k; + while (remain > 0) { + vl = VSETVL((size_t)remain); + FLOAT_V_T v0 = VLSEV_FLOAT(src, lda * sizeof(FLOAT), vl); + VSEV_FLOAT(b_offset, v0, vl); + src += (BLASLONG)vl * lda; + b_offset += vl; + remain -= (BLASLONG)vl; + } + } + *b_offset_p = b_offset; +} + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b) +{ + FLOAT *a_offset = a; + FLOAT *b_offset = b; + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + pack_panel(m, w, a_offset, lda, &b_offset); + a_offset += w * lda; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/gemm_ncopy_contig_rvv.c b/kernel/riscv64/gemm_ncopy_contig_rvv.c new file mode 100644 index 0000000000..51717ba2a1 --- /dev/null +++ b/kernel/riscv64/gemm_ncopy_contig_rvv.c @@ -0,0 +1,37 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m1(n) +#define FLOAT_V_T vfloat32m1_t +#define VLEV_FLOAT __riscv_vle32_v_f32m1 +#define VSEV_FLOAT __riscv_vse32_v_f32m1 +#else +#define VSETVL(n) __riscv_vsetvl_e64m1(n) +#define FLOAT_V_T vfloat64m1_t +#define VLEV_FLOAT __riscv_vle64_v_f64m1 +#define VSEV_FLOAT __riscv_vse64_v_f64m1 +#endif + +/* Contiguous column-panel ncopy for generate_kernel a_pack=contiguous. + * Args: m = K, n = M_rows. Single [n × m] panel (for each k: n values). */ +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, FLOAT *b) +{ + FLOAT *b_offset = b; + BLASLONG k; + + for (k = 0; k < m; k++) { + BLASLONG r = 0; + while (r < n) { + /* scalar along n for simplicity/correctness; K is outer */ + *b_offset++ = a[r * lda + k]; + r++; + } + } + + return 0; +} diff --git a/kernel/riscv64/gemm_tcopy_32_rvv.c b/kernel/riscv64/gemm_tcopy_32_rvv.c new file mode 100644 index 0000000000..4f81806ca0 --- /dev/null +++ b/kernel/riscv64/gemm_tcopy_32_rvv.c @@ -0,0 +1,62 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +/* Bitsliced tcopy for MR=32. Packs n (=M) in power-of-two panels + * 32,16,8,4,2,1 — must match generate_kernel a_pack=bitsliced tail order. */ + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#endif + +static void pack_panel(BLASLONG m, BLASLONG w, IFLOAT *aoffset, BLASLONG lda, IFLOAT **boffset_p) +{ + IFLOAT *boffset = *boffset_p; + BLASLONG k; + size_t vl; + + for (k = 0; k < m; k++) { + BLASLONG remain = w; + IFLOAT *src = aoffset + k * lda; + while (remain > 0) { + vl = VSETVL((size_t)remain); + FLOAT_V_T v0 = VLEV_FLOAT(src, vl); + VSEV_FLOAT(boffset, v0, vl); + src += vl; + boffset += vl; + remain -= (BLASLONG)vl; + } + } + *boffset_p = boffset; +} + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) +{ + /* m = K, n = M_rows (ITCOPY convention). */ + IFLOAT *aoffset = a; + IFLOAT *boffset = b; + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + pack_panel(m, w, aoffset, lda, &boffset); + aoffset += w; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/gemm_tcopy_64_rvv.c b/kernel/riscv64/gemm_tcopy_64_rvv.c new file mode 100644 index 0000000000..9cd2f1f0f2 --- /dev/null +++ b/kernel/riscv64/gemm_tcopy_64_rvv.c @@ -0,0 +1,60 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +/* Bitsliced tcopy for MR=64 — panels 64,32,16,8,4,2,1. */ + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#endif + +static void pack_panel(BLASLONG m, BLASLONG w, IFLOAT *aoffset, BLASLONG lda, IFLOAT **boffset_p) +{ + IFLOAT *boffset = *boffset_p; + BLASLONG k; + size_t vl; + + for (k = 0; k < m; k++) { + BLASLONG remain = w; + IFLOAT *src = aoffset + k * lda; + while (remain > 0) { + vl = VSETVL((size_t)remain); + FLOAT_V_T v0 = VLEV_FLOAT(src, vl); + VSEV_FLOAT(boffset, v0, vl); + src += vl; + boffset += vl; + remain -= (BLASLONG)vl; + } + } + *boffset_p = boffset; +} + +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) +{ + IFLOAT *aoffset = a; + IFLOAT *boffset = b; + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + pack_panel(m, w, aoffset, lda, &boffset); + aoffset += w; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/gemm_tcopy_contig_rvv.c b/kernel/riscv64/gemm_tcopy_contig_rvv.c new file mode 100644 index 0000000000..31e16a1eee --- /dev/null +++ b/kernel/riscv64/gemm_tcopy_contig_rvv.c @@ -0,0 +1,44 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#endif + +/* Contiguous column-panel tcopy for generate_kernel a_pack=contiguous. + * Unlike gemm_tcopy_rvv_v1, does NOT split into VLMAX-sized sub-panels: + * layout is a single [n × m] panel (for each k: n consecutive rows). */ +int CNAME(BLASLONG m, BLASLONG n, IFLOAT *a, BLASLONG lda, IFLOAT *b) +{ + IFLOAT *aoffset = a; + IFLOAT *boffset = b; + BLASLONG i; + + for (i = 0; i < m; i++) { + IFLOAT *aoffset1 = aoffset; + BLASLONG remain = n; + while (remain > 0) { + size_t vl = VSETVL(remain); + FLOAT_V_T v0 = VLEV_FLOAT(aoffset1, vl); + VSEV_FLOAT(boffset, v0, vl); + aoffset1 += vl; + boffset += vl; + remain -= (BLASLONG)vl; + } + aoffset += lda; + } + + return 0; +} diff --git a/kernel/riscv64/generate_kernel.py b/kernel/riscv64/generate_kernel.py index 8be7c9f9cc..4b93f5ff89 100755 --- a/kernel/riscv64/generate_kernel.py +++ b/kernel/riscv64/generate_kernel.py @@ -46,7 +46,7 @@ def _indent_str( self ): return ' ' * (self._level * self._tab_width) #----------------------------------------------------------------------- -def generate_trmm_block( dest ): +def generate_trmm_block( dest, contiguous=False ): dest.write("{index_type} pass_K = K;") dest.write("#ifdef LEFT") with dest.block(): @@ -58,7 +58,11 @@ def generate_trmm_block( dest ): dest.write("#ifdef BACKWARDS") with dest.block(): - dest.write("ai += off*{M}{elt_size};") + # A stride: tile width (bitsliced) or runtime panel M (contiguous / rvv_v1 pack) + if contiguous: + dest.write("ai += off*M{elt_size};") + else: + dest.write("ai += off*{M}{elt_size};") dest.write("bi += off*{N}{elt_size};") dest.write("pass_K -= off;") dest.write("#else") @@ -76,15 +80,19 @@ def generate_trmm_block( dest ): def generate_gemm_kernel_inner_real( settings, dest, M, N, vlen, a_regs ): TRMM = (settings['op'].value == 'trmm') narrow_result = (settings['param_precision'].value != 'double') and settings['force_acc_double'].value + contiguous = (settings['a_pack'].value == 'contiguous') with dest.map( M=M, N=N, ): - dest.write("{index_type} ai=m_top*K{elt_size};") + if contiguous: + dest.write("{index_type} ai=m_top{elt_size};") + else: + dest.write("{index_type} ai=m_top*K{elt_size};") dest.write("{index_type} bi=n_top*K{elt_size};") if TRMM: - generate_trmm_block( dest ) + generate_trmm_block( dest, contiguous=contiguous ) for i in range(N): dest.write("{param_scalar_t} B{i} = B[bi+{i}];", i=i) @@ -93,7 +101,10 @@ def generate_gemm_kernel_inner_real( settings, dest, M, N, vlen, a_regs ): for i in range(a_regs): dest.write("{param_vector_t} A{i} = {VLEV}( &A[ai+{i}*gvl], gvl );", i=i) - dest.write("ai += {M};") + if contiguous: + dest.write("ai += M;") + else: + dest.write("ai += {M};") dest.write() for j in range(N): @@ -109,7 +120,10 @@ def generate_gemm_kernel_inner_real( settings, dest, M, N, vlen, a_regs ): for i in range(a_regs): dest.write("A{i} = {VLEV}( &A[ai+{i}*gvl], gvl );", i=i) - dest.write("ai += {M};") + if contiguous: + dest.write("ai += M;") + else: + dest.write("ai += {M};") dest.write() @@ -171,6 +185,7 @@ def generate_gemm_kernel_inner_real( settings, dest, M, N, vlen, a_regs ): def generate_gemm_kernel_inner_complex( settings, dest, M, N, vlen, a_regs ): TRMM = (settings['op'].value == 'trmm') narrow_result = (settings['param_precision'].value != 'double') and settings['force_acc_double'].value + contiguous = (settings['a_pack'].value == 'contiguous') if narrow_result: raise RuntimeError("wide accumulator not supported for generated complex kernels") @@ -180,10 +195,13 @@ def generate_gemm_kernel_inner_complex( settings, dest, M, N, vlen, a_regs ): M=M, N=N, ): - dest.write("{index_type} ai=m_top*K*2;") + if contiguous: + dest.write("{index_type} ai=m_top*2;") + else: + dest.write("{index_type} ai=m_top*K*2;") dest.write("{index_type} bi=n_top*K*2;") if TRMM: - generate_trmm_block( dest ) + generate_trmm_block( dest, contiguous=contiguous ) for i in range(N): dest.write("{param_scalar_t} B{i}r = B[bi+{i}*2+0];", i=i) @@ -194,7 +212,10 @@ def generate_gemm_kernel_inner_complex( settings, dest, M, N, vlen, a_regs ): for i in range(a_regs): dest.write("{param_vector_t} A{i}r = {VLSEV}( &A[ai+{i}*gvl*2], sizeof(FLOAT)*2, gvl );", i=i) dest.write("{param_vector_t} A{i}i = {VLSEV}( &A[ai+{i}*gvl*2+1], sizeof(FLOAT)*2, gvl );", i=i) - dest.write("ai += {M}*2;") + if contiguous: + dest.write("ai += M*2;") + else: + dest.write("ai += {M}*2;") dest.write() # for each vector register loaded from matrix A, we require N registers to hold vector-scalar multiply-accumulate results @@ -251,7 +272,10 @@ def generate_gemm_kernel_inner_complex( settings, dest, M, N, vlen, a_regs ): dest.write("A{i}r = {VLSEV}( &A[ai+{i}*gvl*2], sizeof(FLOAT)*2, gvl );", i=i) dest.write("A{i}i = {VLSEV}( &A[ai+{i}*gvl*2+1], sizeof(FLOAT)*2, gvl );", i=i) - dest.write("ai += {M}*2;") + if contiguous: + dest.write("ai += M*2;") + else: + dest.write("ai += {M}*2;") dest.write() @@ -482,12 +506,16 @@ def generate_M_tails( dest, settings, M, N ): r=r ) - dest.write("{index_type} ai=m_top*K{elt_size};") + contiguous = (settings['a_pack'].value == 'contiguous') + if contiguous: + dest.write("{index_type} ai=m_top{elt_size};") + else: + dest.write("{index_type} ai=m_top*K{elt_size};") dest.write("{index_type} bi=n_top*K{elt_size};") if TRMM: with dest.map(M=M_tail, N=N): - generate_trmm_block( dest ) + generate_trmm_block( dest, contiguous=contiguous ) with dest.block("for({index_type} k=0; k<{Kend}; k++) {{", "}}", Kend = ('pass_K' if TRMM else 'K') ): for ki in range( N ): @@ -503,7 +531,10 @@ def generate_M_tails( dest, settings, M, N ): dest.write("result{dest}+=A[ai+{kj}]*B[bi+{ki}];".format( dest=ki*M_tail+kj, kj=kj, ki=ki )) - dest.write("ai+={M_tail}{elt_size};") + if contiguous: + dest.write("ai+=M{elt_size};") + else: + dest.write("ai+={M_tail}{elt_size};") dest.write("bi+={N}{elt_size};") dest.write("{index_type} ci=n_top*ldc+m_top;") @@ -574,6 +605,10 @@ def main(): 'reg_width_bits': Setting( 256, int ), 'LMUL': Setting( 1, int ), 'M_tail_scalar_from':Setting( 2, int ), + # bitsliced: ai+=tile; pair with gemm_*copy_{MR}_rvv / gemm_tcopy_16 (panel-major). + # contiguous: ai+=M (full panel); pair with gemm_*copy_contig_rvv (column-major panel). + # Note: gemm_*copy_rvv_v1 is VLMAX-bitsliced, NOT contiguous — use it only when MR==VLMAX. + 'a_pack': Setting( 'bitsliced', Setting.ENUM( 'bitsliced', 'contiguous' ) ), 'cpu': Setting( 'zvl256b', str ), 'param_precision': Setting( 'float', Setting.ENUM( 'float', 'double' ) ), 'force_acc_double': Setting( False, Setting.BOOL ), diff --git a/kernel/riscv64/sgemm_kernel_64x8_zvl1024b.c b/kernel/riscv64/sgemm_kernel_64x8_zvl1024b.c new file mode 100644 index 0000000000..d497911ad4 --- /dev/null +++ b/kernel/riscv64/sgemm_kernel_64x8_zvl1024b.c @@ -0,0 +1,1524 @@ +/* + +AUTOGENERATED KERNEL +Settings: + LMUL=1 + M=64 + M_tail_scalar_from=2 + N=8 + __riscv_='__riscv_' + a_pack='bitsliced' + complex=False + conjugate=False + cpu='zvl1024b' + force_acc_double=False + index_type='BLASLONG' + op='gemm' + param_precision='float' + reg_width_bits=1024 + tail_policy='' + trace=False + +Derived: + ELEN_ACC=32 + ELEN_PARAM=32 + LMUL_ACC=1 + VFMACC='__riscv_vfmacc_vf_f32m1' + VFMUL='__riscv_vfmul_vf_f32m1' + VLEV='__riscv_vle32_v_f32m1' + VLSEV='__riscv_vlse32_v_f32m1' + VMACC_TO_ACC='__riscv_vfmacc_vf_f32m1' + VMUL_TO_ACC='__riscv_vfmul_vf_f32m1' + VSETVL='__riscv_vsetvl_e32m1' + VSEV='__riscv_vse32_v_f32m1' + VSSEV='__riscv_vsse32_v_f32m1' + acc_vector_t='vfloat32m1_t' + output='sgemm_kernel_64x8_zvl1024b.c' + param_scalar_t='float' + param_vector_t='vfloat32m1_t' + +*/ + +#include "common.h" + + +int CNAME(BLASLONG M, BLASLONG N, BLASLONG K, FLOAT alpha, FLOAT* A, FLOAT* B, FLOAT* C, BLASLONG ldc) + +{ + BLASLONG gvl = 0; + BLASLONG m_top = 0; + BLASLONG n_top = 0; + + + // -- MAIN PASS + + for (BLASLONG j=0; j= w) { + vl = VSETVL((size_t)w); + vindex_max = V_UM2_TO_IM2(VID_V_INT(vl)); + offset = posX - posY; + ao1 = a + posX + posY * lda; + ao2 = a + posY + (posX) * lda; + + for (i = m; i > 0; i--, offset--) { + va2 = VLSEV_FLOAT(ao2, stride_lda, vl); + va1 = VLEV_FLOAT(ao1, vl); + vindex = VADD_VX_INT(vindex_max, offset, vl); + vbool = VMSGT_VX_INT(vindex, 0, vl); + vb = VMERGE_VVM_FLOAT(va2, va1, vbool, vl); + VSEV_FLOAT(b, vb, vl); + b += vl; + ao1 += lda; + ao2++; + } + posX += w; + n -= w; + } + } + return 0; +} diff --git a/kernel/riscv64/symm_lcopy_64_rvv.c b/kernel/riscv64/symm_lcopy_64_rvv.c new file mode 100644 index 0000000000..f6c19e68fd --- /dev/null +++ b/kernel/riscv64/symm_lcopy_64_rvv.c @@ -0,0 +1,75 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +/* Bitsliced symm_lcopy: pack n in panels 64,16,8,4,2,1. */ + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define INT_V_T vint32m2_t +#define VID_V_INT __riscv_vid_v_u32m2 +#define VADD_VX_INT __riscv_vadd_vx_i32m2 +#define VMSGT_VX_INT __riscv_vmsgt_vx_i32m2_b16 +#define VBOOL_T vbool16_t +#define VMERGE_VVM_FLOAT __riscv_vmerge_vvm_f32m2 +#define V_UM2_TO_IM2 __riscv_vreinterpret_v_u32m2_i32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define INT_V_T vint64m2_t +#define VID_V_INT __riscv_vid_v_u64m2 +#define VADD_VX_INT __riscv_vadd_vx_i64m2 +#define VMSGT_VX_INT __riscv_vmsgt_vx_i64m2_b32 +#define VBOOL_T vbool32_t +#define VMERGE_VVM_FLOAT __riscv_vmerge_vvm_f64m2 +#define V_UM2_TO_IM2 __riscv_vreinterpret_v_u64m2_i64m2 +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b) +{ + BLASLONG i, offset; + FLOAT *ao1, *ao2; + BLASLONG stride_lda = sizeof(FLOAT) * lda; + FLOAT_V_T vb, va1, va2; + VBOOL_T vbool; + INT_V_T vindex_max, vindex; + size_t vl; + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + vindex_max = V_UM2_TO_IM2(VID_V_INT(vl)); + offset = posX - posY; + ao1 = a + posX + posY * lda; + ao2 = a + posY + (posX) * lda; + + for (i = m; i > 0; i--, offset--) { + va2 = VLSEV_FLOAT(ao2, stride_lda, vl); + va1 = VLEV_FLOAT(ao1, vl); + vindex = VADD_VX_INT(vindex_max, offset, vl); + vbool = VMSGT_VX_INT(vindex, 0, vl); + vb = VMERGE_VVM_FLOAT(va2, va1, vbool, vl); + VSEV_FLOAT(b, vb, vl); + b += vl; + ao1 += lda; + ao2++; + } + posX += w; + n -= w; + } + } + return 0; +} diff --git a/kernel/riscv64/symm_ucopy_32_rvv.c b/kernel/riscv64/symm_ucopy_32_rvv.c new file mode 100644 index 0000000000..dd6164821d --- /dev/null +++ b/kernel/riscv64/symm_ucopy_32_rvv.c @@ -0,0 +1,75 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +/* Bitsliced symm_ucopy: pack n in panels 32,16,8,4,2,1 (a_pack=bitsliced). */ + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define INT_V_T vint32m2_t +#define VID_V_INT __riscv_vid_v_u32m2 +#define VADD_VX_INT __riscv_vadd_vx_i32m2 +#define VMSGT_VX_INT __riscv_vmsgt_vx_i32m2_b16 +#define VBOOL_T vbool16_t +#define VMERGE_VVM_FLOAT __riscv_vmerge_vvm_f32m2 +#define V_UM2_TO_IM2 __riscv_vreinterpret_v_u32m2_i32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define INT_V_T vint64m2_t +#define VID_V_INT __riscv_vid_v_u64m2 +#define VADD_VX_INT __riscv_vadd_vx_i64m2 +#define VMSGT_VX_INT __riscv_vmsgt_vx_i64m2_b32 +#define VBOOL_T vbool32_t +#define VMERGE_VVM_FLOAT __riscv_vmerge_vvm_f64m2 +#define V_UM2_TO_IM2 __riscv_vreinterpret_v_u64m2_i64m2 +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b) +{ + BLASLONG i, offset; + FLOAT *ao1, *ao2; + BLASLONG stride_lda = sizeof(FLOAT) * lda; + FLOAT_V_T vb, va1, va2; + VBOOL_T vbool; + INT_V_T vindex_max, vindex; + size_t vl; + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + vindex_max = V_UM2_TO_IM2(VID_V_INT(vl)); + offset = posX - posY; + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posX + 0 + posY * lda; + + for (i = m; i > 0; i--, offset--) { + va1 = VLSEV_FLOAT(ao1, stride_lda, vl); + va2 = VLEV_FLOAT(ao2, vl); + vindex = VADD_VX_INT(vindex_max, offset, vl); + vbool = VMSGT_VX_INT(vindex, 0, vl); + vb = VMERGE_VVM_FLOAT(va2, va1, vbool, vl); + VSEV_FLOAT(b, vb, vl); + b += vl; + ao1++; + ao2 += lda; + } + posX += w; + n -= w; + } + } + return 0; +} diff --git a/kernel/riscv64/symm_ucopy_64_rvv.c b/kernel/riscv64/symm_ucopy_64_rvv.c new file mode 100644 index 0000000000..7f55caa5ef --- /dev/null +++ b/kernel/riscv64/symm_ucopy_64_rvv.c @@ -0,0 +1,75 @@ +/*************************************************************************** +Copyright (c) 2025, The OpenBLAS Project +All rights reserved. +*****************************************************************************/ + +#include "common.h" + +/* Bitsliced symm_ucopy: pack n in panels 64,16,8,4,2,1 (a_pack=bitsliced). */ + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define INT_V_T vint32m2_t +#define VID_V_INT __riscv_vid_v_u32m2 +#define VADD_VX_INT __riscv_vadd_vx_i32m2 +#define VMSGT_VX_INT __riscv_vmsgt_vx_i32m2_b16 +#define VBOOL_T vbool16_t +#define VMERGE_VVM_FLOAT __riscv_vmerge_vvm_f32m2 +#define V_UM2_TO_IM2 __riscv_vreinterpret_v_u32m2_i32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define INT_V_T vint64m2_t +#define VID_V_INT __riscv_vid_v_u64m2 +#define VADD_VX_INT __riscv_vadd_vx_i64m2 +#define VMSGT_VX_INT __riscv_vmsgt_vx_i64m2_b32 +#define VBOOL_T vbool32_t +#define VMERGE_VVM_FLOAT __riscv_vmerge_vvm_f64m2 +#define V_UM2_TO_IM2 __riscv_vreinterpret_v_u64m2_i64m2 +#endif + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b) +{ + BLASLONG i, offset; + FLOAT *ao1, *ao2; + BLASLONG stride_lda = sizeof(FLOAT) * lda; + FLOAT_V_T vb, va1, va2; + VBOOL_T vbool; + INT_V_T vindex_max, vindex; + size_t vl; + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + vindex_max = V_UM2_TO_IM2(VID_V_INT(vl)); + offset = posX - posY; + ao1 = a + posY + (posX + 0) * lda; + ao2 = a + posX + 0 + posY * lda; + + for (i = m; i > 0; i--, offset--) { + va1 = VLSEV_FLOAT(ao1, stride_lda, vl); + va2 = VLEV_FLOAT(ao2, vl); + vindex = VADD_VX_INT(vindex_max, offset, vl); + vbool = VMSGT_VX_INT(vindex, 0, vl); + vb = VMERGE_VVM_FLOAT(va2, va1, vbool, vl); + VSEV_FLOAT(b, vb, vl); + b += vl; + ao1++; + ao2 += lda; + } + posX += w; + n -= w; + } + } + return 0; +} diff --git a/kernel/riscv64/trmm_lncopy_32_rvv.c b/kernel/riscv64/trmm_lncopy_32_rvv.c new file mode 100644 index 0000000000..40a74256d2 --- /dev/null +++ b/kernel/riscv64/trmm_lncopy_32_rvv.c @@ -0,0 +1,142 @@ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_lncopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, js, X; + + FLOAT *ao; + + BLASLONG stride_lda = sizeof(FLOAT)*lda; + + FLOAT_V_T vb, va1; + + size_t vl; +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posY + posX * lda; + } + else + { + ao = a + posX + posY * lda; + } + + i = 0; + do + { + if (X > posY) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + VSEV_FLOAT(b, va1, vl); + + ao ++; + b += vl; + X ++; + i ++; + } + else if (X < posY) + { + ao += lda; + b += vl; + X ++; + i ++; + } + else + { + vindex = VID_V_UINT(vl); + for (unsigned int j = 0; j < vl; j++) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + vbool_cmp = VMSGTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao++; + b += vl; + } + + X += vl; + i += vl; + } + } while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/trmm_lncopy_64_rvv.c b/kernel/riscv64/trmm_lncopy_64_rvv.c new file mode 100644 index 0000000000..5e2032c791 --- /dev/null +++ b/kernel/riscv64/trmm_lncopy_64_rvv.c @@ -0,0 +1,142 @@ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_lncopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, js, X; + + FLOAT *ao; + + BLASLONG stride_lda = sizeof(FLOAT)*lda; + + FLOAT_V_T vb, va1; + + size_t vl; +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posY + posX * lda; + } + else + { + ao = a + posX + posY * lda; + } + + i = 0; + do + { + if (X > posY) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + VSEV_FLOAT(b, va1, vl); + + ao ++; + b += vl; + X ++; + i ++; + } + else if (X < posY) + { + ao += lda; + b += vl; + X ++; + i ++; + } + else + { + vindex = VID_V_UINT(vl); + for (unsigned int j = 0; j < vl; j++) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + vbool_cmp = VMSGTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao++; + b += vl; + } + + X += vl; + i += vl; + } + } while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/trmm_ltcopy_32_rvv.c b/kernel/riscv64/trmm_ltcopy_32_rvv.c new file mode 100644 index 0000000000..883530d1e5 --- /dev/null +++ b/kernel/riscv64/trmm_ltcopy_32_rvv.c @@ -0,0 +1,138 @@ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_ltcopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, js, X; + + FLOAT *ao; + + FLOAT_V_T vb, va1; + size_t vl; +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posY + posX * lda; + } + else + { + ao = a + posX + posY * lda; + } + + i = 0; + do + { + if (X > posY) + { + ao ++; + b += vl; + X ++; + i ++; + } + else if (X < posY) + { + va1 = VLEV_FLOAT(ao, vl); + VSEV_FLOAT(b, va1, vl); + + ao += lda; + b += vl; + X ++; + i ++; + } + else + { + vindex = VID_V_UINT(vl); + for (unsigned int j = 0; j < vl; j++) + { + va1 = VLEV_FLOAT(ao, vl); + vbool_cmp = VMSLTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao += lda; + b += vl; + } + X += vl; + i += vl; + + } + } while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} + diff --git a/kernel/riscv64/trmm_ltcopy_64_rvv.c b/kernel/riscv64/trmm_ltcopy_64_rvv.c new file mode 100644 index 0000000000..3b57560d12 --- /dev/null +++ b/kernel/riscv64/trmm_ltcopy_64_rvv.c @@ -0,0 +1,138 @@ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_ltcopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, js, X; + + FLOAT *ao; + + FLOAT_V_T vb, va1; + size_t vl; +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posY + posX * lda; + } + else + { + ao = a + posX + posY * lda; + } + + i = 0; + do + { + if (X > posY) + { + ao ++; + b += vl; + X ++; + i ++; + } + else if (X < posY) + { + va1 = VLEV_FLOAT(ao, vl); + VSEV_FLOAT(b, va1, vl); + + ao += lda; + b += vl; + X ++; + i ++; + } + else + { + vindex = VID_V_UINT(vl); + for (unsigned int j = 0; j < vl; j++) + { + va1 = VLEV_FLOAT(ao, vl); + vbool_cmp = VMSLTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao += lda; + b += vl; + } + X += vl; + i += vl; + + } + } while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} + diff --git a/kernel/riscv64/trmm_uncopy_32_rvv.c b/kernel/riscv64/trmm_uncopy_32_rvv.c new file mode 100644 index 0000000000..627f76ecc2 --- /dev/null +++ b/kernel/riscv64/trmm_uncopy_32_rvv.c @@ -0,0 +1,140 @@ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_uncopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, js, X; + BLASLONG stride_lda = sizeof(FLOAT) * lda; + FLOAT *ao; + + FLOAT_V_T vb, va1; + size_t vl; + +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posX + posY * lda; + } + else + { + ao = a + posY + posX * lda; + } + + i = 0; + do + { + if (X < posY) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + VSEV_FLOAT(b, va1, vl); + + ao ++; + b += vl; + X ++; + i ++; + } + else if (X > posY) + { + ao += lda; + b += vl; + X ++; + i ++; + } + else + { + vindex = VID_V_UINT(vl); + for (unsigned int j = 0; j < vl; j++) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + vbool_cmp = VMSLTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao++; + b += vl; + } + + X += vl; + i += vl; + } + }while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/trmm_uncopy_64_rvv.c b/kernel/riscv64/trmm_uncopy_64_rvv.c new file mode 100644 index 0000000000..fc2805dc13 --- /dev/null +++ b/kernel/riscv64/trmm_uncopy_64_rvv.c @@ -0,0 +1,140 @@ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VLSEV_FLOAT __riscv_vlse32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VLSEV_FLOAT __riscv_vlse64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSLTU_VX_UINT __riscv_vmsltu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_uncopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, js, X; + BLASLONG stride_lda = sizeof(FLOAT) * lda; + FLOAT *ao; + + FLOAT_V_T vb, va1; + size_t vl; + +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posX + posY * lda; + } + else + { + ao = a + posY + posX * lda; + } + + i = 0; + do + { + if (X < posY) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + VSEV_FLOAT(b, va1, vl); + + ao ++; + b += vl; + X ++; + i ++; + } + else if (X > posY) + { + ao += lda; + b += vl; + X ++; + i ++; + } + else + { + vindex = VID_V_UINT(vl); + for (unsigned int j = 0; j < vl; j++) + { + va1 = VLSEV_FLOAT(ao, stride_lda, vl); + vbool_cmp = VMSLTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao++; + b += vl; + } + + X += vl; + i += vl; + } + }while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} diff --git a/kernel/riscv64/trmm_utcopy_32_rvv.c b/kernel/riscv64/trmm_utcopy_32_rvv.c new file mode 100644 index 0000000000..d6d3bce126 --- /dev/null +++ b/kernel/riscv64/trmm_utcopy_32_rvv.c @@ -0,0 +1,138 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_utcopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, j, js, X; + + FLOAT *ao; + FLOAT_V_T vb, va1; +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + size_t vl; + + BLASLONG widths[] = {32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 6; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posX + posY * lda; + } + else + { + ao = a + posY + posX * lda; + } + + i = 0; + do + { + if (X < posY) + { + ao ++; + b += vl; + X ++; + i++; + } + else if (X > posY) + { + va1 = VLEV_FLOAT(ao, vl); + VSEV_FLOAT(b, va1, vl); + ao += lda; + b += vl; + X++; + i++; + } + else + { + vindex = VID_V_UINT(vl); + for (j = 0; j < vl; j++) + { + va1 = VLEV_FLOAT(ao, vl); + vbool_cmp = VMSGTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao += lda; + b += vl; + } + X += vl; + i += vl; + } + }while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} + diff --git a/kernel/riscv64/trmm_utcopy_64_rvv.c b/kernel/riscv64/trmm_utcopy_64_rvv.c new file mode 100644 index 0000000000..9a73337939 --- /dev/null +++ b/kernel/riscv64/trmm_utcopy_64_rvv.c @@ -0,0 +1,138 @@ +/*********************************************************************/ +/* Copyright 2009, 2010 The University of Texas at Austin. */ +/*************************************************************************** +Copyright (c) 2022, The OpenBLAS Project +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: +1. Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in +the documentation and/or other materials provided with the +distribution. +3. Neither the name of the OpenBLAS project nor the names of +its contributors may be used to endorse or promote products +derived from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*****************************************************************************/ + + +#include +#include "common.h" + +#if !defined(DOUBLE) +#define VSETVL(n) __riscv_vsetvl_e32m2(n) +#define FLOAT_V_T vfloat32m2_t +#define VLEV_FLOAT __riscv_vle32_v_f32m2 +#define VSEV_FLOAT __riscv_vse32_v_f32m2 +#define VBOOL_T vbool16_t +#define UINT_V_T vuint32m2_t +#define VID_V_UINT __riscv_vid_v_u32m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u32m2_b16 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u32m2_b16 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f32m2 +#else +#define VSETVL(n) __riscv_vsetvl_e64m2(n) +#define FLOAT_V_T vfloat64m2_t +#define VLEV_FLOAT __riscv_vle64_v_f64m2 +#define VSEV_FLOAT __riscv_vse64_v_f64m2 +#define VBOOL_T vbool32_t +#define UINT_V_T vuint64m2_t +#define VID_V_UINT __riscv_vid_v_u64m2 +#define VMSGTU_VX_UINT __riscv_vmsgtu_vx_u64m2_b32 +#define VMSEQ_VX_UINT __riscv_vmseq_vx_u64m2_b32 +#define VFMERGE_VFM_FLOAT __riscv_vfmerge_vfm_f64m2 +#endif + +// Optimizes the implementation in ../arm64/tmmm_utcopy_sve_v1.c + +int CNAME(BLASLONG m, BLASLONG n, FLOAT *a, BLASLONG lda, BLASLONG posX, BLASLONG posY, FLOAT *b){ + + BLASLONG i, j, js, X; + + FLOAT *ao; + FLOAT_V_T vb, va1; +#ifdef UNIT + VBOOL_T vbool_eq; +#endif + + VBOOL_T vbool_cmp; + UINT_V_T vindex; + + size_t vl; + + BLASLONG widths[] = {64, 32, 16, 8, 4, 2, 1}; + int wi; + for (wi = 0; wi < 7; wi++) { + BLASLONG w = widths[wi]; + while (n >= w) { + vl = VSETVL((size_t)w); + X = posX; + + if (posX <= posY) + { + ao = a + posX + posY * lda; + } + else + { + ao = a + posY + posX * lda; + } + + i = 0; + do + { + if (X < posY) + { + ao ++; + b += vl; + X ++; + i++; + } + else if (X > posY) + { + va1 = VLEV_FLOAT(ao, vl); + VSEV_FLOAT(b, va1, vl); + ao += lda; + b += vl; + X++; + i++; + } + else + { + vindex = VID_V_UINT(vl); + for (j = 0; j < vl; j++) + { + va1 = VLEV_FLOAT(ao, vl); + vbool_cmp = VMSGTU_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(va1, ZERO, vbool_cmp, vl); +#ifdef UNIT + vbool_eq = VMSEQ_VX_UINT(vindex, j, vl); + vb = VFMERGE_VFM_FLOAT(vb, ONE, vbool_eq, vl); +#endif + VSEV_FLOAT(b, vb, vl); + ao += lda; + b += vl; + } + X += vl; + i += vl; + } + }while (i < m); + posY += vl; + n -= w; + } + } + + return 0; +} + diff --git a/param.h b/param.h index 920afc1208..a3066c0711 100644 --- a/param.h +++ b/param.h @@ -3337,10 +3337,10 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define SBGEMM_DEFAULT_UNROLL_M 16 #define SBGEMM_DEFAULT_UNROLL_N 8 -#define SGEMM_DEFAULT_UNROLL_M 16 +#define SGEMM_DEFAULT_UNROLL_M 64 #define SGEMM_DEFAULT_UNROLL_N 8 -#define DGEMM_DEFAULT_UNROLL_M 16 +#define DGEMM_DEFAULT_UNROLL_M 32 #define DGEMM_DEFAULT_UNROLL_N 8 #define CGEMM_DEFAULT_UNROLL_M 8