diff --git a/ggml/src/ggml-cpu/arch/s390/typedef.h b/ggml/src/ggml-cpu/arch/s390/typedef.h new file mode 100644 index 000000000..7b1dace2b --- /dev/null +++ b/ggml/src/ggml-cpu/arch/s390/typedef.h @@ -0,0 +1,155 @@ +#ifndef GGML_S390X_TYPEDEF_H +#define GGML_S390X_TYPEDEF_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__s390x__) && defined(__VEC__) +#define vec_neg(a) (-(a)) // Vector Negate +#define vec_add(a, b) ((a) + (b)) // Vector Add +#define vec_sub(a, b) ((a) - (b)) // Vector Subtract +#define vec_mul(a, b) ((a) * (b)) // Vector Multiply +#define vec_div(a, b) ((a) / (b)) // Vector Divide +#define vec_sl(a, b) ((a) << (b)) // Vector Shift Left +#define vec_sra(a, b) ((a) >> (b)) // Vector Shift Right +#define vec_sr(a, b) ((a) >> (b)) // Vector Shift Right Algebraic +#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet +#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet + +#ifndef vec_and +#define vec_and(a, b) ((a) & (b)) // Vector AND +#endif + +#ifndef vec_or +#define vec_or(a, b) ((a) | (b)) // Vector OR +#endif + +#ifndef vec_xor +#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR +#endif + +typedef signed char char8x16_t __attribute__((vector_size(16))); +typedef unsigned char uchar8x16_t __attribute__((vector_size(16))); + +typedef int8_t int8x16_t __attribute__((vector_size(16))); +typedef int16_t int16x8_t __attribute__((vector_size(16))); +typedef int32_t int32x4_t __attribute__((vector_size(16))); + +typedef uint8_t uint8x16_t __attribute__((vector_size(16))); +typedef uint16_t uint16x8_t __attribute__((vector_size(16))); +typedef uint32_t uint32x4_t __attribute__((vector_size(16))); + +typedef float float32x4_t __attribute__((vector_size(16))); +typedef double double64x2_t __attribute__((vector_size(16))); + +typedef signed long long long64x2_t __attribute__((vector_size(16))); +typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); + +typedef struct ggml_uint8x16x2_t { + uint8x16_t val[2]; +} ggml_uint8x16x2_t; + +inline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) { + ggml_uint8x16x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + +typedef struct ggml_uint8x16x4_t { + uint8x16_t val[4]; +} ggml_uint8x16x4_t; + +inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) { + ggml_uint8x16x4_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + res.val[2] = vec_xl(32, ptr); + res.val[3] = vec_xl(48, ptr); + + return res; +} + +typedef struct ggml_int8x16x4_t { + int8x16_t val[4]; +} ggml_int8x16x4_t; + +inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { + ggml_int8x16x4_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + res.val[2] = vec_xl(32, ptr); + res.val[3] = vec_xl(48, ptr); + + return res; +} + +typedef struct ggml_int16x8x2_t { + int16x8_t val[2]; +} ggml_int16x8x2_t; + +inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) { + ggml_int16x8x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + +/* + ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs + ! or iq4_nl for example implementation. +*/ +inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { + int8x16_t res; + + res[ 0] = a[b[ 0]]; + res[ 1] = a[b[ 1]]; + res[ 2] = a[b[ 2]]; + res[ 3] = a[b[ 3]]; + res[ 4] = a[b[ 4]]; + res[ 5] = a[b[ 5]]; + res[ 6] = a[b[ 6]]; + res[ 7] = a[b[ 7]]; + res[ 8] = a[b[ 8]]; + res[ 9] = a[b[ 9]]; + res[10] = a[b[10]]; + res[11] = a[b[11]]; + res[12] = a[b[12]]; + res[13] = a[b[13]]; + res[14] = a[b[14]]; + res[15] = a[b[15]]; + + return res; +} + +inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { + const uchar8x16_t v_maske = { 0, 1, 4, 5, 8, 9, 12, 13, + 16, 17, 20, 21, 24, 25, 28, 29 }; + + const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b); + const int16x8_t v_abe = vec_perm(a, b, v_maske); + return v_abo + v_abe; +} + +inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { + const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); + return acc + (vec_unpackh(p) + vec_unpackl(p)); +} + +#endif // __s390x__ && __VEC__ + +#ifdef __cplusplus +} +#endif // __cplusplus + +#endif // GGML_S390X_TYPEDEF_H diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index d839cf5c5..4715fbd1d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -353,144 +353,7 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #if defined(__VXE__) || defined(__VXE2__) #include - -#define vec_neg(a) (-(a)) // Vector Negate -#define vec_add(a, b) ((a) + (b)) // Vector Add -#define vec_sub(a, b) ((a) - (b)) // Vector Subtract -#define vec_mul(a, b) ((a) * (b)) // Vector Multiply -#define vec_div(a, b) ((a) / (b)) // Vector Divide -#define vec_sl(a, b) ((a) << (b)) // Vector Shift Left -#define vec_sra(a, b) ((a) >> (b)) // Vector Shift Right -#define vec_sr(a, b) ((a) >> (b)) // Vector Shift Right Algebraic -#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet -#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet - -#ifndef vec_and -#define vec_and(a, b) ((a) & (b)) // Vector AND -#endif - -#ifndef vec_or -#define vec_or(a, b) ((a) | (b)) // Vector OR -#endif - -#ifndef vec_xor -#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR -#endif - -typedef signed char char8x16_t __attribute__((vector_size(16))); -typedef unsigned char uchar8x16_t __attribute__((vector_size(16))); - -typedef int8_t int8x16_t __attribute__((vector_size(16))); -typedef int16_t int16x8_t __attribute__((vector_size(16))); -typedef int32_t int32x4_t __attribute__((vector_size(16))); - -typedef uint8_t uint8x16_t __attribute__((vector_size(16))); -typedef uint16_t uint16x8_t __attribute__((vector_size(16))); -typedef uint32_t uint32x4_t __attribute__((vector_size(16))); - -typedef float float32x4_t __attribute__((vector_size(16))); -typedef double double64x2_t __attribute__((vector_size(16))); - -typedef signed long long long64x2_t __attribute__((vector_size(16))); -typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); - -typedef struct ggml_uint8x16x2_t { - uint8x16_t val[2]; -} ggml_uint8x16x2_t; - -inline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) { - ggml_uint8x16x2_t res; - - res.val[0] = vec_xl( 0, ptr); - res.val[1] = vec_xl(16, ptr); - - return res; -} - -typedef struct ggml_uint8x16x4_t { - uint8x16_t val[4]; -} ggml_uint8x16x4_t; - -inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) { - ggml_uint8x16x4_t res; - - res.val[0] = vec_xl( 0, ptr); - res.val[1] = vec_xl(16, ptr); - res.val[2] = vec_xl(32, ptr); - res.val[3] = vec_xl(48, ptr); - - return res; -} - -typedef struct ggml_int8x16x4_t { - int8x16_t val[4]; -} ggml_int8x16x4_t; - -inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { - ggml_int8x16x4_t res; - - res.val[0] = vec_xl( 0, ptr); - res.val[1] = vec_xl(16, ptr); - res.val[2] = vec_xl(32, ptr); - res.val[3] = vec_xl(48, ptr); - - return res; -} - -typedef struct ggml_int16x8x2_t { - int16x8_t val[2]; -} ggml_int16x8x2_t; - -inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) { - ggml_int16x8x2_t res; - - res.val[0] = vec_xl( 0, ptr); - res.val[1] = vec_xl(16, ptr); - - return res; -} - -/* - ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs - ! or iq4_nl for example implementation. -*/ -inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { - int8x16_t res; - - res[ 0] = a[b[ 0]]; - res[ 1] = a[b[ 1]]; - res[ 2] = a[b[ 2]]; - res[ 3] = a[b[ 3]]; - res[ 4] = a[b[ 4]]; - res[ 5] = a[b[ 5]]; - res[ 6] = a[b[ 6]]; - res[ 7] = a[b[ 7]]; - res[ 8] = a[b[ 8]]; - res[ 9] = a[b[ 9]]; - res[10] = a[b[10]]; - res[11] = a[b[11]]; - res[12] = a[b[12]]; - res[13] = a[b[13]]; - res[14] = a[b[14]]; - res[15] = a[b[15]]; - - return res; -} - -inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { - const uchar8x16_t v_maske = { 0, 1, 4, 5, 8, 9, 12, 13, - 16, 17, 20, 21, 24, 25, 28, 29 }; - - const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b); - const int16x8_t v_abe = vec_perm(a, b, v_maske); - return v_abo + v_abe; -} - -inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { - const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); - return acc + (vec_unpackh(p) + vec_unpackl(p)); -} - +#include #endif #if defined(__loongarch_asx) diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index d04b03a39..e8772b376 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -30,7 +30,7 @@ #if defined(GGML_VXE) #include -#include +#include #endif #if defined(GGML_NNPA)