cuda : reuse ggml-common

ggml-ci

Author: ggerganov

ggml-common.h

@@ -32,14 +32,22 @@ typedef half ggml_fp16_t;
 #endif
 #endif
 
+// QK = number of values after dequantization
+// QR = QK / number of values before dequantization
+// QI = number of 32 bit integers before dequantization
+
 #define QK4_0 32
+#define QI4_0 (QK4_0 / (4 * QR4_0))
+#define QR4_0 2
 typedef struct {
     ggml_fp16_t d;          // delta
     uint8_t qs[QK4_0 / 2];  // nibbles / quants
 } block_q4_0;
 static_assert(sizeof(block_q4_0) == sizeof(ggml_fp16_t) + QK4_0 / 2, "wrong q4_0 block size/padding");
 
 #define QK4_1 32
+#define QI4_1 (QK4_1 / (4 * QR4_1))
+#define QR4_1 2
 typedef struct {
     ggml_fp16_t d;          // delta
     ggml_fp16_t m;          // min
@@ -48,6 +56,8 @@ typedef struct {
 static_assert(sizeof(block_q4_1) == 2 * sizeof(ggml_fp16_t) + QK4_1 / 2, "wrong q4_1 block size/padding");
 
 #define QK5_0 32
+#define QI5_0 (QK5_0 / (4 * QR5_0))
+#define QR5_0 2
 typedef struct {
     ggml_fp16_t d;         // delta
     uint8_t qh[4];         // 5-th bit of quants
@@ -56,6 +66,8 @@ typedef struct {
 static_assert(sizeof(block_q5_0) == sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_0 / 2, "wrong q5_0 block size/padding");
 
 #define QK5_1 32
+#define QI5_1 (QK5_1 / (4 * QR5_1))
+#define QR5_1 2
 typedef struct {
     ggml_fp16_t d;         // delta
     ggml_fp16_t m;         // min
@@ -65,13 +77,17 @@ typedef struct {
 static_assert(sizeof(block_q5_1) == 2 * sizeof(ggml_fp16_t) + sizeof(uint32_t) + QK5_1 / 2, "wrong q5_1 block size/padding");
 
 #define QK8_0 32
+#define QI8_0 (QK8_0 / (4 * QR8_0))
+#define QR8_0 1
 typedef struct {
     ggml_fp16_t d;         // delta
     int8_t  qs[QK8_0];     // quants
 } block_q8_0;
 static_assert(sizeof(block_q8_0) == sizeof(ggml_fp16_t) + QK8_0, "wrong q8_0 block size/padding");
 
 #define QK8_1 32
+#define QI8_1 (QK8_1 / (4 * QR8_1))
+#define QR8_1 1
 typedef struct {
     float d;               // delta
     float s;               // d * sum(qs[i])
@@ -96,6 +112,8 @@ static_assert(sizeof(block_q8_1) == 2*sizeof(float) + QK8_1, "wrong q8_1 block s
 // weight is represented as x = a * q + b
 // 16 blocks of 16 elements each
 // Effectively 2.625 bits per weight
+#define QI2_K (QK_K / (4*QR2_K))
+#define QR2_K 4
 typedef struct {
     uint8_t scales[QK_K/16]; // scales and mins, quantized with 4 bits
     uint8_t qs[QK_K/4];      // quants
@@ -108,6 +126,8 @@ static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_fp16_t) + QK_K/16 + QK_K/4, "w
 // weight is represented as x = a * q
 // 16 blocks of 16 elements each
 // Effectively 3.4375 bits per weight
+#define QI3_K (QK_K / (4*QR3_K))
+#define QR3_K 4
 #ifdef GGML_QKK_64
 typedef struct {
     uint8_t hmask[QK_K/8];     // quants - high bit
@@ -130,6 +150,8 @@ static_assert(sizeof(block_q3_K) == sizeof(ggml_fp16_t) + QK_K / 4 + QK_K / 8 +
 // 8 blocks of 32 elements each
 // weight is represented as x = a * q + b
 // Effectively 4.5 bits per weight
+#define QI4_K (QK_K / (4*QR4_K))
+#define QR4_K 2
 #ifdef GGML_QKK_64
 typedef struct {
     ggml_fp16_t d[2];          // super-block scales/mins
@@ -151,6 +173,8 @@ static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/
 // 8 blocks of 32 elements each
 // weight is represented as x = a * q + b
 // Effectively 5.5 bits per weight
+#define QI5_K (QK_K / (4*QR5_K))
+#define QR5_K 2
 #ifdef GGML_QKK_64
 typedef struct {
     ggml_fp16_t d;               // super-block scale
@@ -174,6 +198,8 @@ static_assert(sizeof(block_q5_K) == 2*sizeof(ggml_fp16_t) + K_SCALE_SIZE + QK_K/
 // weight is represented as x = a * q
 // 16 blocks of 16 elements each
 // Effectively 6.5625 bits per weight
+#define QI6_K (QK_K / (4*QR6_K))
+#define QR6_K 2
 typedef struct {
     uint8_t ql[QK_K/2];      // quants, lower 4 bits
     uint8_t qh[QK_K/4];      // quants, upper 2 bits
@@ -193,13 +219,17 @@ static_assert(sizeof(block_q8_K) == sizeof(float) + QK_K + QK_K/16*sizeof(int16_
 // (Almost) "true" 2-bit quantization.
 // Due to the need to use blocks as per ggml design, it ends up using
 // 2.0625 bpw because of the 16-bit scale for each block of 256.
+#define QI2_XXS (QK_K / (4*QR2_XXS))
+#define QR2_XXS 8
 typedef struct {
     ggml_fp16_t d;
     uint16_t qs[QK_K/8];
 } block_iq2_xxs;
 static_assert(sizeof(block_iq2_xxs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t), "wrong iq2_xxs block size/padding");
 
 // 2.3125 bpw quants
+#define QI2_XS (QK_K / (4*QR2_XS))
+#define QR2_XS 8
 typedef struct {
     ggml_fp16_t d;
     uint16_t qs[QK_K/8];
@@ -208,6 +238,8 @@ typedef struct {
 static_assert(sizeof(block_iq2_xs) == sizeof(ggml_fp16_t) + QK_K/8*sizeof(uint16_t) + QK_K/32, "wrong iq2_xs block size/padding");
 
 // 2.5625 bpw quants
+#define QI2_S (QK_K / (4*QR2_S))
+#define QR2_S 8
 typedef struct {
     ggml_fp16_t d;
     uint8_t qs[QK_K/4];
@@ -219,6 +251,8 @@ static_assert(sizeof(block_iq2_s) == sizeof(ggml_fp16_t) + QK_K/4 + QK_K/16, "wr
 // (Almost) "true" 3-bit quantization.
 // Due to the need to use blocks as per ggml design, it ends up using
 // 3.0625 bpw because of the 16-bit scale for each block of 256.
+#define QI3_XXS (QK_K / (4*QR3_XXS))
+#define QR3_XXS 8
 typedef struct {
     ggml_fp16_t d;
     uint8_t qs[3*QK_K/8];
@@ -231,6 +265,8 @@ static_assert(sizeof(block_iq3_xxs) == sizeof(ggml_fp16_t) + 3*(QK_K/8), "wrong
 #else
 #define IQ3S_N_SCALE QK_K/64
 #endif
+#define QI3_XS (QK_K / (4*QR3_XS))
+#define QR3_XS 8
 typedef struct {
     ggml_fp16_t d;
     uint8_t qs[QK_K/4];
@@ -240,6 +276,8 @@ typedef struct {
 } block_iq3_s;
 static_assert(sizeof(block_iq3_s) == sizeof(ggml_fp16_t) + 13*(QK_K/32) + IQ3S_N_SCALE, "wrong iq3_s block size/padding");
 
+#define QI1_S (QK_K / (4*QR1_S))
+#define QR1_S 8
 typedef struct {
     ggml_fp16_t d;
     uint8_t qs[QK_K/8];
@@ -249,6 +287,8 @@ static_assert(sizeof(block_iq1_s) == sizeof(ggml_fp16_t) + QK_K/8 + QK_K/16, "wr
 
 // Non-linear quants
 #define QK4_NL 32
+#define QI4_NL (QK4_NL / (4*QR4_NL))
+#define QR4_NL 2
 typedef struct {
     ggml_fp16_t d;
     uint8_t qs[QK4_NL/2];
@@ -257,8 +297,12 @@ static_assert(sizeof(block_iq4_nl) == sizeof(ggml_fp16_t) + QK4_NL/2, "wrong iq4
 
 #if QK_K == 64
 #define block_iq4_xs block_iq4_nl
+#define QI4_XS QI4_NL
+#define QR4_XS QR4_NL
 //typedef struct block_iq4_nl block_iq4_xs;
 #else
+#define QI4_XS (QK_K / (4*QR4_XS))
+#define QR4_XS 8
 typedef struct {
     ggml_fp16_t d;
     uint16_t scales_h;