ggml : update softmax n_task calculation (ggml-org#5126)

snadampal · pull[bot] · commit 156604369d80 · 2024-06-25T19:07:34.000Z
updated the n_task calculation to use max number of
threads possible. This has improved the prompt eval
performance by around 5% for DOT kernels and by
around 10% for MMLA kernels on AWS Graviton3.
diff --git a/ggml.c b/ggml.c
@@ -16597,7 +16597,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
             } break;
         case GGML_OP_SOFT_MAX:
             {
-                n_tasks = MIN(MIN(4, n_threads), ggml_nrows(node->src[0]));
+                n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
             } break;
         case GGML_OP_CONV_TRANSPOSE_1D:
             {

Original file line number	Diff line number	Diff line change
`@@ -16597,7 +16597,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {`
`16597`	`16597`	`} break;`
`16598`	`16598`	`case GGML_OP_SOFT_MAX:`
`16599`	`16599`	`{`
`16600`		`- n_tasks = MIN(MIN(4, n_threads), ggml_nrows(node->src[0]));`
	`16600`	`+ n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));`
`16601`	`16601`	`} break;`
`16602`	`16602`	`case GGML_OP_CONV_TRANSPOSE_1D:`
`16603`	`16603`	`{`