Skip to content

Commit 5c9222d

Browse files
author
fmz
committed
Restrict threadpool to CPU backend
1 parent 1d9d39a commit 5c9222d

File tree

8 files changed

+52
-53
lines changed

8 files changed

+52
-53
lines changed

‎examples/llava/clip.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1915,7 +1915,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
19151915
}
19161916
#endif
19171917

1918-
ggml_backend_graph_compute(ctx->backend, gf, NULL);
1918+
ggml_backend_graph_compute(ctx->backend, gf);
19191919

19201920
// the last node is the embedding tensor
19211921
struct ggml_tensor * embeddings = gf->nodes[gf->n_nodes - 1];

‎ggml-backend-impl.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,14 +92,14 @@ extern "C" {
9292
void (*GGML_CALL synchronize)(ggml_backend_t backend);
9393

9494
// compute graph with a plan (not used currently)
95-
ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph, ggml_compute_threadpool_t threadpool);
95+
ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);
9696
void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
9797

9898
// compute graph with a plan
9999
enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);
100100

101101
// compute graph without a plan (async)
102-
enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph, ggml_compute_threadpool_t threadpool);
102+
enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);
103103

104104
// check if the backend supports an operation
105105
bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);

‎ggml-backend.c‎

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -255,13 +255,12 @@ void ggml_backend_synchronize(ggml_backend_t backend) {
255255
}
256256

257257
ggml_backend_graph_plan_t ggml_backend_graph_plan_create(
258-
ggml_backend_t backend,
259-
const struct ggml_cgraph * cgraph,
260-
ggml_compute_threadpool_t threadpool
258+
ggml_backend_t backend,
259+
const struct ggml_cgraph * cgraph
261260
) {
262261
GGML_ASSERT(backend->iface.graph_plan_create != NULL);
263262

264-
return backend->iface.graph_plan_create(backend, cgraph, threadpool);
263+
return backend->iface.graph_plan_create(backend, cgraph);
265264
}
266265

267266
void ggml_backend_graph_plan_free(ggml_backend_t backend, ggml_backend_graph_plan_t plan) {
@@ -281,20 +280,18 @@ enum ggml_status ggml_backend_graph_plan_compute(
281280

282281
enum ggml_status ggml_backend_graph_compute(
283282
ggml_backend_t backend,
284-
struct ggml_cgraph * cgraph,
285-
ggml_compute_threadpool_t threadpool
283+
struct ggml_cgraph * cgraph
286284
) {
287-
enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph, threadpool);
285+
enum ggml_status err = ggml_backend_graph_compute_async(backend, cgraph);
288286
ggml_backend_synchronize(backend);
289287
return err;
290288
}
291289

292290
enum ggml_status ggml_backend_graph_compute_async(
293-
ggml_backend_t backend,
294-
struct ggml_cgraph * cgraph,
295-
ggml_compute_threadpool_t threadpool
291+
ggml_backend_t backend,
292+
struct ggml_cgraph * cgraph
296293
) {
297-
return backend->iface.graph_compute(backend, cgraph, threadpool);
294+
return backend->iface.graph_compute(backend, cgraph);
298295
}
299296

300297
bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op) {
@@ -741,7 +738,9 @@ ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void) {
741738
#endif
742739

743740
struct ggml_backend_cpu_context {
744-
int n_threads;
741+
int n_threads;
742+
ggml_compute_threadpool_t threadpool;
743+
745744
void * work_data;
746745
size_t work_size;
747746

@@ -774,15 +773,14 @@ struct ggml_backend_plan_cpu {
774773
};
775774

776775
GGML_CALL static ggml_backend_graph_plan_t ggml_backend_cpu_graph_plan_create(
777-
ggml_backend_t backend,
778-
const struct ggml_cgraph * cgraph,
779-
ggml_compute_threadpool_t threadpool
776+
ggml_backend_t backend,
777+
const struct ggml_cgraph * cgraph
780778
) {
781779
struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
782780

783781
struct ggml_backend_plan_cpu * cpu_plan = malloc(sizeof(struct ggml_backend_plan_cpu));
784782

785-
cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, threadpool);
783+
cpu_plan->cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
786784
cpu_plan->cgraph = *cgraph; // FIXME: deep copy
787785

788786
if (cpu_plan->cplan.work_size > 0) {
@@ -817,13 +815,12 @@ GGML_CALL static enum ggml_status ggml_backend_cpu_graph_plan_compute(ggml_backe
817815
}
818816

819817
GGML_CALL static enum ggml_status ggml_backend_cpu_graph_compute(
820-
ggml_backend_t backend,
821-
struct ggml_cgraph * cgraph,
822-
ggml_compute_threadpool_t threadpool
818+
ggml_backend_t backend,
819+
struct ggml_cgraph * cgraph
823820
) {
824821
struct ggml_backend_cpu_context * cpu_ctx = (struct ggml_backend_cpu_context *)backend->context;
825822

826-
struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, threadpool);
823+
struct ggml_cplan cplan = ggml_graph_plan(cgraph, cpu_ctx->n_threads, cpu_ctx->threadpool);
827824

828825
if (cpu_ctx->work_size < cplan.work_size) {
829826
free(cpu_ctx->work_data);
@@ -892,6 +889,7 @@ ggml_backend_t ggml_backend_cpu_init(void) {
892889
}
893890

894891
ctx->n_threads = GGML_DEFAULT_N_THREADS;
892+
ctx->threadpool = NULL;
895893
ctx->work_data = NULL;
896894
ctx->work_size = 0;
897895
ctx->abort_callback = NULL;
@@ -922,6 +920,13 @@ void ggml_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads) {
922920
ctx->n_threads = n_threads;
923921
}
924922

923+
void ggml_backend_cpu_set_threadpool(ggml_backend_t backend_cpu, ggml_compute_threadpool_t threadpool) {
924+
GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
925+
926+
struct ggml_backend_cpu_context * ctx = (struct ggml_backend_cpu_context *)backend_cpu->context;
927+
ctx->threadpool = threadpool;
928+
}
929+
925930
void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data) {
926931
GGML_ASSERT(ggml_backend_is_cpu(backend_cpu));
927932

@@ -1653,10 +1658,7 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {
16531658
return true;
16541659
}
16551660

1656-
static enum ggml_status ggml_backend_sched_compute_splits(
1657-
ggml_backend_sched_t sched,
1658-
ggml_compute_threadpool_t threadpool
1659-
) {
1661+
static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) {
16601662
struct ggml_backend_sched_split * splits = sched->splits;
16611663

16621664
for (int i = 0; i < sched->n_splits; i++) {
@@ -1690,7 +1692,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(
16901692
}
16911693

16921694
if (!sched->callback_eval) {
1693-
enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph, threadpool);
1695+
enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph);
16941696
if (ec != GGML_STATUS_SUCCESS) {
16951697
return ec;
16961698
}
@@ -1712,7 +1714,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(
17121714

17131715
struct ggml_cgraph gv = ggml_graph_view(&split->graph, j0, j1 + 1);
17141716

1715-
enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv, threadpool);
1717+
enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &gv);
17161718
if (ec != GGML_STATUS_SUCCESS) {
17171719
return ec;
17181720
}
@@ -1852,19 +1854,17 @@ bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgra
18521854
}
18531855

18541856
enum ggml_status ggml_backend_sched_graph_compute(
1855-
ggml_backend_sched_t sched,
1856-
struct ggml_cgraph * graph,
1857-
ggml_compute_threadpool_t threadpool
1857+
ggml_backend_sched_t sched,
1858+
struct ggml_cgraph * graph
18581859
) {
1859-
enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph, threadpool);
1860+
enum ggml_status err = ggml_backend_sched_graph_compute_async(sched, graph);
18601861
ggml_backend_sched_synchronize(sched);
18611862
return err;
18621863
}
18631864

18641865
enum ggml_status ggml_backend_sched_graph_compute_async(
1865-
ggml_backend_sched_t sched,
1866-
struct ggml_cgraph * graph,
1867-
ggml_compute_threadpool_t threadpool
1866+
ggml_backend_sched_t sched,
1867+
struct ggml_cgraph * graph
18681868
) {
18691869
if (!sched->is_reset && !sched->is_alloc) {
18701870
ggml_backend_sched_reset(sched);
@@ -1876,7 +1876,7 @@ enum ggml_status ggml_backend_sched_graph_compute_async(
18761876
}
18771877
}
18781878

1879-
return ggml_backend_sched_compute_splits(sched, threadpool);
1879+
return ggml_backend_sched_compute_splits(sched);
18801880
}
18811881

18821882
void ggml_backend_sched_synchronize(ggml_backend_sched_t sched) {
@@ -2115,8 +2115,8 @@ bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t
21152115
struct ggml_cgraph g1v = ggml_graph_view(g1, i, i + 1);
21162116
struct ggml_cgraph g2v = ggml_graph_view(g2, i, i + 1);
21172117

2118-
ggml_backend_graph_compute(backend1, &g1v, NULL);
2119-
ggml_backend_graph_compute(backend2, &g2v, NULL);
2118+
ggml_backend_graph_compute(backend1, &g1v);
2119+
ggml_backend_graph_compute(backend2, &g2v);
21202120

21212121
if (ggml_is_view_op(t1->op)) {
21222122
continue;

‎ggml-backend.h‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,7 @@ extern "C" {
6969

7070
GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(
7171
ggml_backend_t backend,
72-
const struct ggml_cgraph * cgraph,
73-
ggml_compute_threadpool_t threadpool);
72+
const struct ggml_cgraph * cgraph);
7473

7574
GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
7675

@@ -79,12 +78,10 @@ extern "C" {
7978
ggml_backend_graph_plan_t plan);
8079
GGML_API enum ggml_status ggml_backend_graph_compute(
8180
ggml_backend_t backend,
82-
struct ggml_cgraph * cgraph,
83-
ggml_compute_threadpool_t threadpool);
81+
struct ggml_cgraph * cgraph);
8482
GGML_API enum ggml_status ggml_backend_graph_compute_async(
8583
ggml_backend_t backend,
86-
struct ggml_cgraph * cgraph,
87-
ggml_compute_threadpool_t threadpool);
84+
struct ggml_cgraph * cgraph);
8885
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
8986
GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
9087

@@ -112,6 +109,7 @@ extern "C" {
112109

113110
GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);
114111
GGML_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);
112+
GGML_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_compute_threadpool_t threadpool);
115113
GGML_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
116114

117115
// Create a backend buffer from an existing pointer
@@ -205,8 +203,8 @@ extern "C" {
205203

206204
// Allocate and compute graph on the backend scheduler
207205
GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
208-
GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph, ggml_compute_threadpool_t threadpool);
209-
GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph, ggml_compute_threadpool_t threadpool);
206+
GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
207+
GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
210208
GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
211209

212210
// Reset all assignments and allocators - must be called before changing the node backends

‎ggml.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19501,7 +19501,7 @@ static void __cpumask_next(const bool * global_mask, bool * local_mask, bool str
1950119501
int32_t base_idx = *iter;
1950219502
for (int32_t i = 0; i < GGML_N_CORES_MAX; i++) {
1950319503
int32_t idx = base_idx + i;
19504-
if (idx > GGML_N_CORES_MAX) {
19504+
if (idx >= GGML_N_CORES_MAX) {
1950519505
// Just a cheaper modulo
1950619506
idx -= GGML_N_CORES_MAX;
1950719507
}

‎ggml.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2051,7 +2051,7 @@ extern "C" {
20512051
const struct ggml_cgraph * cgraph,
20522052
int n_threads,
20532053
struct ggml_compute_threadpool * threadpool);
2054-
GGML_API enum ggml_status ggml_graph_compute ( struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);
2054+
GGML_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan);
20552055
// same as ggml_graph_compute() but the work data is allocated as a part of the context
20562056
// note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
20572057
GGML_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads);

‎llama.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11366,10 +11366,11 @@ static void llama_graph_compute(
1136611366

1136711367
if (lctx.backend_cpu != nullptr) {
1136811368
ggml_backend_cpu_set_n_threads(lctx.backend_cpu, n_threads);
11369+
ggml_backend_cpu_set_threadpool(lctx.backend_cpu, threadpool);
1136911370
ggml_backend_cpu_set_abort_callback(lctx.backend_cpu, lctx.abort_callback, lctx.abort_callback_data);
1137011371
}
1137111372

11372-
ggml_backend_sched_graph_compute_async(lctx.sched, gf, threadpool);
11373+
ggml_backend_sched_graph_compute_async(lctx.sched, gf);
1137311374

1137411375
// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(lctx.sched));
1137511376
}
@@ -15428,7 +15429,7 @@ static int llama_apply_lora_from_file_internal(
1542815429
return 1;
1542915430
}
1543015431

15431-
ggml_backend_graph_compute(backend_cpu, gf, nullptr);
15432+
ggml_backend_graph_compute(backend_cpu, gf);
1543215433

1543315434
ggml_backend_tensor_set(model_t, r->data, 0, ggml_nbytes(r));
1543415435

‎tests/test-backend-ops.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,7 @@ struct test_case {
587587
ggml_build_forward_expand(gf, out);
588588

589589
// warmup run
590-
ggml_backend_graph_compute(backend, gf, nullptr);
590+
ggml_backend_graph_compute(backend, gf);
591591

592592
// duplicate the op
593593
size_t target_size = ggml_backend_is_cpu(backend) ? 1ULL << 33 : 1ULL << 35; // 8 GB CPU, 32 GB GPU
@@ -619,7 +619,7 @@ struct test_case {
619619
ggml_backend_synchronize(backend);
620620

621621
int64_t start_time = ggml_time_us();
622-
ggml_backend_graph_compute(backend, gf, nullptr);
622+
ggml_backend_graph_compute(backend, gf);
623623
ggml_backend_synchronize(backend);
624624
int64_t end_time = ggml_time_us();
625625
double time_us = end_time - start_time;

0 commit comments

Comments
 (0)