Skip to content

Refactor init_system_thp_mode and print it in malloc stats. #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 13, 2025
Merged

Refactor init_system_thp_mode and print it in malloc stats. #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions include/jemalloc/internal/pages.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,19 +97,31 @@ static const bool pages_can_hugify =
#endif
;

/*
* thp_mode_t are values for opt.thp, while system_thp_mode_t is for kernel thp
* settings, i.e., init_system_thp_mode.
*/
typedef enum {
thp_mode_default = 0, /* Do not change hugepage settings. */
thp_mode_do_nothing = 0, /* Respect kernel thp settings. */
thp_mode_always = 1, /* Always set MADV_HUGEPAGE. */
thp_mode_never = 2, /* Always set MADV_NOHUGEPAGE. */

thp_mode_names_limit = 3, /* Used for option processing. */
thp_mode_not_supported = 3 /* No THP support detected. */
} thp_mode_t;

#define THP_MODE_DEFAULT thp_mode_default
typedef enum {
system_thp_mode_madvise = 0, /* Kernel THP mode: madvise */
system_thp_mode_always = 1, /* Kernel THP mode: always */
system_thp_mode_never = 2, /* Kernel THP mode: never */
system_thp_mode_not_supported = 3 /* No THP support detected. */
} system_thp_mode_t;

#define THP_MODE_DEFAULT thp_mode_do_nothing
extern thp_mode_t opt_thp;
extern thp_mode_t init_system_thp_mode; /* Initial system wide state. */
extern system_thp_mode_t init_system_thp_mode; /* Initial system wide state. */
extern const char *const thp_mode_names[];
extern const char *const system_thp_mode_names[];

void *pages_map(void *addr, size_t size, size_t alignment, bool *commit);
void pages_unmap(void *addr, size_t size);
Expand Down
4 changes: 2 additions & 2 deletions src/arena.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1890,8 +1890,8 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
/* Make sure that b0 thp auto-switch won't happen concurrently here. */
malloc_mutex_lock(tsdn, &b0->mtx);
(&huge_arena_pac_thp)->thp_madvise = opt_huge_arena_pac_thp
&& metadata_thp_enabled() && (opt_thp == thp_mode_default)
&& (init_system_thp_mode == thp_mode_default);
&& metadata_thp_enabled() && (opt_thp == thp_mode_do_nothing)
&& (init_system_thp_mode == system_thp_mode_madvise);
(&huge_arena_pac_thp)->auto_thp_switched =
b0->auto_thp_switched;
malloc_mutex_init(&(&huge_arena_pac_thp)->lock, "pac_thp",
Expand Down
4 changes: 2 additions & 2 deletions src/base.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ const char *const metadata_thp_mode_names[] = {"disabled", "auto", "always"};

static inline bool
metadata_thp_madvise(void) {
return (metadata_thp_enabled()
&& (init_system_thp_mode == thp_mode_default));
return (metadata_thp_enabled() &&
(init_system_thp_mode == system_thp_mode_madvise));
}

static void *
Expand Down
41 changes: 27 additions & 14 deletions src/pages.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,10 @@ static bool os_overcommits;

const char *const thp_mode_names[] = {
"default", "always", "never", "not supported"};
thp_mode_t opt_thp = THP_MODE_DEFAULT;
thp_mode_t init_system_thp_mode;
const char *const system_thp_mode_names[] = {
"madvise", "always", "never", "not supported"};
thp_mode_t opt_thp = THP_MODE_DEFAULT;
system_thp_mode_t init_system_thp_mode;

/* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */
static bool pages_can_purge_lazy_runtime = true;
Expand Down Expand Up @@ -778,21 +780,31 @@ os_overcommits_proc(void) {
}
#endif

static bool
pages_should_skip_set_thp_state() {
if (opt_thp == thp_mode_do_nothing
|| (opt_thp == thp_mode_always
&& init_system_thp_mode == system_thp_mode_always)
|| (opt_thp == thp_mode_never
&& init_system_thp_mode == system_thp_mode_never)) {
return true;
}
return false;
}
void
pages_set_thp_state(void *ptr, size_t size) {
if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) {
if (pages_should_skip_set_thp_state()) {
return;
}
assert(opt_thp != thp_mode_not_supported
&& init_system_thp_mode != thp_mode_not_supported);
&& init_system_thp_mode != system_thp_mode_not_supported);

if (opt_thp == thp_mode_always
&& init_system_thp_mode != thp_mode_never) {
assert(init_system_thp_mode == thp_mode_default);
&& init_system_thp_mode == system_thp_mode_madvise) {
pages_huge_unaligned(ptr, size);
} else if (opt_thp == thp_mode_never) {
assert(init_system_thp_mode == thp_mode_default
|| init_system_thp_mode == thp_mode_always);
assert(init_system_thp_mode == system_thp_mode_madvise
|| init_system_thp_mode == system_thp_mode_always);
pages_nohuge_unaligned(ptr, size);
}
}
Expand Down Expand Up @@ -825,31 +837,32 @@ init_thp_state(void) {
}

if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_default;
init_system_thp_mode = system_thp_mode_madvise;
} else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_always;
init_system_thp_mode = system_thp_mode_always;
} else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_never;
init_system_thp_mode = system_thp_mode_never;
} else {
goto label_error;
}
if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) {
if (init_system_thp_mode == thp_mode_default) {
if (init_system_thp_mode == system_thp_mode_madvise) {
opt_hpa_opts.hugify_style = hpa_hugify_style_lazy;
} else {
opt_hpa_opts.hugify_style = hpa_hugify_style_none;
}
}
return;
#elif defined(JEMALLOC_HAVE_MEMCNTL)
init_system_thp_mode = thp_mode_default;
init_system_thp_mode = system_thp_mode_madvise;
if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) {
opt_hpa_opts.hugify_style = hpa_hugify_style_eager;
}
return;
#endif
label_error:
opt_thp = init_system_thp_mode = thp_mode_not_supported;
opt_thp = thp_mode_not_supported;
init_system_thp_mode = system_thp_mode_not_supported;
}

bool
Expand Down
16 changes: 16 additions & 0 deletions src/stats.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1516,6 +1516,22 @@ stats_general_print(emitter_t *emitter) {
#undef CONFIG_WRITE_BOOL
emitter_dict_end(emitter); /* Close "config" dict. */

/* system. */
emitter_dict_begin(emitter, "system", "System configuration");

/*
* This shows system's THP mode detected at jemalloc's init time.
* jemalloc does not re-detect the mode even if it changes after
* jemalloc's init. It is assumed that system's THP mode is stable
* during the process's lifetime and a violation could lead to
* undefined behavior.
*/
const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode];
emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string,
&thp_mode_name);

emitter_dict_end(emitter); /* Close "system". */

/* opt. */
#define OPT_WRITE(name, var, size, emitter_type) \
if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \
Expand Down
2 changes: 1 addition & 1 deletion test/unit/pages.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ TEST_BEGIN(test_pages_huge) {
pages = pages_map(NULL, alloc_size, PAGE, &commit);
expect_ptr_not_null(pages, "Unexpected pages_map() error");

if (init_system_thp_mode == thp_mode_default) {
if (init_system_thp_mode == system_thp_mode_madvise) {
hugepage = (void *)(ALIGNMENT_CEILING(
(uintptr_t)pages, HUGEPAGE));
expect_b_ne(pages_huge(hugepage, HUGEPAGE), have_madvise_huge,
Expand Down