diff --git a/include/jemalloc/internal/pages.h b/include/jemalloc/internal/pages.h index 319099345..a4282c9bb 100644 --- a/include/jemalloc/internal/pages.h +++ b/include/jemalloc/internal/pages.h @@ -97,8 +97,12 @@ static const bool pages_can_hugify = #endif ; +/* + * thp_mode_t are values for opt.thp, while system_thp_mode_t is for kernel thp + * settings, i.e., init_system_thp_mode. + */ typedef enum { - thp_mode_default = 0, /* Do not change hugepage settings. */ + thp_mode_do_nothing = 0, /* Respect kernel thp settings. */ thp_mode_always = 1, /* Always set MADV_HUGEPAGE. */ thp_mode_never = 2, /* Always set MADV_NOHUGEPAGE. */ @@ -106,10 +110,18 @@ typedef enum { thp_mode_not_supported = 3 /* No THP support detected. */ } thp_mode_t; -#define THP_MODE_DEFAULT thp_mode_default +typedef enum { + system_thp_mode_madvise = 0, /* Kernel THP mode: madvise */ + system_thp_mode_always = 1, /* Kernel THP mode: always */ + system_thp_mode_never = 2, /* Kernel THP mode: never */ + system_thp_mode_not_supported = 3 /* No THP support detected. */ +} system_thp_mode_t; + +#define THP_MODE_DEFAULT thp_mode_do_nothing extern thp_mode_t opt_thp; -extern thp_mode_t init_system_thp_mode; /* Initial system wide state. */ +extern system_thp_mode_t init_system_thp_mode; /* Initial system wide state. */ extern const char *const thp_mode_names[]; +extern const char *const system_thp_mode_names[]; void *pages_map(void *addr, size_t size, size_t alignment, bool *commit); void pages_unmap(void *addr, size_t size); diff --git a/src/arena.c b/src/arena.c index 962a325d0..224a9b63f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1890,8 +1890,8 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) { /* Make sure that b0 thp auto-switch won't happen concurrently here. */ malloc_mutex_lock(tsdn, &b0->mtx); (&huge_arena_pac_thp)->thp_madvise = opt_huge_arena_pac_thp - && metadata_thp_enabled() && (opt_thp == thp_mode_default) - && (init_system_thp_mode == thp_mode_default); + && metadata_thp_enabled() && (opt_thp == thp_mode_do_nothing) + && (init_system_thp_mode == system_thp_mode_madvise); (&huge_arena_pac_thp)->auto_thp_switched = b0->auto_thp_switched; malloc_mutex_init(&(&huge_arena_pac_thp)->lock, "pac_thp", diff --git a/src/base.c b/src/base.c index c494556cb..ef7f0dd45 100644 --- a/src/base.c +++ b/src/base.c @@ -28,8 +28,8 @@ const char *const metadata_thp_mode_names[] = {"disabled", "auto", "always"}; static inline bool metadata_thp_madvise(void) { - return (metadata_thp_enabled() - && (init_system_thp_mode == thp_mode_default)); + return (metadata_thp_enabled() && + (init_system_thp_mode == system_thp_mode_madvise)); } static void * diff --git a/src/pages.c b/src/pages.c index 000b87fef..e7766fcc8 100644 --- a/src/pages.c +++ b/src/pages.c @@ -44,8 +44,10 @@ static bool os_overcommits; const char *const thp_mode_names[] = { "default", "always", "never", "not supported"}; -thp_mode_t opt_thp = THP_MODE_DEFAULT; -thp_mode_t init_system_thp_mode; +const char *const system_thp_mode_names[] = { + "madvise", "always", "never", "not supported"}; +thp_mode_t opt_thp = THP_MODE_DEFAULT; +system_thp_mode_t init_system_thp_mode; /* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */ static bool pages_can_purge_lazy_runtime = true; @@ -778,21 +780,31 @@ os_overcommits_proc(void) { } #endif +static bool +pages_should_skip_set_thp_state() { + if (opt_thp == thp_mode_do_nothing + || (opt_thp == thp_mode_always + && init_system_thp_mode == system_thp_mode_always) + || (opt_thp == thp_mode_never + && init_system_thp_mode == system_thp_mode_never)) { + return true; + } + return false; +} void pages_set_thp_state(void *ptr, size_t size) { - if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) { + if (pages_should_skip_set_thp_state()) { return; } assert(opt_thp != thp_mode_not_supported - && init_system_thp_mode != thp_mode_not_supported); + && init_system_thp_mode != system_thp_mode_not_supported); if (opt_thp == thp_mode_always - && init_system_thp_mode != thp_mode_never) { - assert(init_system_thp_mode == thp_mode_default); + && init_system_thp_mode == system_thp_mode_madvise) { pages_huge_unaligned(ptr, size); } else if (opt_thp == thp_mode_never) { - assert(init_system_thp_mode == thp_mode_default - || init_system_thp_mode == thp_mode_always); + assert(init_system_thp_mode == system_thp_mode_madvise + || init_system_thp_mode == system_thp_mode_always); pages_nohuge_unaligned(ptr, size); } } @@ -825,16 +837,16 @@ init_thp_state(void) { } if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) { - init_system_thp_mode = thp_mode_default; + init_system_thp_mode = system_thp_mode_madvise; } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) { - init_system_thp_mode = thp_mode_always; + init_system_thp_mode = system_thp_mode_always; } else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) { - init_system_thp_mode = thp_mode_never; + init_system_thp_mode = system_thp_mode_never; } else { goto label_error; } if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) { - if (init_system_thp_mode == thp_mode_default) { + if (init_system_thp_mode == system_thp_mode_madvise) { opt_hpa_opts.hugify_style = hpa_hugify_style_lazy; } else { opt_hpa_opts.hugify_style = hpa_hugify_style_none; @@ -842,14 +854,15 @@ init_thp_state(void) { } return; #elif defined(JEMALLOC_HAVE_MEMCNTL) - init_system_thp_mode = thp_mode_default; + init_system_thp_mode = system_thp_mode_madvise; if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) { opt_hpa_opts.hugify_style = hpa_hugify_style_eager; } return; #endif label_error: - opt_thp = init_system_thp_mode = thp_mode_not_supported; + opt_thp = thp_mode_not_supported; + init_system_thp_mode = system_thp_mode_not_supported; } bool diff --git a/src/stats.c b/src/stats.c index ea7a4e2e5..366f96f72 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1516,6 +1516,22 @@ stats_general_print(emitter_t *emitter) { #undef CONFIG_WRITE_BOOL emitter_dict_end(emitter); /* Close "config" dict. */ + /* system. */ + emitter_dict_begin(emitter, "system", "System configuration"); + + /* + * This shows system's THP mode detected at jemalloc's init time. + * jemalloc does not re-detect the mode even if it changes after + * jemalloc's init. It is assumed that system's THP mode is stable + * during the process's lifetime and a violation could lead to + * undefined behavior. + */ + const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode]; + emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string, + &thp_mode_name); + + emitter_dict_end(emitter); /* Close "system". */ + /* opt. */ #define OPT_WRITE(name, var, size, emitter_type) \ if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \ diff --git a/test/unit/pages.c b/test/unit/pages.c index dbee2f0c0..66afb84b0 100644 --- a/test/unit/pages.c +++ b/test/unit/pages.c @@ -10,7 +10,7 @@ TEST_BEGIN(test_pages_huge) { pages = pages_map(NULL, alloc_size, PAGE, &commit); expect_ptr_not_null(pages, "Unexpected pages_map() error"); - if (init_system_thp_mode == thp_mode_default) { + if (init_system_thp_mode == system_thp_mode_madvise) { hugepage = (void *)(ALIGNMENT_CEILING( (uintptr_t)pages, HUGEPAGE)); expect_b_ne(pages_huge(hugepage, HUGEPAGE), have_madvise_huge,