blob: 28a9b1ff5d44743e7e7ff771c3bf3c5b5be19f95 [file] [edit]
// SPDX-License-Identifier: GPL-2.0
/*
* Check that khugepaged_full_pass() drives khugepaged in step: one barrier
* over one prepared window must collapse it with exactly one collapse
* attempt attributed to its source pages, step after step.
*
* scan_sleep_millisecs is a minute so that a step which slept instead of
* being woken blows the budget.
*/
#define _GNU_SOURCE
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>
#include "kselftest.h"
#include "vm_util.h"
#include <mm/hugepage_settings.h>
#define BASE_ADDR ((void *)(1UL << 30))
/* Smallest order khugepaged considers */
#define TARGET_ORDER 2
#define NR_ITERATIONS 5
#define PASS_TIMEOUT_S 30
static int pagemap_fd;
static int kpageflags_fd;
static int trace_events_fd = -1;
static unsigned long hpage_pmd_size;
/*
* The events are system-wide: switch them off however the test ends,
* including from inside a helper that gives up.
*/
static void trace_events_off(void)
{
if (trace_events_fd >= 0)
tracing_events_enable(trace_events_fd, false);
}
/* Count the isolate events whose scan_pfn is one of the window's source PFNs */
static int count_attributed(unsigned long *pfns, int nr_pfns,
unsigned int order)
{
char line[1024];
int count = 0;
FILE *fp;
fp = tracing_open_trace();
if (!fp)
ksft_exit_fail_msg("Cannot open trace buffer\n");
while (fgets(line, sizeof(line), fp)) {
unsigned long val;
unsigned int ord;
char *s, *o;
int i;
s = strstr(line, "mm_collapse_huge_page_isolate:");
if (!s)
continue;
if (sscanf(s, "mm_collapse_huge_page_isolate: scan_pfn=0x%lx",
&val) != 1)
continue;
o = strstr(s, "order=");
if (!o || sscanf(o, "order=%u", &ord) != 1 || ord != order)
continue;
for (i = 0; i < nr_pfns; i++) {
if (val == pfns[i]) {
count++;
break;
}
}
}
fclose(fp);
return count;
}
static void one_step(int iteration)
{
const size_t window = getpagesize() << TARGET_ORDER;
const int nr_pages = 1 << TARGET_ORDER;
unsigned long pfns[1 << TARGET_ORDER];
bool collapsed, passed;
int attributed;
char *p;
int i;
p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE | MAP_FIXED_NOREPLACE, -1, 0);
if (p != BASE_ADDR)
ksft_exit_fail_perror("mmap() window");
for (i = 0; i < nr_pages; i++) {
p[i * getpagesize()] = i + 1;
pfns[i] = pagemap_get_pfn(pagemap_fd, p + i * getpagesize());
if (pfns[i] == -1UL)
ksft_exit_fail_msg("Source page not present\n");
}
/* Clear before enabling so the buffer holds only this step's events */
if (tracing_clear_trace())
ksft_exit_fail_msg("Cannot clear the trace buffer\n");
if (tracing_events_enable(trace_events_fd, true))
ksft_exit_fail_msg("Cannot enable huge_memory events\n");
if (madvise(p, hpage_pmd_size, MADV_HUGEPAGE))
ksft_exit_fail_perror("madvise(MADV_HUGEPAGE)");
passed = khugepaged_full_pass(PASS_TIMEOUT_S);
/* Off before anything that can give up: the events are system-wide */
if (tracing_events_enable(trace_events_fd, false))
ksft_exit_fail_msg("Cannot disable huge_memory events\n");
if (!passed)
ksft_exit_fail_msg("khugepaged did not complete a full pass\n");
collapsed = is_range_backed_by_order(p, window, TARGET_ORDER,
pagemap_fd, kpageflags_fd);
attributed = count_attributed(pfns, nr_pages, TARGET_ORDER);
ksft_test_result(collapsed && attributed == 1,
"step %d: window collapsed, %d attributed result(s)\n",
iteration, attributed);
munmap(p, hpage_pmd_size);
}
int main(void)
{
struct thp_settings settings;
int i;
ksft_print_header();
if (!thp_available())
ksft_exit_skip("Transparent Hugepages not available\n");
if (!(thp_supported_orders() & (1UL << TARGET_ORDER)))
ksft_exit_skip("Order %d is not a supported anon THP order\n",
TARGET_ORDER);
hpage_pmd_size = read_pmd_pagesize();
if (!hpage_pmd_size)
ksft_exit_fail_msg("Reading PMD pagesize failed\n");
pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
if (pagemap_fd < 0)
ksft_exit_fail_perror("open(/proc/self/pagemap)");
kpageflags_fd = open("/proc/kpageflags", O_RDONLY);
if (kpageflags_fd < 0)
ksft_exit_skip("open(/proc/kpageflags) requires root\n");
trace_events_fd = tracing_events_open("huge_memory");
if (trace_events_fd < 0)
ksft_exit_skip("huge_memory events require tracefs and root\n");
atexit(trace_events_off);
ksft_set_plan(NR_ITERATIONS);
thp_save_settings();
thp_read_settings(&settings);
settings.thp_enabled = THP_MADVISE;
settings.thp_defrag = THP_DEFRAG_ALWAYS;
settings.khugepaged.defrag = 1;
settings.khugepaged.scan_sleep_millisecs = 60 * 1000;
settings.khugepaged.alloc_sleep_millisecs = 60 * 1000;
settings.khugepaged.max_ptes_none = (hpage_pmd_size / getpagesize()) - 1;
/* One wake must complete one full pass; see khugepaged_full_pass() */
settings.khugepaged.pages_to_scan = 1UL << 24;
for (i = 0; i < NR_ORDERS; i++)
settings.hugepages[i].enabled = THP_NEVER;
settings.hugepages[TARGET_ORDER].enabled = THP_INHERIT;
/* Base of the settings stack; the bottom entry is never popped */
thp_push_settings(&settings);
for (i = 0; i < NR_ITERATIONS; i++)
one_step(i);
ksft_finished();
}