blob: f811bb5c42a1deee4fbb894d732327676fdb5403 [file]
// SPDX-License-Identifier: GPL-2.0
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/clockchips.h>
#include <linux/slab.h>
#include <linux/cpuhotplug.h>
#include <linux/minmax.h>
#include <linux/export.h>
#include <asm/mshyperv.h>
#define HV_DEPOSIT_MAX 512
#define HV_DEPOSIT_INP_MAX ((HV_HYP_PAGE_SIZE - \
offsetof(struct hv_deposit_memory, gpa_page_list)) / sizeof(u64))
/*
* Allocate free pages for deposit to hypervisor. pfna[] must be large enough
* to hold HV_DEPOSIT_INP_MAX (511) pages. If num_pages is 512, return last
* pfn in lastpfn. If @single, then it must be a single allocation (not split
* over multiple contiguous ranges).
*
* Returns: number of pages allocated or -ENOMEM
*/
static int hv_alloc_dep_pages(int node, u64 *pfna, u64 *lastpfnp, int num_pages,
bool single)
{
struct page *page;
int num_allocd, count = 0;
/* Published ABI, enforce its immutability. */
BUILD_BUG_ON(HV_DEPOSIT_INP_MAX != 511);
if (num_pages > HV_DEPOSIT_MAX ||
(num_pages == HV_DEPOSIT_MAX && lastpfnp == NULL))
return -EINVAL;
while (num_pages) {
/* Find highest order we can actually allocate */
int order = 31 - __builtin_clz(num_pages);
gfp_t gfp_flags = GFP_KERNEL;
if (!single)
gfp_flags |= __GFP_NOWARN;
while (1) {
page = alloc_pages_node(node, gfp_flags, order);
if (page || order == 0 || single)
break;
order--;
}
if (page == NULL)
break;
split_page(page, order);
num_allocd = 1 << order;
num_pages -= num_allocd;
while (num_allocd && count < HV_DEPOSIT_INP_MAX) {
pfna[count++] = page_to_pfn(page++);
num_allocd--;
}
if (num_allocd-- && count == HV_DEPOSIT_INP_MAX) {
*lastpfnp = page_to_pfn(page);
count++;
break;
}
}
return count ? count : -ENOMEM;
}
/*
* Deposit memory in the hypervisor. Even if @contiguous is false, a contiguous
* 2M worth of pfns is utmost desired for performance reasons. But short of
* that, we deposit whatever contiguous chunks we can get. If @contiguous is
* true, then the entire range has to be physically contiguous. Note, in that
* case, upon withdrawl, hypervisor could return any page in between the range,
* so we must split that also. Lastly, HV_MAX_CONTIGUOUS_ALLOCATION_PAGES is
* not guaranteed to always be power of 2.
*/
static int hv_call_deposit_pages(int node, u64 partition_id, bool contiguous)
{
struct hv_deposit_memory *hc_input;
int i, rc, num_pages;
u64 status, *pfna, lastpfn = 0;
bool trunc_extra = false;
BUILD_BUG_ON(HV_MAX_CONTIGUOUS_ALLOCATION_PAGES > HV_DEPOSIT_MAX);
if (contiguous) {
num_pages = roundup_pow_of_two(
HV_MAX_CONTIGUOUS_ALLOCATION_PAGES);
trunc_extra = num_pages != HV_MAX_CONTIGUOUS_ALLOCATION_PAGES;
} else {
num_pages = HV_DEPOSIT_MAX;
}
hc_input = (struct hv_deposit_memory *)get_zeroed_page(GFP_KERNEL);
if (hc_input == NULL)
return -ENOMEM;
hc_input->partition_id = partition_id;
pfna = hc_input->gpa_page_list;
rc = hv_alloc_dep_pages(node, pfna, &lastpfn, num_pages, contiguous);
if (rc < 0)
goto out_free;
num_pages = rc;
if (num_pages > HV_DEPOSIT_INP_MAX)
num_pages = HV_DEPOSIT_INP_MAX;
if (contiguous && trunc_extra) {
for (i = HV_MAX_CONTIGUOUS_ALLOCATION_PAGES; i < num_pages; i++)
__free_page(pfn_to_page(pfna[i]));
if (lastpfn) {
__free_page(pfn_to_page(lastpfn));
lastpfn = 0;
}
num_pages = HV_MAX_CONTIGUOUS_ALLOCATION_PAGES;
}
/* Not using hyperv_pcpu_input_arg, so no need to disable interrupts */
status = hv_do_rep_hypercall(HVCALL_DEPOSIT_MEMORY, num_pages, 0,
hc_input, NULL);
if (!hv_result_success(status))
goto err_free_dep_pages;
if (lastpfn) {
hc_input->gpa_page_list[0] = lastpfn;
status = hv_do_rep_hypercall(HVCALL_DEPOSIT_MEMORY, 1, 0,
hc_input, NULL);
if (!hv_result_success(status) && hv_repcomp(status) == 0)
/* We deposited lot earlier, so give it a go */
__free_page(pfn_to_page(lastpfn));
}
free_page((unsigned long)hc_input);
return 0;
err_free_dep_pages:
hv_status_err(status, "\n");
rc = hv_result_to_errno(status);
for (i = hv_repcomp(status); i < num_pages; i++)
__free_page(pfn_to_page(pfna[i]));
if (lastpfn)
__free_page(pfn_to_page(lastpfn));
out_free:
free_page((unsigned long)hc_input);
return rc;
}
int hv_deposit_memory_node(int node, u64 pt_id, u64 hv_status)
{
int result = hv_result(hv_status);
bool contiguous = false;
if (result == HV_STATUS_INSUFFICIENT_ROOT_MEMORY ||
result == HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY) {
if (!hv_root_partition()) {
hv_status_err(hv_status,
"Unexpected root memory deposit\n");
return -EINVAL;
}
pt_id = HV_PARTITION_ID_SELF;
}
if (result == HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY ||
result == HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY)
contiguous = true;
return hv_call_deposit_pages(node, pt_id, contiguous);
}
EXPORT_SYMBOL_GPL(hv_deposit_memory_node);
bool hv_result_needs_memory(u64 status)
{
switch (hv_result(status)) {
case HV_STATUS_INSUFFICIENT_MEMORY:
case HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY:
case HV_STATUS_INSUFFICIENT_ROOT_MEMORY:
case HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY:
return true;
}
return false;
}
EXPORT_SYMBOL_GPL(hv_result_needs_memory);
int hv_call_add_logical_proc(int node, u32 lp_index, u32 apic_id)
{
struct hv_input_add_logical_processor *input;
struct hv_output_add_logical_processor *output;
u64 status;
unsigned long flags;
int ret = 0;
/*
* When adding a logical processor, the hypervisor may return
* HV_STATUS_INSUFFICIENT_MEMORY. When that happens, we deposit more
* pages and retry.
*/
do {
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
/* We don't do anything with the output right now */
output = *this_cpu_ptr(hyperv_pcpu_output_arg);
input->lp_index = lp_index;
input->apic_id = apic_id;
input->proximity_domain_info = hv_numa_node_to_pxm_info(node);
status = hv_do_hypercall(HVCALL_ADD_LOGICAL_PROCESSOR,
input, output);
local_irq_restore(flags);
if (!hv_result_needs_memory(status)) {
if (!hv_result_success(status)) {
hv_status_err(status, "cpu %u apic ID: %u\n",
lp_index, apic_id);
ret = hv_result_to_errno(status);
}
break;
}
ret = hv_deposit_memory_node(node, hv_current_partition_id,
status);
} while (!ret);
return ret;
}
int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags)
{
struct hv_create_vp *input;
u64 status;
unsigned long irq_flags;
int ret = 0;
do {
local_irq_save(irq_flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
input->partition_id = partition_id;
input->vp_index = vp_index;
input->flags = flags;
input->subnode_type = HV_SUBNODE_ANY;
input->proximity_domain_info = hv_numa_node_to_pxm_info(node);
status = hv_do_hypercall(HVCALL_CREATE_VP, input, NULL);
local_irq_restore(irq_flags);
if (!hv_result_needs_memory(status)) {
if (!hv_result_success(status)) {
hv_status_err(status, "vcpu: %u, lp: %u\n",
vp_index, flags);
ret = hv_result_to_errno(status);
}
break;
}
ret = hv_deposit_memory_node(node, partition_id, status);
} while (!ret);
return ret;
}
EXPORT_SYMBOL_GPL(hv_call_create_vp);
int hv_call_notify_all_processors_started(void)
{
struct hv_input_notify_partition_event *input;
u64 status;
unsigned long irq_flags;
int ret = 0;
local_irq_save(irq_flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
memset(input, 0, sizeof(*input));
input->event = HV_PARTITION_ALL_LOGICAL_PROCESSORS_STARTED;
status = hv_do_hypercall(HVCALL_NOTIFY_PARTITION_EVENT,
input, NULL);
local_irq_restore(irq_flags);
if (!hv_result_success(status)) {
hv_status_err(status, "\n");
ret = hv_result_to_errno(status);
}
return ret;
}
bool hv_lp_exists(u32 lp_index)
{
struct hv_input_get_logical_processor_run_time *input;
struct hv_output_get_logical_processor_run_time *output;
unsigned long flags;
u64 status;
local_irq_save(flags);
input = *this_cpu_ptr(hyperv_pcpu_input_arg);
output = *this_cpu_ptr(hyperv_pcpu_output_arg);
input->lp_index = lp_index;
status = hv_do_hypercall(HVCALL_GET_LOGICAL_PROCESSOR_RUN_TIME,
input, output);
local_irq_restore(flags);
if (!hv_result_success(status) &&
hv_result(status) != HV_STATUS_INVALID_LP_INDEX) {
hv_status_err(status, "\n");
BUG();
}
return hv_result_success(status);
}