arch/riscv/kernel/ftrace.c - linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright (C) 2013 Linaro Limited
  * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
  * Copyright (C) 2017 Andes Technology Corporation
  */

 #include <linux/ftrace.h>
 #include <linux/uaccess.h>
 #include <linux/memory.h>
 #include <linux/irqflags.h>
 #include <linux/stop_machine.h>
 #include <asm/cacheflush.h>
 #include <asm/text-patching.h>

 #ifdef CONFIG_DYNAMIC_FTRACE
 void ftrace_arch_code_modify_prepare(void)
 	__acquires(&text_mutex)
 {
 	mutex_lock(&text_mutex);
 }

 void ftrace_arch_code_modify_post_process(void)
 	__releases(&text_mutex)
 {
 	mutex_unlock(&text_mutex);
 }

 unsigned long ftrace_call_adjust(unsigned long addr)
 {
 	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
 		return addr + 8 + MCOUNT_AUIPC_SIZE;

 	return addr + MCOUNT_AUIPC_SIZE;
 }

 unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip)
 {
 	return fentry_ip - MCOUNT_AUIPC_SIZE;
 }

 void arch_ftrace_update_code(int command)
 {
 	command |= FTRACE_MAY_SLEEP;
 	ftrace_modify_all_code(command);
 	flush_icache_all();
 }

 static int __ftrace_modify_call(unsigned long source, unsigned long target, bool validate)
 {
 	unsigned int call[2], offset;
 	unsigned int replaced[2];

 	offset = target - source;
 	call[1] = to_jalr_t0(offset);

 	if (validate) {
 		call[0] = to_auipc_t0(offset);
 		/*
 		 * Read the text we want to modify;
 		 * return must be -EFAULT on read error
 		 */
 		if (copy_from_kernel_nofault(replaced, (void *)source, 2 * MCOUNT_INSN_SIZE))
 			return -EFAULT;

 		if (replaced[0] != call[0]) {
 			pr_err("%p: expected (%08x) but got (%08x)\n",
 			       (void *)source, call[0], replaced[0]);
 			return -EINVAL;
 		}
 	}

 	/* Replace the jalr at once. Return -EPERM on write error. */
 	if (patch_insn_write((void *)(source + MCOUNT_AUIPC_SIZE), call + 1, MCOUNT_JALR_SIZE))
 		return -EPERM;

 	return 0;
 }

 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
 static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec)
 {
 	const struct ftrace_ops *ops = NULL;

 	if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
 		ops = ftrace_find_unique_ops(rec);
 		WARN_ON_ONCE(!ops);
 	}

 	if (!ops)
 		ops = &ftrace_list_ops;

 	return ops;
 }

 static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct ftrace_ops *ops)
 {
 	unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);

 	return patch_text_nosync((void *)literal, &ops, sizeof(ops));
 }

 static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec)
 {
 	return ftrace_rec_set_ops(rec, &ftrace_nop_ops);
 }

 static int ftrace_rec_update_ops(struct dyn_ftrace *rec)
 {
 	return ftrace_rec_set_ops(rec, riscv64_rec_get_ops(rec));
 }
 #else
 static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; }
 static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; }
 #endif

 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
 {
 	unsigned long distance, orig_addr, pc = rec->ip - MCOUNT_AUIPC_SIZE;
 	int ret;

 	ret = ftrace_rec_update_ops(rec);
 	if (ret)
 		return ret;

 	orig_addr = (unsigned long)&ftrace_caller;
 	distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;
 	if (distance > JALR_RANGE)
 		addr = FTRACE_ADDR;

 	return __ftrace_modify_call(pc, addr, false);
 }

 int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr)
 {
 	u32 nop4 = RISCV_INSN_NOP4;
 	int ret;

 	ret = ftrace_rec_set_nop_ops(rec);
 	if (ret)
 		return ret;

 	if (patch_insn_write((void *)rec->ip, &nop4, MCOUNT_NOP4_SIZE))
 		return -EPERM;

 	return 0;
 }

 /*
  * This is called early on, and isn't wrapped by
  * ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
  * text_mutex, which triggers a lockdep failure.  SMP isn't running so we could
  * just directly poke the text, but it's simpler to just take the lock
  * ourselves.
  */
 int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec)
 {
 	unsigned long pc = rec->ip - MCOUNT_AUIPC_SIZE;
 	unsigned int nops[2], offset;
 	int ret;

 	guard(mutex)(&text_mutex);

 	ret = ftrace_rec_set_nop_ops(rec);
 	if (ret)
 		return ret;

 	offset = (unsigned long) &ftrace_caller - pc;
 	nops[0] = to_auipc_t0(offset);
 	nops[1] = RISCV_INSN_NOP4;

 	ret = patch_insn_write((void *)pc, nops, 2 * MCOUNT_INSN_SIZE);

 	return ret;
 }

 ftrace_func_t ftrace_call_dest = ftrace_stub;
 int ftrace_update_ftrace_func(ftrace_func_t func)
 {
 	/*
 	 * When using CALL_OPS, the function to call is associated with the
 	 * call site, and we don't have a global function pointer to update.
 	 */
 	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
 		return 0;

 	WRITE_ONCE(ftrace_call_dest, func);
 	/*
 	 * The data fence ensure that the update to ftrace_call_dest happens
 	 * before the write to function_trace_op later in the generic ftrace.
 	 * If the sequence is not enforced, then an old ftrace_call_dest may
 	 * race loading a new function_trace_op set in ftrace_modify_all_code
 	 */
 	smp_wmb();
 	/*
 	 * Updating ftrace dpes not take stop_machine path, so irqs should not
 	 * be disabled.
 	 */
 	WARN_ON(irqs_disabled());
 	smp_call_function(ftrace_sync_ipi, NULL, 1);
 	return 0;
 }

 #else /* CONFIG_DYNAMIC_FTRACE */
 unsigned long ftrace_call_adjust(unsigned long addr)
 {
 	return addr;
 }
 #endif /* CONFIG_DYNAMIC_FTRACE */

 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
 int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
 		       unsigned long addr)
 {
 	unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE;
 	int ret;

 	ret = ftrace_rec_update_ops(rec);
 	if (ret)
 		return ret;

 	return __ftrace_modify_call(caller, FTRACE_ADDR, true);
 }
 #endif

 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 /*
  * Most of this function is copied from arm64.
  */
 void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
 			   unsigned long frame_pointer)
 {
 	unsigned long return_hooker = (unsigned long)&return_to_handler;
 	unsigned long old;

 	if (unlikely(atomic_read(&current->tracing_graph_pause)))
 		return;

 	/*
 	 * We don't suffer access faults, so no extra fault-recovery assembly
 	 * is needed here.
 	 */
 	old = *parent;

 	if (!function_graph_enter(old, self_addr, frame_pointer, parent))
 		*parent = return_hooker;
 }

 #ifdef CONFIG_DYNAMIC_FTRACE
 void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
 		       struct ftrace_ops *op, struct ftrace_regs *fregs)
 {
 	unsigned long return_hooker = (unsigned long)&return_to_handler;
 	unsigned long frame_pointer = arch_ftrace_regs(fregs)->s0;
 	unsigned long *parent = &arch_ftrace_regs(fregs)->ra;
 	unsigned long old;

 	if (unlikely(atomic_read(&current->tracing_graph_pause)))
 		return;

 	/*
 	 * We don't suffer access faults, so no extra fault-recovery assembly
 	 * is needed here.
 	 */
 	old = *parent;

 	if (!function_graph_enter_regs(old, ip, frame_pointer, parent, fregs))
 		*parent = return_hooker;
 }
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright (C) 2013 Linaro Limited
	* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
	* Copyright (C) 2017 Andes Technology Corporation
	*/

	#include <linux/ftrace.h>
	#include <linux/uaccess.h>
	#include <linux/memory.h>
	#include <linux/irqflags.h>
	#include <linux/stop_machine.h>
	#include <asm/cacheflush.h>
	#include <asm/text-patching.h>

	#ifdef CONFIG_DYNAMIC_FTRACE
	void ftrace_arch_code_modify_prepare(void)
	__acquires(&text_mutex)
	{
	mutex_lock(&text_mutex);
	}

	void ftrace_arch_code_modify_post_process(void)
	__releases(&text_mutex)
	{
	mutex_unlock(&text_mutex);
	}

	unsigned long ftrace_call_adjust(unsigned long addr)
	{
	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
	return addr + 8 + MCOUNT_AUIPC_SIZE;

	return addr + MCOUNT_AUIPC_SIZE;
	}

	unsigned long arch_ftrace_get_symaddr(unsigned long fentry_ip)
	{
	return fentry_ip - MCOUNT_AUIPC_SIZE;
	}

	void arch_ftrace_update_code(int command)
	{
	command \|= FTRACE_MAY_SLEEP;
	ftrace_modify_all_code(command);
	flush_icache_all();
	}

	static int __ftrace_modify_call(unsigned long source, unsigned long target, bool validate)
	{
	unsigned int call[2], offset;
	unsigned int replaced[2];

	offset = target - source;
	call[1] = to_jalr_t0(offset);

	if (validate) {
	call[0] = to_auipc_t0(offset);
	/*
	* Read the text we want to modify;
	* return must be -EFAULT on read error
	*/
	if (copy_from_kernel_nofault(replaced, (void )source, 2 MCOUNT_INSN_SIZE))
	return -EFAULT;

	if (replaced[0] != call[0]) {
	pr_err("%p: expected (%08x) but got (%08x)\n",
	(void *)source, call[0], replaced[0]);
	return -EINVAL;
	}
	}

	/* Replace the jalr at once. Return -EPERM on write error. */
	if (patch_insn_write((void *)(source + MCOUNT_AUIPC_SIZE), call + 1, MCOUNT_JALR_SIZE))
	return -EPERM;

	return 0;
	}

	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
	static const struct ftrace_ops riscv64_rec_get_ops(struct dyn_ftrace rec)
	{
	const struct ftrace_ops *ops = NULL;

	if (rec->flags & FTRACE_FL_CALL_OPS_EN) {
	ops = ftrace_find_unique_ops(rec);
	WARN_ON_ONCE(!ops);
	}

	if (!ops)
	ops = &ftrace_list_ops;

	return ops;
	}

	static int ftrace_rec_set_ops(const struct dyn_ftrace rec, const struct ftrace_ops ops)
	{
	unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);

	return patch_text_nosync((void *)literal, &ops, sizeof(ops));
	}

	static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec)
	{
	return ftrace_rec_set_ops(rec, &ftrace_nop_ops);
	}

	static int ftrace_rec_update_ops(struct dyn_ftrace *rec)
	{
	return ftrace_rec_set_ops(rec, riscv64_rec_get_ops(rec));
	}
	#else
	static int ftrace_rec_set_nop_ops(struct dyn_ftrace *rec) { return 0; }
	static int ftrace_rec_update_ops(struct dyn_ftrace *rec) { return 0; }
	#endif

	int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
	{
	unsigned long distance, orig_addr, pc = rec->ip - MCOUNT_AUIPC_SIZE;
	int ret;

	ret = ftrace_rec_update_ops(rec);
	if (ret)
	return ret;

	orig_addr = (unsigned long)&ftrace_caller;
	distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;
	if (distance > JALR_RANGE)
	addr = FTRACE_ADDR;

	return __ftrace_modify_call(pc, addr, false);
	}

	int ftrace_make_nop(struct module mod, struct dyn_ftrace rec, unsigned long addr)
	{
	u32 nop4 = RISCV_INSN_NOP4;
	int ret;

	ret = ftrace_rec_set_nop_ops(rec);
	if (ret)
	return ret;

	if (patch_insn_write((void *)rec->ip, &nop4, MCOUNT_NOP4_SIZE))
	return -EPERM;

	return 0;
	}

	/*
	* This is called early on, and isn't wrapped by
	* ftrace_arch_code_modify_{prepare,post_process}() and therefor doesn't hold
	* text_mutex, which triggers a lockdep failure. SMP isn't running so we could
	* just directly poke the text, but it's simpler to just take the lock
	* ourselves.
	*/
	int ftrace_init_nop(struct module mod, struct dyn_ftrace rec)
	{
	unsigned long pc = rec->ip - MCOUNT_AUIPC_SIZE;
	unsigned int nops[2], offset;
	int ret;

	guard(mutex)(&text_mutex);

	ret = ftrace_rec_set_nop_ops(rec);
	if (ret)
	return ret;

	offset = (unsigned long) &ftrace_caller - pc;
	nops[0] = to_auipc_t0(offset);
	nops[1] = RISCV_INSN_NOP4;

	ret = patch_insn_write((void )pc, nops, 2 MCOUNT_INSN_SIZE);

	return ret;
	}

	ftrace_func_t ftrace_call_dest = ftrace_stub;
	int ftrace_update_ftrace_func(ftrace_func_t func)
	{
	/*
	* When using CALL_OPS, the function to call is associated with the
	* call site, and we don't have a global function pointer to update.
	*/
	if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
	return 0;

	WRITE_ONCE(ftrace_call_dest, func);
	/*
	* The data fence ensure that the update to ftrace_call_dest happens
	* before the write to function_trace_op later in the generic ftrace.
	* If the sequence is not enforced, then an old ftrace_call_dest may
	* race loading a new function_trace_op set in ftrace_modify_all_code
	*/
	smp_wmb();
	/*
	* Updating ftrace dpes not take stop_machine path, so irqs should not
	* be disabled.
	*/
	WARN_ON(irqs_disabled());
	smp_call_function(ftrace_sync_ipi, NULL, 1);
	return 0;
	}

	#else /* CONFIG_DYNAMIC_FTRACE */
	unsigned long ftrace_call_adjust(unsigned long addr)
	{
	return addr;
	}
	#endif /* CONFIG_DYNAMIC_FTRACE */

	#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
	int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
	unsigned long addr)
	{
	unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE;
	int ret;

	ret = ftrace_rec_update_ops(rec);
	if (ret)
	return ret;

	return __ftrace_modify_call(caller, FTRACE_ADDR, true);
	}
	#endif

	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
	/*
	* Most of this function is copied from arm64.
	*/
	void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
	unsigned long frame_pointer)
	{
	unsigned long return_hooker = (unsigned long)&return_to_handler;
	unsigned long old;

	if (unlikely(atomic_read(&current->tracing_graph_pause)))
	return;

	/*
	* We don't suffer access faults, so no extra fault-recovery assembly
	* is needed here.
	*/
	old = *parent;

	if (!function_graph_enter(old, self_addr, frame_pointer, parent))
	*parent = return_hooker;
	}

	#ifdef CONFIG_DYNAMIC_FTRACE
	void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
	struct ftrace_ops op, struct ftrace_regs fregs)
	{
	unsigned long return_hooker = (unsigned long)&return_to_handler;
	unsigned long frame_pointer = arch_ftrace_regs(fregs)->s0;
	unsigned long *parent = &arch_ftrace_regs(fregs)->ra;
	unsigned long old;

	if (unlikely(atomic_read(&current->tracing_graph_pause)))
	return;

	/*
	* We don't suffer access faults, so no extra fault-recovery assembly
	* is needed here.
	*/
	old = *parent;

	if (!function_graph_enter_regs(old, ip, frame_pointer, parent, fregs))
	*parent = return_hooker;
	}
	#endif /* CONFIG_DYNAMIC_FTRACE */
	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */