Memory: Swap probe_read to kernel or user version

bpf/cgroup/bpf_cgroup_events.h

@@ -49,7 +49,7 @@ send_cgrp_event(struct bpf_raw_tracepoint_args *ctx,
 	msg->cgrp_data.level = cgrp_track->level;
 	msg->cgrp_data.hierarchy_id = cgrp_track->hierarchy_id;
 	memcpy(&msg->cgrp_data.name, &cgrp_track->name, KN_NAME_LENGTH);
-	probe_read_str(&msg->path, PATH_MAP_SIZE - 1, path);
+	probe_read_kernel_str(&msg->path, PATH_MAP_SIZE - 1, path);
 
 	perf_event_output_metric(ctx, MSG_OP_CGROUP, &tcpmon_map, BPF_F_CURRENT_CPU, msg, size);
 

bpf/include/api.h

@@ -204,6 +204,9 @@ static int BPF_FUNC(fib_lookup, void *ctx, struct bpf_fib_lookup *params, uint32
 static int BPF_FUNC(probe_read, void *dst, uint32_t size, const void *src);
 static int BPF_FUNC(probe_read_str, void *dst, int size, const void *src);
 static int BPF_FUNC(probe_read_kernel, void *dst, uint32_t size, const void *src);
+static int BPF_FUNC(probe_read_kernel_str, void *dst, int size, const void *src);
+static int BPF_FUNC(probe_read_user, void *dst, uint32_t size, const void *src);
+static int BPF_FUNC(probe_read_user_str, void *dst, int size, const void *src);
 
 static uint64_t BPF_FUNC(get_current_task);
 

bpf/lib/bpf_cgroup.h

@@ -109,7 +109,7 @@ __get_cgroup_kn_name(const struct kernfs_node *kn)
 	const char *name = NULL;
 
 	if (kn)
-		probe_read(&name, sizeof(name), _(&kn->name));
+		probe_read_kernel(&name, sizeof(name), _(&kn->name));
 
 	return name;
 }
@@ -139,7 +139,7 @@ __get_cgroup_kn_id(const struct kernfs_node *kn)
 		if (BPF_CORE_READ_INTO(&id, old_kn, id.id) != 0)
 			return 0;
 	} else {
-		probe_read(&id, sizeof(id), _(&kn->id));
+		probe_read_kernel(&id, sizeof(id), _(&kn->id));
 	}
 
 	return id;
@@ -157,7 +157,7 @@ __get_cgroup_kn(const struct cgroup *cgrp)
 	struct kernfs_node *kn = NULL;
 
 	if (cgrp)
-		probe_read(&kn, sizeof(cgrp->kn), _(&cgrp->kn));
+		probe_read_kernel(&kn, sizeof(cgrp->kn), _(&cgrp->kn));
 
 	return kn;
 }
@@ -187,7 +187,7 @@ get_cgroup_hierarchy_id(const struct cgroup *cgrp)
  * @cgrp: target cgroup
  *
  * Returns a pointer to the cgroup node name on success that can
- * be read with probe_read(). NULL on failures.
+ * be read with probe_read_kernel(). NULL on failures.
  */
 static inline __attribute__((always_inline)) const char *
 get_cgroup_name(const struct cgroup *cgrp)
@@ -214,7 +214,7 @@ get_cgroup_level(const struct cgroup *cgrp)
 {
 	__u32 level = 0;
 
-	probe_read(&level, sizeof(level), _(&cgrp->level));
+	probe_read_kernel(&level, sizeof(level), _(&cgrp->level));
 	return level;
 }
 
@@ -264,7 +264,7 @@ get_task_cgroup(struct task_struct *task, __u32 subsys_idx, __u32 *error_flags)
 	struct css_set *cgroups;
 	struct cgroup *cgrp = NULL;
 
-	probe_read(&cgroups, sizeof(cgroups), _(&task->cgroups));
+	probe_read_kernel(&cgroups, sizeof(cgroups), _(&task->cgroups));
 	if (unlikely(!cgroups)) {
 		*error_flags |= EVENT_ERROR_CGROUPS;
 		return cgrp;
@@ -297,13 +297,13 @@ get_task_cgroup(struct task_struct *task, __u32 subsys_idx, __u32 *error_flags)
 	 * support as much as workload as possible. It also reduces errors
 	 * in a significant way.
 	 */
-	probe_read(&subsys, sizeof(subsys), _(&cgroups->subsys[subsys_idx]));
+	probe_read_kernel(&subsys, sizeof(subsys), _(&cgroups->subsys[subsys_idx]));
 	if (unlikely(!subsys)) {
 		*error_flags |= EVENT_ERROR_CGROUP_SUBSYS;
 		return cgrp;
 	}
 
-	probe_read(&cgrp, sizeof(cgrp), _(&subsys->cgroup));
+	probe_read_kernel(&cgrp, sizeof(cgrp), _(&subsys->cgroup));
 	if (!cgrp)
 		*error_flags |= EVENT_ERROR_CGROUP_SUBSYSCGRP;
 
@@ -426,7 +426,7 @@ __init_cgrp_tracking_val_heap(struct cgroup *cgrp, cgroup_state state)
 	kn = __get_cgroup_kn(cgrp);
 	name = __get_cgroup_kn_name(kn);
 	if (name)
-		probe_read_str(&heap->name, KN_NAME_LENGTH - 1, name);
+		probe_read_kernel_str(&heap->name, KN_NAME_LENGTH - 1, name);
 
 	return heap;
 }

bpf/lib/bpf_task.h

@@ -7,6 +7,7 @@
 #include "bpf_event.h"
 #include "bpf_helpers.h"
 #include "generic.h"
+#include "bpf_tracing.h"
 
 /* __d_path_local flags */
 // #define UNRESOLVED_MOUNT_POINTS	   0x01 // (deprecated)
@@ -27,7 +28,7 @@ get_parent(struct task_struct *t)
 	struct task_struct *task;
 
 	/* Read the real parent */
-	probe_read(&task, sizeof(task), _(&t->real_parent));
+	probe_read_kernel(&task, sizeof(task), _(&t->real_parent));
 	if (!task)
 		return 0;
 	return task;
@@ -47,7 +48,7 @@ get_task_from_pid(__u32 pid)
 			i = TASK_PID_LOOP;
 			continue;
 		}
-		probe_read(&cpid, sizeof(cpid), _(&task->tgid));
+		probe_read_kernel(&cpid, sizeof(cpid), _(&task->tgid));
 		if (cpid == pid) {
 			i = TASK_PID_LOOP;
 			continue;
@@ -70,7 +71,7 @@ static inline __attribute__((always_inline)) __u32 get_task_pid_vnr(void)
 
 	thread_pid_exists = bpf_core_field_exists(task->thread_pid);
 	if (thread_pid_exists) {
-		probe_read(&pid, sizeof(pid), _(&task->thread_pid));
+		probe_read_kernel(&pid, sizeof(pid), _(&task->thread_pid));
 		if (!pid)
 			return 0;
 	} else {
@@ -85,16 +86,16 @@ static inline __attribute__((always_inline)) __u32 get_task_pid_vnr(void)
 		if (!thread_pid_exists)
 			link_sz =
 				24; // voodoo magic, hard-code 24 to init stack
-		probe_read(&link, link_sz,
-			   (void *)_(&task->pids) + (PIDTYPE_PID * link_sz));
+		probe_read_kernel(&link, link_sz,
+				  (void *)_(&task->pids) + (PIDTYPE_PID * link_sz));
 		pid = link.pid;
 	}
 	upid_sz = bpf_core_field_size(pid->numbers[0]);
-	probe_read(&level, sizeof(level), _(&pid->level));
+	probe_read_kernel(&level, sizeof(level), _(&pid->level));
 	if (level < 1)
 		return 0;
-	probe_read(&upid, upid_sz,
-		   (void *)_(&pid->numbers) + (level * upid_sz));
+	probe_read_kernel(&upid, upid_sz,
+			  (void *)_(&pid->numbers) + (level * upid_sz));
 	return upid.nr;
 }
 
@@ -106,7 +107,7 @@ event_find_parent_pid(struct task_struct *t)
 
 	if (!task)
 		return 0;
-	probe_read(&pid, sizeof(pid), _(&task->tgid));
+	probe_read_kernel(&pid, sizeof(pid), _(&task->tgid));
 	return pid;
 }
 
@@ -119,10 +120,10 @@ __event_find_parent(struct task_struct *task)
 
 #pragma unroll
 	for (i = 0; i < 4; i++) {
-		probe_read(&task, sizeof(task), _(&task->real_parent));
+		probe_read_kernel(&task, sizeof(task), _(&task->real_parent));
 		if (!task)
 			break;
-		probe_read(&pid, sizeof(pid), _(&task->tgid));
+		probe_read_kernel(&pid, sizeof(pid), _(&task->tgid));
 		value = execve_map_get_noinit(pid);
 		if (value && value->key.ktime != 0)
 			return value;
@@ -164,13 +165,13 @@ event_find_curr(__u32 *ppid, bool *walked)
 
 #pragma unroll
 	for (i = 0; i < 4; i++) {
-		probe_read(&pid, sizeof(pid), _(&task->tgid));
+		probe_read_kernel(&pid, sizeof(pid), _(&task->tgid));
 		value = execve_map_get_noinit(pid);
 		if (value && value->key.ktime != 0)
 			break;
 		value = 0;
 		*walked = 1;
-		probe_read(&task, sizeof(task), _(&task->real_parent));
+		probe_read_kernel(&task, sizeof(task), _(&task->real_parent));
 		if (!task)
 			break;
 	}

bpf/lib/process.h

@@ -51,7 +51,7 @@
  * Now we want to read this with call 45 aka probe_read_str as follows,
  * where 'kernel_struct_arg' is the kernel data struct we are reading.
  *
- *   probe_read_str(args[offset], size, kernel_struct_arg)
+ *   probe_read_kernel_str(args[offset], size, kernel_struct_arg)
  *
  * But we have a bit of a problem determining if 'size' is out of array
  * range. The math would be,

bpf/libbpf/bpf_tracing.h

@@ -2,6 +2,8 @@
 #ifndef __BPF_TRACING_H__
 #define __BPF_TRACING_H__
 
+#include "bpf_core_read.h"
+
 /* Scan the ARCH passed in from ARCH env variable (see Makefile) */
 #if defined(__TARGET_ARCH_x86)
 	#define bpf_target_x86

bpf/process/bpf_execve_event.c

@@ -41,13 +41,13 @@ read_args(void *ctx, struct msg_execve_event *event)
 	long off;
 	int err;
 
-	probe_read(&mm, sizeof(mm), _(&task->mm));
+	probe_read_kernel(&mm, sizeof(mm), _(&task->mm));
 	if (!mm)
 		return 0;
 
-	probe_read(&start_stack, sizeof(start_stack),
-		   _(&mm->arg_start));
-	probe_read(&end_stack, sizeof(start_stack), _(&mm->arg_end));
+	probe_read_kernel(&start_stack, sizeof(start_stack),
+			  _(&mm->arg_start));
+	probe_read_kernel(&end_stack, sizeof(start_stack), _(&mm->arg_end));
 
 	if (!start_stack || !end_stack)
 		return 0;
@@ -58,7 +58,7 @@ read_args(void *ctx, struct msg_execve_event *event)
 		return 0;
 
 	/* poor man's strlen */
-	off = probe_read_str(&heap->maxpath, 4096, (char *)start_stack);
+	off = probe_read_user_str(&heap->maxpath, 4096, (char *)start_stack);
 	if (off < 0)
 		return 0;
 
@@ -78,7 +78,7 @@ read_args(void *ctx, struct msg_execve_event *event)
 
 	if (args_size < BUFFER && args_size < free_size) {
 		size = args_size & 0x3ff /* BUFFER - 1 */;
-		err = probe_read(args, size, (char *)start_stack);
+		err = probe_read_user(args, size, (char *)start_stack);
 		if (err < 0) {
 			p->flags |= EVENT_ERROR_ARGS;
 			size = 0;
@@ -104,7 +104,7 @@ read_path(void *ctx, struct msg_execve_event *event, void *filename)
 
 	earg = (void *)p + offsetof(struct msg_process, args);
 
-	size = probe_read_str(earg, MAXARGLENGTH - 1, filename);
+	size = probe_read_kernel_str(earg, MAXARGLENGTH - 1, filename);
 	if (size < 0) {
 		flags |= EVENT_ERROR_FILENAME;
 		size = 0;
@@ -305,15 +305,15 @@ execve_send(struct sched_execve_args *ctx)
 #ifdef __LARGE_BPF_PROG
 		// read from proc exe stored at execve time
 		if (event->exe.len <= BINARY_PATH_MAX_LEN) {
-			curr->bin.path_length = probe_read(curr->bin.path, event->exe.len, event->exe.off);
+			curr->bin.path_length = probe_read_kernel(curr->bin.path, event->exe.len, event->exe.off);
 			if (curr->bin.path_length == 0)
 				curr->bin.path_length = event->exe.len;
 		}
 #else
 		// reuse p->args first string that contains the filename, this can't be
 		// above 256 in size (otherwise the complete will be send via data msg)
 		// which is okay because we need the 256 first bytes.
-		curr->bin.path_length = probe_read_str(curr->bin.path, BINARY_PATH_MAX_LEN, &p->args);
+		curr->bin.path_length = probe_read_kernel_str(curr->bin.path, BINARY_PATH_MAX_LEN, &p->args);
 		if (curr->bin.path_length > 1) {
 			// don't include the NULL byte in the length
 			curr->bin.path_length--;

bpf/process/bpf_exit.h

@@ -63,8 +63,8 @@ static inline __attribute__((always_inline)) void event_exit_send(void *ctx, __u
 		 *  entry from the execve_map anyway and explicitly set it to the to tgid.
 		 */
 		exit->info.tid = tgid;
-		probe_read(&exit->info.code, sizeof(exit->info.code),
-			   _(&task->exit_code));
+		probe_read_kernel(&exit->info.code, sizeof(exit->info.code),
+				  _(&task->exit_code));
 
 		perf_event_output_metric(ctx, MSG_OP_EXIT, &tcpmon_map, BPF_F_CURRENT_CPU, exit, size);
 	}

bpf/process/bpf_generic_tracepoint.c

@@ -69,44 +69,44 @@ static inline __attribute__((always_inline)) unsigned long get_ctx_ul(void *src,
 	case u64_ty: {
 		u64 ret;
 
-		probe_read(&ret, sizeof(u64), src);
+		probe_read_kernel(&ret, sizeof(u64), src);
 		return ret;
 	}
 
 	case size_type: {
 		size_t ret;
 
-		probe_read(&ret, sizeof(size_t), src);
+		probe_read_kernel(&ret, sizeof(size_t), src);
 		return (unsigned long)ret;
 	}
 
 	case nop_s32_ty:
 	case s32_ty: {
 		s32 ret;
 
-		probe_read(&ret, sizeof(u32), src);
+		probe_read_kernel(&ret, sizeof(u32), src);
 		return ret;
 	}
 
 	case nop_u32_ty:
 	case u32_ty: {
 		u32 ret;
 
-		probe_read(&ret, sizeof(u32), src);
+		probe_read_kernel(&ret, sizeof(u32), src);
 		return ret;
 	}
 
 	case char_buf:
 	case string_type: {
 		char *buff;
-		probe_read(&buff, sizeof(char *), src);
+		probe_read_kernel(&buff, sizeof(char *), src);
 		return (unsigned long)buff;
 	}
 
 	case data_loc_type: {
 		u32 ret;
 
-		probe_read(&ret, sizeof(ret), src);
+		probe_read_kernel(&ret, sizeof(ret), src);
 		return ret;
 	}
 
@@ -117,14 +117,14 @@ static inline __attribute__((always_inline)) unsigned long get_ctx_ul(void *src,
 	case skb_type: {
 		struct sk_buff *skb;
 
-		probe_read(&skb, sizeof(struct sk_buff *), src);
+		probe_read_kernel(&skb, sizeof(struct sk_buff *), src);
 		return (unsigned long)skb;
 	}
 
 	case sock_type: {
 		struct sock *sk;
 
-		probe_read(&sk, sizeof(struct sock *), src);
+		probe_read_kernel(&sk, sizeof(struct sock *), src);
 		return (unsigned long)sk;
 	}
 

bpf/process/bpf_loader.c

@@ -115,11 +115,11 @@ loader_kprobe(struct pt_regs *ctx)
 	if (!msg->buildid_size)
 		return 0;
 
-	probe_read(&msg->buildid[0], sizeof(msg->buildid),
-		   _(&mmap_event->build_id[0]));
+	probe_read_kernel(&msg->buildid[0], sizeof(msg->buildid),
+			  _(&mmap_event->build_id[0]));
 
 	path = BPF_CORE_READ(mmap_event, file_name);
-	len = probe_read_str(&msg->path, sizeof(msg->path), path);
+	len = probe_read_kernel_str(&msg->path, sizeof(msg->path), path);
 	msg->path_size = (__u32)len;
 
 	msg->pid = tgid;