From 8c89a013d049aca7131a1d5bc98ba8622b7eb72f Mon Sep 17 00:00:00 2001 From: Kevin Sheldrake Date: Mon, 11 Mar 2024 16:28:15 +0000 Subject: [PATCH] Memory: Swap probe_read to kernel or user version We should always use the probe_read_kernel or probe_read_user helpers over the probe_read helper (ditto for _str versions). This commit changes all probe_read to either probe_read_kernel or probe_read_user (ditto for _str versions). Signed-off-by: Kevin Sheldrake --- bpf/cgroup/bpf_cgroup_events.h | 2 +- bpf/include/api.h | 3 + bpf/lib/bpf_cgroup.h | 24 +-- bpf/lib/bpf_helpers.h | 2 +- bpf/lib/bpf_task.h | 26 +-- bpf/lib/process.h | 2 +- bpf/libbpf/bpf_core_read.h | 149 ++++++++++++--- bpf/libbpf/bpf_tracing.h | 46 ++--- bpf/process/bpf_execve_bprm_commit_creds.c | 26 +-- bpf/process/bpf_execve_event.c | 26 +-- bpf/process/bpf_exit.h | 4 +- bpf/process/bpf_fork.c | 4 +- bpf/process/bpf_generic_tracepoint.c | 16 +- bpf/process/bpf_loader.c | 14 +- bpf/process/bpf_process_event.h | 140 +++++++------- bpf/process/data_event.h | 66 +++++-- bpf/process/generic_calls.h | 2 +- bpf/process/types/basic.h | 173 +++++++++--------- bpf/process/types/probe_read_kernel_or_user.h | 17 ++ bpf/process/types/skb.h | 59 +++--- bpf/process/types/sock.h | 33 ++-- 21 files changed, 496 insertions(+), 338 deletions(-) create mode 100644 bpf/process/types/probe_read_kernel_or_user.h diff --git a/bpf/cgroup/bpf_cgroup_events.h b/bpf/cgroup/bpf_cgroup_events.h index 026ac358617..9227eaf85e1 100644 --- a/bpf/cgroup/bpf_cgroup_events.h +++ b/bpf/cgroup/bpf_cgroup_events.h @@ -49,7 +49,7 @@ send_cgrp_event(struct bpf_raw_tracepoint_args *ctx, msg->cgrp_data.level = cgrp_track->level; msg->cgrp_data.hierarchy_id = cgrp_track->hierarchy_id; memcpy(&msg->cgrp_data.name, &cgrp_track->name, KN_NAME_LENGTH); - probe_read_str(&msg->path, PATH_MAP_SIZE - 1, path); + probe_read_kernel_str(&msg->path, PATH_MAP_SIZE - 1, path); perf_event_output_metric(ctx, MSG_OP_CGROUP, &tcpmon_map, BPF_F_CURRENT_CPU, msg, size); diff --git a/bpf/include/api.h b/bpf/include/api.h index 48ded18dc89..fcd67ed3b5d 100644 --- a/bpf/include/api.h +++ b/bpf/include/api.h @@ -204,6 +204,9 @@ static int BPF_FUNC(fib_lookup, void *ctx, struct bpf_fib_lookup *params, uint32 static int BPF_FUNC(probe_read, void *dst, uint32_t size, const void *src); static int BPF_FUNC(probe_read_str, void *dst, int size, const void *src); static int BPF_FUNC(probe_read_kernel, void *dst, uint32_t size, const void *src); +static int BPF_FUNC(probe_read_kernel_str, void *dst, int size, const void *src); +static int BPF_FUNC(probe_read_user, void *dst, uint32_t size, const void *src); +static int BPF_FUNC(probe_read_user_str, void *dst, int size, const void *src); static uint64_t BPF_FUNC(get_current_task); diff --git a/bpf/lib/bpf_cgroup.h b/bpf/lib/bpf_cgroup.h index 94ccb106d7c..8c535a05e40 100644 --- a/bpf/lib/bpf_cgroup.h +++ b/bpf/lib/bpf_cgroup.h @@ -109,7 +109,7 @@ __get_cgroup_kn_name(const struct kernfs_node *kn) const char *name = NULL; if (kn) - probe_read(&name, sizeof(name), _(&kn->name)); + probe_read_kernel(&name, sizeof(name), _(&kn->name)); return name; } @@ -136,10 +136,10 @@ __get_cgroup_kn_id(const struct kernfs_node *kn) struct kernfs_node___old *old_kn; old_kn = (void *)kn; - if (BPF_CORE_READ_INTO(&id, old_kn, id.id) != 0) + if (BPF_CORE_READ_KERNEL_INTO(&id, old_kn, id.id) != 0) return 0; } else { - probe_read(&id, sizeof(id), _(&kn->id)); + probe_read_kernel(&id, sizeof(id), _(&kn->id)); } return id; @@ -157,7 +157,7 @@ __get_cgroup_kn(const struct cgroup *cgrp) struct kernfs_node *kn = NULL; if (cgrp) - probe_read(&kn, sizeof(cgrp->kn), _(&cgrp->kn)); + probe_read_kernel(&kn, sizeof(cgrp->kn), _(&cgrp->kn)); return kn; } @@ -177,7 +177,7 @@ get_cgroup_hierarchy_id(const struct cgroup *cgrp) { __u32 id; - BPF_CORE_READ_INTO(&id, cgrp, root, hierarchy_id); + BPF_CORE_READ_KERNEL_INTO(&id, cgrp, root, hierarchy_id); return id; } @@ -187,7 +187,7 @@ get_cgroup_hierarchy_id(const struct cgroup *cgrp) * @cgrp: target cgroup * * Returns a pointer to the cgroup node name on success that can - * be read with probe_read(). NULL on failures. + * be read with probe_read_kernel(). NULL on failures. */ static inline __attribute__((always_inline)) const char * get_cgroup_name(const struct cgroup *cgrp) @@ -197,7 +197,7 @@ get_cgroup_name(const struct cgroup *cgrp) if (unlikely(!cgrp)) return NULL; - if (BPF_CORE_READ_INTO(&name, cgrp, kn, name) != 0) + if (BPF_CORE_READ_KERNEL_INTO(&name, cgrp, kn, name) != 0) return NULL; return name; @@ -214,7 +214,7 @@ get_cgroup_level(const struct cgroup *cgrp) { __u32 level = 0; - probe_read(&level, sizeof(level), _(&cgrp->level)); + probe_read_kernel(&level, sizeof(level), _(&cgrp->level)); return level; } @@ -264,7 +264,7 @@ get_task_cgroup(struct task_struct *task, __u32 subsys_idx, __u32 *error_flags) struct css_set *cgroups; struct cgroup *cgrp = NULL; - probe_read(&cgroups, sizeof(cgroups), _(&task->cgroups)); + probe_read_kernel(&cgroups, sizeof(cgroups), _(&task->cgroups)); if (unlikely(!cgroups)) { *error_flags |= EVENT_ERROR_CGROUPS; return cgrp; @@ -297,13 +297,13 @@ get_task_cgroup(struct task_struct *task, __u32 subsys_idx, __u32 *error_flags) * support as much as workload as possible. It also reduces errors * in a significant way. */ - probe_read(&subsys, sizeof(subsys), _(&cgroups->subsys[subsys_idx])); + probe_read_kernel(&subsys, sizeof(subsys), _(&cgroups->subsys[subsys_idx])); if (unlikely(!subsys)) { *error_flags |= EVENT_ERROR_CGROUP_SUBSYS; return cgrp; } - probe_read(&cgrp, sizeof(cgrp), _(&subsys->cgroup)); + probe_read_kernel(&cgrp, sizeof(cgrp), _(&subsys->cgroup)); if (!cgrp) *error_flags |= EVENT_ERROR_CGROUP_SUBSYSCGRP; @@ -426,7 +426,7 @@ __init_cgrp_tracking_val_heap(struct cgroup *cgrp, cgroup_state state) kn = __get_cgroup_kn(cgrp); name = __get_cgroup_kn_name(kn); if (name) - probe_read_str(&heap->name, KN_NAME_LENGTH - 1, name); + probe_read_kernel_str(&heap->name, KN_NAME_LENGTH - 1, name); return heap; } diff --git a/bpf/lib/bpf_helpers.h b/bpf/lib/bpf_helpers.h index f57410880af..338e5210a8b 100644 --- a/bpf/lib/bpf_helpers.h +++ b/bpf/lib/bpf_helpers.h @@ -43,7 +43,7 @@ * Following define is to assist VSCode Intellisense so that it treats * __builtin_preserve_access_index() as a const void * instead of a * simple void (because it doesn't have a definition for it). This stops - * Intellisense marking all _(P) macros (used in probe_read()) as errors. + * Intellisense marking all _(P) macros (used in probe_read_kernel()) as errors. * To use this, just define VSCODE in 'C/C++: Edit Configurations (JSON)' * in the Command Palette in VSCODE (F1 or View->Command Palette...): * "defines": ["VSCODE"] diff --git a/bpf/lib/bpf_task.h b/bpf/lib/bpf_task.h index f4e2e9bc13a..05cbe697414 100644 --- a/bpf/lib/bpf_task.h +++ b/bpf/lib/bpf_task.h @@ -27,7 +27,7 @@ get_parent(struct task_struct *t) struct task_struct *task; /* Read the real parent */ - probe_read(&task, sizeof(task), _(&t->real_parent)); + probe_read_kernel(&task, sizeof(task), _(&t->real_parent)); if (!task) return 0; return task; @@ -47,7 +47,7 @@ get_task_from_pid(__u32 pid) i = TASK_PID_LOOP; continue; } - probe_read(&cpid, sizeof(cpid), _(&task->tgid)); + probe_read_kernel(&cpid, sizeof(cpid), _(&task->tgid)); if (cpid == pid) { i = TASK_PID_LOOP; continue; @@ -70,7 +70,7 @@ static inline __attribute__((always_inline)) __u32 get_task_pid_vnr(void) thread_pid_exists = bpf_core_field_exists(task->thread_pid); if (thread_pid_exists) { - probe_read(&pid, sizeof(pid), _(&task->thread_pid)); + probe_read_kernel(&pid, sizeof(pid), _(&task->thread_pid)); if (!pid) return 0; } else { @@ -85,16 +85,16 @@ static inline __attribute__((always_inline)) __u32 get_task_pid_vnr(void) if (!thread_pid_exists) link_sz = 24; // voodoo magic, hard-code 24 to init stack - probe_read(&link, link_sz, - (void *)_(&task->pids) + (PIDTYPE_PID * link_sz)); + probe_read_kernel(&link, link_sz, + (void *)_(&task->pids) + (PIDTYPE_PID * link_sz)); pid = link.pid; } upid_sz = bpf_core_field_size(pid->numbers[0]); - probe_read(&level, sizeof(level), _(&pid->level)); + probe_read_kernel(&level, sizeof(level), _(&pid->level)); if (level < 1) return 0; - probe_read(&upid, upid_sz, - (void *)_(&pid->numbers) + (level * upid_sz)); + probe_read_kernel(&upid, upid_sz, + (void *)_(&pid->numbers) + (level * upid_sz)); return upid.nr; } @@ -106,7 +106,7 @@ event_find_parent_pid(struct task_struct *t) if (!task) return 0; - probe_read(&pid, sizeof(pid), _(&task->tgid)); + probe_read_kernel(&pid, sizeof(pid), _(&task->tgid)); return pid; } @@ -119,10 +119,10 @@ __event_find_parent(struct task_struct *task) #pragma unroll for (i = 0; i < 4; i++) { - probe_read(&task, sizeof(task), _(&task->real_parent)); + probe_read_kernel(&task, sizeof(task), _(&task->real_parent)); if (!task) break; - probe_read(&pid, sizeof(pid), _(&task->tgid)); + probe_read_kernel(&pid, sizeof(pid), _(&task->tgid)); value = execve_map_get_noinit(pid); if (value && value->key.ktime != 0) return value; @@ -164,13 +164,13 @@ event_find_curr(__u32 *ppid, bool *walked) #pragma unroll for (i = 0; i < 4; i++) { - probe_read(&pid, sizeof(pid), _(&task->tgid)); + probe_read_kernel(&pid, sizeof(pid), _(&task->tgid)); value = execve_map_get_noinit(pid); if (value && value->key.ktime != 0) break; value = 0; *walked = 1; - probe_read(&task, sizeof(task), _(&task->real_parent)); + probe_read_kernel(&task, sizeof(task), _(&task->real_parent)); if (!task) break; } diff --git a/bpf/lib/process.h b/bpf/lib/process.h index bdadea67115..6f4c90cf597 100644 --- a/bpf/lib/process.h +++ b/bpf/lib/process.h @@ -51,7 +51,7 @@ * Now we want to read this with call 45 aka probe_read_str as follows, * where 'kernel_struct_arg' is the kernel data struct we are reading. * - * probe_read_str(args[offset], size, kernel_struct_arg) + * probe_read_kernel_str(args[offset], size, kernel_struct_arg) * * But we have a bit of a problem determining if 'size' is out of array * range. The math would be, diff --git a/bpf/libbpf/bpf_core_read.h b/bpf/libbpf/bpf_core_read.h index 27634770a94..1d9481ead88 100644 --- a/bpf/libbpf/bpf_core_read.h +++ b/bpf/libbpf/bpf_core_read.h @@ -30,8 +30,8 @@ enum bpf_type_info_kind { __builtin_preserve_field_info((src)->field, BPF_FIELD_##info) #if __BYTE_ORDER == __LITTLE_ENDIAN -#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ - bpf_probe_read((void *)dst, \ +#define __CORE_BITFIELD_PROBE_READ(fn, dst, src, fld) \ + fn((void *)dst, \ __CORE_RELO(src, fld, BYTE_SIZE), \ (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) #else @@ -39,8 +39,8 @@ enum bpf_type_info_kind { * for big-endian we need to adjust destination pointer accordingly, based on * field byte size */ -#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ - bpf_probe_read((void *)dst + (8 - __CORE_RELO(src, fld, BYTE_SIZE)), \ +#define __CORE_BITFIELD_PROBE_READ(fn, dst, src, fld) \ + fn((void *)dst + (8 - __CORE_RELO(src, fld, BYTE_SIZE)), \ __CORE_RELO(src, fld, BYTE_SIZE), \ (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) #endif @@ -49,14 +49,14 @@ enum bpf_type_info_kind { * Extract bitfield, identified by s->field, and return its value as u64. * All this is done in relocatable manner, so bitfield changes such as * signedness, bit size, offset changes, this will be handled automatically. - * This version of macro is using bpf_probe_read() to read underlying integer + * This version of macro is using bpf_probe_read_kernel() to read underlying integer * storage. Macro functions as an expression and its return type is - * bpf_probe_read()'s return value: 0, on success, <0 on error. + * bpf_probe_read_kernel()'s return value: 0, on success, <0 on error. */ -#define BPF_CORE_READ_BITFIELD_PROBED(s, field) ({ \ +#define __BPF_CORE_READ_BITFIELD_PROBED(fn, s, field) ({ \ unsigned long long val = 0; \ \ - __CORE_BITFIELD_PROBE_READ(&val, s, field); \ + __CORE_BITFIELD_PROBE_READ(fn, &val, s, field); \ val <<= __CORE_RELO(s, field, LSHIFT_U64); \ if (__CORE_RELO(s, field, SIGNED)) \ val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \ @@ -65,6 +65,17 @@ enum bpf_type_info_kind { val; \ }) +#define BPF_CORE_READ_BITFIELD_PROBED_KERNEL(s, field) \ + __BPF_CORE_READ_BITFIELD_PROBED(bpf_probe_read_kernel, s, field) +#define BPF_CORE_READ_BITFIELD_PROBED_USER(s, field) \ + __BPF_CORE_READ_BITFIELD_PROBED(bpf_probe_read_user, s, field) + +/* + * Provided for consistency. + */ +#define BPF_CORE_READ_BITFIELD_PROBED(s, field) \ + __BPF_CORE_READ_BITFIELD_PROBED(bpf_probe_read, s, field) + /* * Extract bitfield, identified by s->field, and return its value as u64. * This version of macro is using direct memory reads and should be used from @@ -157,6 +168,24 @@ enum bpf_type_info_kind { probe_read(dst, sz, \ (const void *)__builtin_preserve_access_index(src)) +/* + * bpf_core_read_kernel() abstracts away bpf_probe_read_kernel() call and captures offset + * relocation for source address using __builtin_preserve_access_index() + * built-in, provided by Clang. + */ +#define bpf_core_read_kernel(dst, sz, src) \ + probe_read_kernel(dst, sz, \ + (const void *)__builtin_preserve_access_index(src)) + +/* + * bpf_core_read_user() abstracts away bpf_probe_read_user() call and captures offset + * relocation for source address using __builtin_preserve_access_index() + * built-in, provided by Clang. + */ +#define bpf_core_read_user(dst, sz, src) \ + probe_read_user(dst, sz, \ + (const void *)__builtin_preserve_access_index(src)) + /* * bpf_core_read_str() is a thin wrapper around bpf_probe_read_str() * additionally emitting BPF CO-RE field relocation for specified source @@ -166,6 +195,24 @@ enum bpf_type_info_kind { bpf_probe_read_str(dst, sz, \ (const void *)__builtin_preserve_access_index(src)) +/* + * bpf_core_read_kernel_str() is a thin wrapper around bpf_probe_read_kernel_str() + * additionally emitting BPF CO-RE field relocation for specified source + * argument. + */ +#define bpf_core_read_kernel_str(dst, sz, src) \ + bpf_probe_read_kernel_str(dst, sz, \ + (const void *)__builtin_preserve_access_index(src)) + +/* + * bpf_core_read_user_str() is a thin wrapper around bpf_probe_read_user_str() + * additionally emitting BPF CO-RE field relocation for specified source + * argument. + */ +#define bpf_core_read_user_str(dst, sz, src) \ + bpf_probe_read_user_str(dst, sz, \ + (const void *)__builtin_preserve_access_index(src)) + #define ___concat(a, b) a ## b #define ___apply(fn, n) ___concat(fn, n) #define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N @@ -223,26 +270,26 @@ enum bpf_type_info_kind { read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor) /* "recursively" read a sequence of inner pointers using local __t var */ -#define ___rd_first(src, a) ___read(bpf_core_read, &__t, ___type(src), src, a); -#define ___rd_last(...) \ - ___read(bpf_core_read, &__t, \ +#define ___rd_first(fn, src, a) ___read(fn, &__t, ___type(src), src, a); +#define ___rd_last(fn, ...) \ + ___read(fn, &__t, \ ___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__)); -#define ___rd_p1(...) const void *__t; ___rd_first(__VA_ARGS__) -#define ___rd_p2(...) ___rd_p1(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p3(...) ___rd_p2(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p4(...) ___rd_p3(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p5(...) ___rd_p4(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p6(...) ___rd_p5(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p7(...) ___rd_p6(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p8(...) ___rd_p7(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___rd_p9(...) ___rd_p8(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) -#define ___read_ptrs(src, ...) \ - ___apply(___rd_p, ___narg(__VA_ARGS__))(src, __VA_ARGS__) +#define ___rd_p1(fn, ...) const void *__t; ___rd_first(fn, __VA_ARGS__) +#define ___rd_p2(fn, ...) ___rd_p1(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p3(fn, ...) ___rd_p2(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p4(fn, ...) ___rd_p3(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p5(fn, ...) ___rd_p4(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p6(fn, ...) ___rd_p5(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p7(fn, ...) ___rd_p6(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p8(fn, ...) ___rd_p7(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___rd_p9(fn, ...) ___rd_p8(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) +#define ___read_ptrs(fn, src, ...) \ + ___apply(___rd_p, ___narg(__VA_ARGS__))(fn, src, __VA_ARGS__) #define ___core_read0(fn, dst, src, a) \ ___read(fn, dst, ___type(src), src, a); #define ___core_readN(fn, dst, src, ...) \ - ___read_ptrs(src, ___nolast(__VA_ARGS__)) \ + ___read_ptrs(fn, src, ___nolast(__VA_ARGS__)) \ ___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \ ___last(__VA_ARGS__)); #define ___core_read(fn, dst, src, a, ...) \ @@ -259,9 +306,29 @@ enum bpf_type_info_kind { ___core_read(bpf_core_read, dst, src, a, ##__VA_ARGS__) \ }) +/* + * BPF_CORE_READ_KERNEL_INTO() is a more performance-conscious variant of + * BPF_CORE_READ_KERNEL(), in which final field is read into user-provided storage. + * See BPF_CORE_READ_KERNEL() below for more details on general usage. + */ +#define BPF_CORE_READ_KERNEL_INTO(dst, src, a, ...) \ + ({ \ + ___core_read(bpf_core_read_kernel, dst, src, a, ##__VA_ARGS__) \ + }) + +/* + * BPF_CORE_READ_USER_INTO() is a more performance-conscious variant of + * BPF_CORE_READ_USER(), in which final field is read into user-provided storage. + * See BPF_CORE_READ_USER() below for more details on general usage. + */ +#define BPF_CORE_READ_USER_INTO(dst, src, a, ...) \ + ({ \ + ___core_read(bpf_core_read_user, dst, src, a, ##__VA_ARGS__) \ + }) + /* * BPF_CORE_READ_STR_INTO() does same "pointer chasing" as - * BPF_CORE_READ() for intermediate pointers, but then executes (and returns + * BPF_CORE_READ_STR() for intermediate pointers, but then executes (and returns * corresponding error code) bpf_core_read_str() for final string read. */ #define BPF_CORE_READ_STR_INTO(dst, src, a, ...) \ @@ -269,6 +336,26 @@ enum bpf_type_info_kind { ___core_read(bpf_core_read_str, dst, src, a, ##__VA_ARGS__) \ }) +/* + * BPF_CORE_READ_KERNEL_STR_INTO() does same "pointer chasing" as + * BPF_CORE_READ_KERNEL_STR() for intermediate pointers, but then executes (and returns + * corresponding error code) bpf_core_read_kernel_str() for final string read. + */ +#define BPF_CORE_READ_KERNEL_STR_INTO(dst, src, a, ...) \ + ({ \ + ___core_read(bpf_core_read_kernel_str, dst, src, a, ##__VA_ARGS__) \ + }) + +/* + * BPF_CORE_READ_USER_STR_INTO() does same "pointer chasing" as + * BPF_CORE_READ_USER_STR() for intermediate pointers, but then executes (and returns + * corresponding error code) bpf_core_read_user_str() for final string read. + */ +#define BPF_CORE_READ_USER_STR_INTO(dst, src, a, ...) \ + ({ \ + ___core_read(bpf_core_read_user_str, dst, src, a, ##__VA_ARGS__) \ + }) + /* * BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially * when there are few pointer chasing steps. @@ -298,5 +385,19 @@ enum bpf_type_info_kind { __r; \ }) +#define BPF_CORE_READ_KERNEL(src, a, ...) \ + ({ \ + ___type(src, a, ##__VA_ARGS__) __r; \ + BPF_CORE_READ_KERNEL_INTO(&__r, src, a, ##__VA_ARGS__); \ + __r; \ + }) + +#define BPF_CORE_READ_USER(src, a, ...) \ + ({ \ + ___type(src, a, ##__VA_ARGS__) __r; \ + BPF_CORE_READ_USER_INTO(&__r, src, a, ##__VA_ARGS__); \ + __r; \ + }) + #endif diff --git a/bpf/libbpf/bpf_tracing.h b/bpf/libbpf/bpf_tracing.h index 789556811e2..50f429092eb 100644 --- a/bpf/libbpf/bpf_tracing.h +++ b/bpf/libbpf/bpf_tracing.h @@ -134,7 +134,7 @@ struct pt_regs___s390 { #define __PT_PARM6_SYSCALL_REG gprs[7] #define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) #define PT_REGS_PARM1_CORE_SYSCALL(x) \ - (unsigned long)BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) + (unsigned long)BPF_CORE_READ_KERNEL((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) #define __PT_RET_REG gprs[14] #define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ @@ -196,7 +196,7 @@ struct pt_regs___arm64 { #define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG #define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) #define PT_REGS_PARM1_CORE_SYSCALL(x) \ - (unsigned long)BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) + (unsigned long)BPF_CORE_READ_KERNEL((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) #define __PT_RET_REG regs[30] #define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ @@ -416,19 +416,19 @@ struct pt_regs; #define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG) #define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG) -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG) -#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG) -#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG) -#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG) +#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM1_REG) +#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM2_REG) +#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM3_REG) +#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM4_REG) +#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM5_REG) +#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM6_REG) +#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM7_REG) +#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM8_REG) +#define PT_REGS_RET_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_RET_REG) +#define PT_REGS_FP_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_FP_REG) +#define PT_REGS_RC_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_RC_REG) +#define PT_REGS_SP_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_SP_REG) +#define PT_REGS_IP_CORE(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_IP_REG) #if defined(bpf_target_powerpc) @@ -451,31 +451,31 @@ struct pt_regs; #ifndef PT_REGS_PARM1_SYSCALL #define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG) -#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG) +#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG) #endif #ifndef PT_REGS_PARM2_SYSCALL #define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG) -#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG) +#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG) #endif #ifndef PT_REGS_PARM3_SYSCALL #define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG) -#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG) +#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG) #endif #ifndef PT_REGS_PARM4_SYSCALL #define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG) -#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG) +#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG) #endif #ifndef PT_REGS_PARM5_SYSCALL #define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG) -#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG) +#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG) #endif #ifndef PT_REGS_PARM6_SYSCALL #define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG) -#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG) +#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG) #endif #ifndef PT_REGS_PARM7_SYSCALL #define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG) -#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG) +#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ_KERNEL(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG) #endif #else /* defined(bpf_target_defined) */ @@ -667,7 +667,7 @@ static inline __attribute__((always_inline)) typeof(name(0)) ____##name(struct p #define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (void *)PT_REGS_PARM7_SYSCALL(regs) #define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) -/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ +/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ_KERNEL from pt_regs */ #define ___bpf_syswrap_args0() ctx #define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) #define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) diff --git a/bpf/process/bpf_execve_bprm_commit_creds.c b/bpf/process/bpf_execve_bprm_commit_creds.c index ef75a5945fe..3c5e42df2c1 100644 --- a/bpf/process/bpf_execve_bprm_commit_creds.c +++ b/bpf/process/bpf_execve_bprm_commit_creds.c @@ -67,32 +67,32 @@ BPF_KPROBE(tg_kp_bprm_committing_creds, struct linux_binprm *bprm) memset(&heap->info, 0, sizeof(struct execve_info)); /* Read binary file information */ - if (BPF_CORE_READ_INTO(&file, bprm, file) != 0) + if (BPF_CORE_READ_KERNEL_INTO(&file, bprm, file) != 0) return; - if (BPF_CORE_READ_INTO(&heap->info.i_nlink, file, f_inode, __i_nlink) != 0) + if (BPF_CORE_READ_KERNEL_INTO(&heap->info.i_nlink, file, f_inode, __i_nlink) != 0) return; - if (BPF_CORE_READ_INTO(&heap->info.i_ino, file, f_inode, i_ino) != 0) + if (BPF_CORE_READ_KERNEL_INTO(&heap->info.i_ino, file, f_inode, i_ino) != 0) return; /* If no flags to clear then this is not a privileged execution */ - if (BPF_CORE_READ_INTO(&sec, bprm, per_clear) != 0 || sec == 0) + if (BPF_CORE_READ_KERNEL_INTO(&sec, bprm, per_clear) != 0 || sec == 0) goto out; /* Check if this is a setuid or setgid */ - euid = BPF_CORE_READ(bprm, cred, euid.val); - egid = BPF_CORE_READ(bprm, cred, egid.val); + euid = BPF_CORE_READ_KERNEL(bprm, cred, euid.val); + egid = BPF_CORE_READ_KERNEL(bprm, cred, egid.val); task = (struct task_struct *)get_current_task(); - uid = BPF_CORE_READ(task, cred, uid.val); - gid = BPF_CORE_READ(task, cred, gid.val); + uid = BPF_CORE_READ_KERNEL(task, cred, uid.val); + gid = BPF_CORE_READ_KERNEL(task, cred, gid.val); /* Is setuid? */ if (euid != uid) { heap->info.secureexec |= EXEC_SETUID; /* If euid is being changed to root? */ - ruid = BPF_CORE_READ(bprm, cred, uid.val); + ruid = BPF_CORE_READ_KERNEL(bprm, cred, uid.val); if (!__is_uid_global_root(ruid) && __is_uid_global_root(euid)) /* If we executed from a non root and became global effective root * then set the EXEC_FS_SETUID to indicate that there was a privilege @@ -115,7 +115,7 @@ BPF_KPROBE(tg_kp_bprm_committing_creds, struct linux_binprm *bprm) if (egid != gid) { heap->info.secureexec |= EXEC_SETGID; /* Is egid is being changed to real root? */ - gid = BPF_CORE_READ(bprm, cred, gid.val); + gid = BPF_CORE_READ_KERNEL(bprm, cred, gid.val); if (!__is_uid_global_root(gid) && __is_uid_global_root(egid)) /* If we executed a setgid to root binary then this is a * privilege elevation since it can now access root files, etc @@ -131,7 +131,7 @@ BPF_KPROBE(tg_kp_bprm_committing_creds, struct linux_binprm *bprm) * there is no way the binary could provide extra capabilities, they cancel * each other. */ - BPF_CORE_READ_INTO(&new_ambient, bprm, cred, cap_ambient); + BPF_CORE_READ_KERNEL_INTO(&new_ambient, bprm, cred, cap_ambient); if (new_ambient) goto out; @@ -144,8 +144,8 @@ BPF_KPROBE(tg_kp_bprm_committing_creds, struct linux_binprm *bprm) * (2) The file capabilities are set on the binary. If the setuid bit is not set * then the gained capabilities are from file capabilities execution. */ - BPF_CORE_READ_INTO(&permitted, task, cred, cap_permitted); - BPF_CORE_READ_INTO(&new_permitted, bprm, cred, cap_permitted); + BPF_CORE_READ_KERNEL_INTO(&permitted, task, cred, cap_permitted); + BPF_CORE_READ_KERNEL_INTO(&new_permitted, bprm, cred, cap_permitted); if (__cap_gained(new_permitted, permitted) && euid == uid) { /* If the setuid bit is not set then this is probably a file cap execution. */ heap->info.secureexec |= EXEC_FILE_CAPS; diff --git a/bpf/process/bpf_execve_event.c b/bpf/process/bpf_execve_event.c index a00ae3a115f..8033078e094 100644 --- a/bpf/process/bpf_execve_event.c +++ b/bpf/process/bpf_execve_event.c @@ -41,13 +41,13 @@ read_args(void *ctx, struct msg_execve_event *event) long off; int err; - probe_read(&mm, sizeof(mm), _(&task->mm)); + probe_read_kernel(&mm, sizeof(mm), _(&task->mm)); if (!mm) return 0; - probe_read(&start_stack, sizeof(start_stack), - _(&mm->arg_start)); - probe_read(&end_stack, sizeof(start_stack), _(&mm->arg_end)); + probe_read_kernel(&start_stack, sizeof(start_stack), + _(&mm->arg_start)); + probe_read_kernel(&end_stack, sizeof(start_stack), _(&mm->arg_end)); if (!start_stack || !end_stack) return 0; @@ -58,7 +58,7 @@ read_args(void *ctx, struct msg_execve_event *event) return 0; /* poor man's strlen */ - off = probe_read_str(&heap->maxpath, 4096, (char *)start_stack); + off = probe_read_user_str(&heap->maxpath, 4096, (char *)start_stack); if (off < 0) return 0; @@ -78,13 +78,13 @@ read_args(void *ctx, struct msg_execve_event *event) if (args_size < BUFFER && args_size < free_size) { size = args_size & 0x3ff /* BUFFER - 1 */; - err = probe_read(args, size, (char *)start_stack); + err = probe_read_user(args, size, (char *)start_stack); if (err < 0) { p->flags |= EVENT_ERROR_ARGS; size = 0; } } else { - size = data_event_bytes(ctx, (struct data_event_desc *)args, + size = data_event_bytes(ctx, DATA_EVENT_MEM_USER, (struct data_event_desc *)args, (unsigned long)start_stack, args_size, (struct bpf_map_def *)&data_heap); @@ -104,12 +104,12 @@ read_path(void *ctx, struct msg_execve_event *event, void *filename) earg = (void *)p + offsetof(struct msg_process, args); - size = probe_read_str(earg, MAXARGLENGTH - 1, filename); + size = probe_read_kernel_str(earg, MAXARGLENGTH - 1, filename); if (size < 0) { flags |= EVENT_ERROR_FILENAME; size = 0; } else if (size == MAXARGLENGTH - 1) { - size = data_event_str(ctx, (struct data_event_desc *)earg, + size = data_event_str(ctx, DATA_EVENT_MEM_KERNEL, (struct data_event_desc *)earg, (unsigned long)filename, (struct bpf_map_def *)&data_heap); if (size == 0) @@ -162,7 +162,7 @@ read_execve_shared_info(void *ctx, struct msg_process *p, __u64 pid) static inline __attribute__((always_inline)) __u32 read_exe(struct task_struct *task, struct heap_exe *exe) { - struct file *file = BPF_CORE_READ(task, mm, exe_file); + struct file *file = BPF_CORE_READ_KERNEL(task, mm, exe_file); struct path *path = __builtin_preserve_access_index(&file->f_path); exe->len = BINARY_PATH_MAX_LEN; @@ -223,7 +223,7 @@ event_execve(struct sched_execve_args *ctx) event->common.ktime = p->ktime; event->common.size = offsetof(struct msg_execve_event, process) + p->size; - BPF_CORE_READ_INTO(&event->kube.net_ns, task, nsproxy, net_ns, ns.inum); + BPF_CORE_READ_KERNEL_INTO(&event->kube.net_ns, task, nsproxy, net_ns, ns.inum); // At this time objective and subjective creds are same get_current_subj_caps(&event->caps, task); @@ -307,7 +307,7 @@ execve_send(struct sched_execve_args *ctx) #ifdef __LARGE_BPF_PROG // read from proc exe stored at execve time if (event->exe.len <= BINARY_PATH_MAX_LEN) { - curr->bin.path_length = probe_read(curr->bin.path, event->exe.len, event->exe.off); + curr->bin.path_length = probe_read_kernel(curr->bin.path, event->exe.len, event->exe.off); if (curr->bin.path_length == 0) curr->bin.path_length = event->exe.len; } @@ -315,7 +315,7 @@ execve_send(struct sched_execve_args *ctx) // reuse p->args first string that contains the filename, this can't be // above 256 in size (otherwise the complete will be send via data msg) // which is okay because we need the 256 first bytes. - curr->bin.path_length = probe_read_str(curr->bin.path, BINARY_PATH_MAX_LEN, &p->args); + curr->bin.path_length = probe_read_kernel_str(curr->bin.path, BINARY_PATH_MAX_LEN, &p->args); if (curr->bin.path_length > 1) { // don't include the NULL byte in the length curr->bin.path_length--; diff --git a/bpf/process/bpf_exit.h b/bpf/process/bpf_exit.h index b683bac7a5d..7df5bdd84af 100644 --- a/bpf/process/bpf_exit.h +++ b/bpf/process/bpf_exit.h @@ -63,8 +63,8 @@ static inline __attribute__((always_inline)) void event_exit_send(void *ctx, __u * entry from the execve_map anyway and explicitly set it to the to tgid. */ exit->info.tid = tgid; - probe_read(&exit->info.code, sizeof(exit->info.code), - _(&task->exit_code)); + probe_read_kernel(&exit->info.code, sizeof(exit->info.code), + _(&task->exit_code)); perf_event_output_metric(ctx, MSG_OP_EXIT, &tcpmon_map, BPF_F_CURRENT_CPU, exit, size); } diff --git a/bpf/process/bpf_fork.c b/bpf/process/bpf_fork.c index f4b8e0f1215..9e8d51bf894 100644 --- a/bpf/process/bpf_fork.c +++ b/bpf/process/bpf_fork.c @@ -28,7 +28,7 @@ BPF_KPROBE(event_wake_up_new_task, struct task_struct *task) if (!task) return 0; - tgid = BPF_CORE_READ(task, tgid); + tgid = BPF_CORE_READ_KERNEL(task, tgid); /* Do not try to create any msg or calling execve_map_get * (that will add a new process in the execve_map) if we @@ -67,7 +67,7 @@ BPF_KPROBE(event_wake_up_new_task, struct task_struct *task) * wakes up it will be the only one in the thread group, and it is * the leader. Ensure to pass TID to user space. */ - msg.tid = BPF_CORE_READ(task, pid); + msg.tid = BPF_CORE_READ_KERNEL(task, pid); msg.ktime = curr->key.ktime; msg.nspid = curr->nspid; msg.flags = curr->flags; diff --git a/bpf/process/bpf_generic_tracepoint.c b/bpf/process/bpf_generic_tracepoint.c index f84367b9b23..1456784fa22 100644 --- a/bpf/process/bpf_generic_tracepoint.c +++ b/bpf/process/bpf_generic_tracepoint.c @@ -69,14 +69,14 @@ static inline __attribute__((always_inline)) unsigned long get_ctx_ul(void *src, case u64_ty: { u64 ret; - probe_read(&ret, sizeof(u64), src); + probe_read_kernel(&ret, sizeof(u64), src); return ret; } case size_type: { size_t ret; - probe_read(&ret, sizeof(size_t), src); + probe_read_kernel(&ret, sizeof(size_t), src); return (unsigned long)ret; } @@ -84,7 +84,7 @@ static inline __attribute__((always_inline)) unsigned long get_ctx_ul(void *src, case s32_ty: { s32 ret; - probe_read(&ret, sizeof(u32), src); + probe_read_kernel(&ret, sizeof(u32), src); return ret; } @@ -92,21 +92,21 @@ static inline __attribute__((always_inline)) unsigned long get_ctx_ul(void *src, case u32_ty: { u32 ret; - probe_read(&ret, sizeof(u32), src); + probe_read_kernel(&ret, sizeof(u32), src); return ret; } case char_buf: case string_type: { char *buff; - probe_read(&buff, sizeof(char *), src); + probe_read_kernel(&buff, sizeof(char *), src); return (unsigned long)buff; } case data_loc_type: { u32 ret; - probe_read(&ret, sizeof(ret), src); + probe_read_kernel(&ret, sizeof(ret), src); return ret; } @@ -117,14 +117,14 @@ static inline __attribute__((always_inline)) unsigned long get_ctx_ul(void *src, case skb_type: { struct sk_buff *skb; - probe_read(&skb, sizeof(struct sk_buff *), src); + probe_read_kernel(&skb, sizeof(struct sk_buff *), src); return (unsigned long)skb; } case sock_type: { struct sock *sk; - probe_read(&sk, sizeof(struct sock *), src); + probe_read_kernel(&sk, sizeof(struct sock *), src); return (unsigned long)sk; } diff --git a/bpf/process/bpf_loader.c b/bpf/process/bpf_loader.c index 332fc289bfd..b07fa3dd802 100644 --- a/bpf/process/bpf_loader.c +++ b/bpf/process/bpf_loader.c @@ -88,7 +88,7 @@ loader_kprobe(struct pt_regs *ctx) id_map = map_lookup_elem(&ids_map, &(__u32){ 0 }); if (!id_map) return 0; - id_pe = BPF_CORE_READ(pe, id); + id_pe = BPF_CORE_READ_KERNEL(pe, id); if (*id_map != id_pe) return 0; msg->pe = pe; @@ -97,7 +97,7 @@ loader_kprobe(struct pt_regs *ctx) } current = (struct task_struct *)get_current_task(); - tgid = BPF_CORE_READ(current, tgid); + tgid = BPF_CORE_READ_KERNEL(current, tgid); curr = execve_map_get_noinit(tgid); if (!curr) @@ -111,15 +111,15 @@ loader_kprobe(struct pt_regs *ctx) /* Send all events with valid build id, user space will sort * out duplicates. */ - msg->buildid_size = BPF_CORE_READ(mmap_event, build_id_size); + msg->buildid_size = BPF_CORE_READ_KERNEL(mmap_event, build_id_size); if (!msg->buildid_size) return 0; - probe_read(&msg->buildid[0], sizeof(msg->buildid), - _(&mmap_event->build_id[0])); + probe_read_kernel(&msg->buildid[0], sizeof(msg->buildid), + _(&mmap_event->build_id[0])); - path = BPF_CORE_READ(mmap_event, file_name); - len = probe_read_str(&msg->path, sizeof(msg->path), path); + path = BPF_CORE_READ_KERNEL(mmap_event, file_name); + len = probe_read_kernel_str(&msg->path, sizeof(msg->path), path); msg->path_size = (__u32)len; msg->pid = tgid; diff --git a/bpf/process/bpf_process_event.h b/bpf/process/bpf_process_event.h index b2b46845469..7a018acadf3 100644 --- a/bpf/process/bpf_process_event.h +++ b/bpf/process/bpf_process_event.h @@ -39,15 +39,15 @@ __get_auid(struct task_struct *task) return auid; if (bpf_core_field_exists(task->loginuid)) { - probe_read(&auid, sizeof(auid), _(&task->loginuid.val)); + probe_read_kernel(&auid, sizeof(auid), _(&task->loginuid.val)); } else { struct audit_task_info *audit; if (bpf_core_field_exists(task->audit)) { - probe_read(&audit, sizeof(audit), _(&task->audit)); + probe_read_kernel(&audit, sizeof(audit), _(&task->audit)); if (audit) { - probe_read(&auid, sizeof(__u32), - _(&audit->loginuid)); + probe_read_kernel(&auid, sizeof(__u32), + _(&audit->loginuid)); } } } @@ -88,7 +88,7 @@ static inline __attribute__((always_inline)) bool IS_ROOT(struct dentry *dentry) { struct dentry *d_parent; - probe_read(&d_parent, sizeof(d_parent), _(&dentry->d_parent)); + probe_read_kernel(&d_parent, sizeof(d_parent), _(&dentry->d_parent)); return (dentry == d_parent); } @@ -97,7 +97,7 @@ hlist_bl_unhashed(const struct hlist_bl_node *h) { struct hlist_bl_node **pprev; - probe_read(&pprev, sizeof(pprev), _(&h->pprev)); + probe_read_kernel(&pprev, sizeof(pprev), _(&h->pprev)); return !pprev; } @@ -153,7 +153,7 @@ prepend_name(char *buf, char **bufptr, int *buflen, const char *name, u32 namele // Needed to bound that for probe_read call. asm volatile("%[namelen] &= 0xff;\n" ::[namelen] "+r"(namelen) :); - probe_read(buf + buffer_offset + write_slash, namelen * sizeof(char), name); + probe_read_kernel(buf + buffer_offset + write_slash, namelen * sizeof(char), name); *bufptr = buf + buffer_offset; return write_slash ? 0 : -ENAMETOOLONG; @@ -204,28 +204,28 @@ cwd_read(struct cwd_read_data *data) return 1; } - probe_read(&vfsmnt_mnt_root, sizeof(vfsmnt_mnt_root), - _(&vfsmnt->mnt_root)); + probe_read_kernel(&vfsmnt_mnt_root, sizeof(vfsmnt_mnt_root), + _(&vfsmnt->mnt_root)); if (dentry == vfsmnt_mnt_root || IS_ROOT(dentry)) { struct mount *parent; - probe_read(&parent, sizeof(parent), _(&mnt->mnt_parent)); + probe_read_kernel(&parent, sizeof(parent), _(&mnt->mnt_parent)); /* Global root? */ if (data->mnt != parent) { - probe_read(&data->dentry, sizeof(data->dentry), - _(&mnt->mnt_mountpoint)); + probe_read_kernel(&data->dentry, sizeof(data->dentry), + _(&mnt->mnt_mountpoint)); data->mnt = parent; - probe_read(&data->vfsmnt, sizeof(data->vfsmnt), - _(&mnt->mnt)); + probe_read_kernel(&data->vfsmnt, sizeof(data->vfsmnt), + _(&mnt->mnt)); return 0; } // resolved all path components successfully data->resolved = true; return 1; } - probe_read(&parent, sizeof(parent), _(&dentry->d_parent)); - probe_read(&d_name, sizeof(d_name), _(&dentry->d_name)); + probe_read_kernel(&parent, sizeof(parent), _(&dentry->d_parent)); + probe_read_kernel(&d_name, sizeof(d_name), _(&dentry->d_name)); error = prepend_name(data->bf, &data->bptr, &data->blen, (const char *)d_name.name, d_name.len); // This will happen where the dentry name does not fit in the buffer. @@ -256,11 +256,11 @@ prepend_path(const struct path *path, const struct path *root, char *bf, }; int error = 0; - probe_read(&data.root_dentry, sizeof(data.root_dentry), - _(&root->dentry)); - probe_read(&data.root_mnt, sizeof(data.root_mnt), _(&root->mnt)); - probe_read(&data.dentry, sizeof(data.dentry), _(&path->dentry)); - probe_read(&data.vfsmnt, sizeof(data.vfsmnt), _(&path->mnt)); + probe_read_kernel(&data.root_dentry, sizeof(data.root_dentry), + _(&root->dentry)); + probe_read_kernel(&data.root_mnt, sizeof(data.root_mnt), _(&root->mnt)); + probe_read_kernel(&data.dentry, sizeof(data.dentry), _(&path->dentry)); + probe_read_kernel(&data.vfsmnt, sizeof(data.vfsmnt), _(&path->mnt)); data.mnt = real_mount(data.vfsmnt); #ifndef __V61_BPF_PROG @@ -290,7 +290,7 @@ path_with_deleted(const struct path *path, const struct path *root, char *bf, { struct dentry *dentry; - probe_read(&dentry, sizeof(dentry), _(&path->dentry)); + probe_read_kernel(&dentry, sizeof(dentry), _(&path->dentry)); if (d_unlinked(dentry)) { int error = prepend(buf, buflen, " (deleted)", 10); if (error) // will never happen as prepend will never return a value != 0 @@ -342,7 +342,7 @@ __d_path_local(const struct path *path, char *buf, int *buflen, int *error) struct fs_struct *fs; task = (struct task_struct *)get_current_task(); - probe_read(&fs, sizeof(fs), _(&task->fs)); + probe_read_kernel(&fs, sizeof(fs), _(&task->fs)); *error = path_with_deleted(path, _(&fs->root), buf, &res, buflen); return res; } @@ -386,7 +386,7 @@ getcwd(struct msg_process *curr, __u32 offset, __u32 proc_pid) int flags = 0, size; char *buffer; - probe_read(&fs, sizeof(fs), _(&task->fs)); + probe_read_kernel(&fs, sizeof(fs), _(&task->fs)); if (!fs) { curr->flags |= EVENT_ERROR_CWD; return 0; @@ -400,7 +400,7 @@ getcwd(struct msg_process *curr, __u32 offset, __u32 proc_pid) :); asm volatile("%[size] &= 0xff;\n" ::[size] "+r"(size) :); - probe_read((char *)curr + offset, size, buffer); + probe_read_kernel((char *)curr + offset, size, buffer); // Unfortunate special case for '/' where nothing was added we need // to truncate with '\n' for parser. @@ -421,9 +421,9 @@ event_set_clone(struct msg_process *pid) static inline __attribute__((always_inline)) void __get_caps(struct msg_capabilities *msg, const struct cred *cred) { - probe_read(&msg->effective, sizeof(__u64), _(&cred->cap_effective)); - probe_read(&msg->inheritable, sizeof(__u64), _(&cred->cap_inheritable)); - probe_read(&msg->permitted, sizeof(__u64), _(&cred->cap_permitted)); + probe_read_kernel(&msg->effective, sizeof(__u64), _(&cred->cap_effective)); + probe_read_kernel(&msg->inheritable, sizeof(__u64), _(&cred->cap_inheritable)); + probe_read_kernel(&msg->permitted, sizeof(__u64), _(&cred->cap_permitted)); } /* @get_current_subj_caps: @@ -463,7 +463,7 @@ get_current_subj_caps(struct msg_capabilities *msg, struct task_struct *task) const struct cred *cred; /* Get the task's subjective creds */ - probe_read(&cred, sizeof(cred), _(&task->cred)); + probe_read_kernel(&cred, sizeof(cred), _(&task->cred)); __get_caps(msg, cred); } @@ -473,17 +473,17 @@ get_current_subj_creds_uids(struct msg_cred_minimal *info, struct task_struct *t const struct cred *cred; /* Get the task's subjective creds */ - probe_read(&cred, sizeof(cred), _(&task->cred)); - - probe_read(&info->uid, sizeof(__u32), _(&cred->uid)); - probe_read(&info->gid, sizeof(__u32), _(&cred->gid)); - probe_read(&info->euid, sizeof(__u32), _(&cred->euid)); - probe_read(&info->egid, sizeof(__u32), _(&cred->egid)); - probe_read(&info->suid, sizeof(__u32), _(&cred->suid)); - probe_read(&info->sgid, sizeof(__u32), _(&cred->sgid)); - probe_read(&info->fsuid, sizeof(__u32), _(&cred->fsuid)); - probe_read(&info->fsgid, sizeof(__u32), _(&cred->fsgid)); - probe_read(&info->securebits, sizeof(__u32), _(&cred->securebits)); + probe_read_kernel(&cred, sizeof(cred), _(&task->cred)); + + probe_read_kernel(&info->uid, sizeof(__u32), _(&cred->uid)); + probe_read_kernel(&info->gid, sizeof(__u32), _(&cred->gid)); + probe_read_kernel(&info->euid, sizeof(__u32), _(&cred->euid)); + probe_read_kernel(&info->egid, sizeof(__u32), _(&cred->egid)); + probe_read_kernel(&info->suid, sizeof(__u32), _(&cred->suid)); + probe_read_kernel(&info->sgid, sizeof(__u32), _(&cred->sgid)); + probe_read_kernel(&info->fsuid, sizeof(__u32), _(&cred->fsuid)); + probe_read_kernel(&info->fsgid, sizeof(__u32), _(&cred->fsgid)); + probe_read_kernel(&info->securebits, sizeof(__u32), _(&cred->securebits)); } static inline __attribute__((always_inline)) void @@ -492,55 +492,55 @@ get_namespaces(struct msg_ns *msg, struct task_struct *task) struct nsproxy *nsproxy; struct nsproxy nsp; - probe_read(&nsproxy, sizeof(nsproxy), _(&task->nsproxy)); - probe_read(&nsp, sizeof(nsp), _(nsproxy)); + probe_read_kernel(&nsproxy, sizeof(nsproxy), _(&task->nsproxy)); + probe_read_kernel(&nsp, sizeof(nsp), _(nsproxy)); - probe_read(&msg->uts_inum, sizeof(msg->uts_inum), - _(&nsp.uts_ns->ns.inum)); - probe_read(&msg->ipc_inum, sizeof(msg->ipc_inum), - _(&nsp.ipc_ns->ns.inum)); - probe_read(&msg->mnt_inum, sizeof(msg->mnt_inum), - _(&nsp.mnt_ns->ns.inum)); + probe_read_kernel(&msg->uts_inum, sizeof(msg->uts_inum), + _(&nsp.uts_ns->ns.inum)); + probe_read_kernel(&msg->ipc_inum, sizeof(msg->ipc_inum), + _(&nsp.ipc_ns->ns.inum)); + probe_read_kernel(&msg->mnt_inum, sizeof(msg->mnt_inum), + _(&nsp.mnt_ns->ns.inum)); { struct pid *p = 0; - probe_read(&p, sizeof(p), _(&task->thread_pid)); + probe_read_kernel(&p, sizeof(p), _(&task->thread_pid)); if (p) { int level = 0; struct upid up; - probe_read(&level, sizeof(level), _(&p->level)); - probe_read(&up, sizeof(up), _(&p->numbers[level])); - probe_read(&msg->pid_inum, sizeof(msg->pid_inum), - _(&up.ns->ns.inum)); + probe_read_kernel(&level, sizeof(level), _(&p->level)); + probe_read_kernel(&up, sizeof(up), _(&p->numbers[level])); + probe_read_kernel(&msg->pid_inum, sizeof(msg->pid_inum), + _(&up.ns->ns.inum)); } else msg->pid_inum = 0; } - probe_read(&msg->pid_for_children_inum, - sizeof(msg->pid_for_children_inum), - _(&nsp.pid_ns_for_children->ns.inum)); - probe_read(&msg->net_inum, sizeof(msg->net_inum), - _(&nsp.net_ns->ns.inum)); + probe_read_kernel(&msg->pid_for_children_inum, + sizeof(msg->pid_for_children_inum), + _(&nsp.pid_ns_for_children->ns.inum)); + probe_read_kernel(&msg->net_inum, sizeof(msg->net_inum), + _(&nsp.net_ns->ns.inum)); // this also includes time_ns_for_children if (bpf_core_field_exists(nsproxy->time_ns)) { - probe_read(&msg->time_inum, sizeof(msg->time_inum), - _(&nsp.time_ns->ns.inum)); - probe_read(&msg->time_for_children_inum, - sizeof(msg->time_for_children_inum), - _(&nsp.time_ns_for_children->ns.inum)); + probe_read_kernel(&msg->time_inum, sizeof(msg->time_inum), + _(&nsp.time_ns->ns.inum)); + probe_read_kernel(&msg->time_for_children_inum, + sizeof(msg->time_for_children_inum), + _(&nsp.time_ns_for_children->ns.inum)); } - probe_read(&msg->cgroup_inum, sizeof(msg->cgroup_inum), - _(&nsp.cgroup_ns->ns.inum)); + probe_read_kernel(&msg->cgroup_inum, sizeof(msg->cgroup_inum), + _(&nsp.cgroup_ns->ns.inum)); { struct mm_struct *mm; struct user_namespace *user_ns; - probe_read(&mm, sizeof(mm), _(&task->mm)); - probe_read(&user_ns, sizeof(user_ns), _(&mm->user_ns)); - probe_read(&msg->user_inum, sizeof(msg->user_inum), - _(&user_ns->ns.inum)); + probe_read_kernel(&mm, sizeof(mm), _(&task->mm)); + probe_read_kernel(&user_ns, sizeof(user_ns), _(&mm->user_ns)); + probe_read_kernel(&msg->user_inum, sizeof(msg->user_inum), + _(&user_ns->ns.inum)); } } @@ -566,7 +566,7 @@ __event_get_current_cgroup_name(struct cgroup *cgrp, name = get_cgroup_name(cgrp); if (name) - probe_read_str(msg->kube.docker_id, KN_NAME_LENGTH, name); + probe_read_kernel_str(msg->kube.docker_id, KN_NAME_LENGTH, name); else process->flags |= EVENT_ERROR_CGROUP_NAME; } diff --git a/bpf/process/data_event.h b/bpf/process/data_event.h index 974fee26b2c..d6f01cf5616 100644 --- a/bpf/process/data_event.h +++ b/bpf/process/data_event.h @@ -6,9 +6,15 @@ #include "bpf_tracing.h" #include "data_msg.h" +#include "types/probe_read_kernel_or_user.h" + +#define DATA_EVENT_MEM_UNKNOWN 0 +#define DATA_EVENT_MEM_KERNEL 1 +#define DATA_EVENT_MEM_USER 2 +#define DATA_EVENT_MEM_KERNEL_OR_USER 3 static inline __attribute__((always_inline)) long -__do_bytes(void *ctx, struct msg_data *msg, unsigned long uptr, size_t bytes) +__do_bytes(void *ctx, int mem, struct msg_data *msg, unsigned long uptr, size_t bytes) { int err; @@ -27,7 +33,21 @@ __do_bytes(void *ctx, struct msg_data *msg, unsigned long uptr, size_t bytes) : [bytes] "+r"(bytes) :); #endif - err = probe_read(&msg->arg[0], bytes, (char *)uptr); + switch (mem) { + case DATA_EVENT_MEM_UNKNOWN: + err = probe_read(&msg->arg[0], bytes, (char *)uptr); + break; + case DATA_EVENT_MEM_KERNEL: + err = probe_read_kernel(&msg->arg[0], bytes, (char *)uptr); + break; + case DATA_EVENT_MEM_USER: + err = probe_read_user(&msg->arg[0], bytes, (char *)uptr); + break; + case DATA_EVENT_MEM_KERNEL_OR_USER: + err = probe_read_kernel_or_user(&msg->arg[0], bytes, (char *)uptr); + break; + } + if (err < 0) return err; @@ -39,14 +59,14 @@ __do_bytes(void *ctx, struct msg_data *msg, unsigned long uptr, size_t bytes) } static inline __attribute__((always_inline)) long -do_bytes(void *ctx, struct msg_data *msg, unsigned long arg, size_t bytes) +do_bytes(void *ctx, int mem, struct msg_data *msg, unsigned long arg, size_t bytes) { size_t rd_bytes = 0; int err, i __maybe_unused; #ifdef __LARGE_BPF_PROG for (i = 0; i < 10; i++) { - err = __do_bytes(ctx, msg, arg + rd_bytes, bytes - rd_bytes); + err = __do_bytes(ctx, mem, msg, arg + rd_bytes, bytes - rd_bytes); if (err < 0) return err; rd_bytes += err; @@ -55,7 +75,7 @@ do_bytes(void *ctx, struct msg_data *msg, unsigned long arg, size_t bytes) } #else #define BYTES_COPY \ - err = __do_bytes(ctx, msg, arg + rd_bytes, bytes - rd_bytes); \ + err = __do_bytes(ctx, mem, msg, arg + rd_bytes, bytes - rd_bytes); \ if (err < 0) \ return err; \ rd_bytes += err; \ @@ -75,7 +95,7 @@ do_bytes(void *ctx, struct msg_data *msg, unsigned long arg, size_t bytes) } static inline __attribute__((always_inline)) long -__do_str(void *ctx, struct msg_data *msg, unsigned long arg, bool *done) +__do_str(void *ctx, int mem, struct msg_data *msg, unsigned long arg, bool *done) { size_t size, max = sizeof(msg->arg) - 1; long ret; @@ -88,7 +108,21 @@ __do_str(void *ctx, struct msg_data *msg, unsigned long arg, bool *done) : [max] "+r"(max) :); - ret = probe_read_str(&msg->arg[0], max, (char *)arg); + switch (mem) { + case DATA_EVENT_MEM_UNKNOWN: + ret = probe_read_str(&msg->arg[0], max, (char *)arg); + break; + case DATA_EVENT_MEM_KERNEL: + ret = probe_read_kernel_str(&msg->arg[0], max, (char *)arg); + break; + case DATA_EVENT_MEM_USER: + ret = probe_read_user_str(&msg->arg[0], max, (char *)arg); + break; + case DATA_EVENT_MEM_KERNEL_OR_USER: + ret = probe_read_kernel_or_user_str(&msg->arg[0], max, (char *)arg); + break; + } + if (ret < 0) return ret; @@ -110,7 +144,7 @@ __do_str(void *ctx, struct msg_data *msg, unsigned long arg, bool *done) } static inline __attribute__((always_inline)) long -do_str(void *ctx, struct msg_data *msg, unsigned long arg, +do_str(void *ctx, int mem, struct msg_data *msg, unsigned long arg, size_t bytes __maybe_unused) { size_t rd_bytes = 0; @@ -121,7 +155,7 @@ do_str(void *ctx, struct msg_data *msg, unsigned long arg, #define __CNT 2 #pragma unroll for (i = 0; i < __CNT; i++) { - ret = __do_str(ctx, msg, arg + rd_bytes, &done); + ret = __do_str(ctx, mem, msg, arg + rd_bytes, &done); if (ret < 0) return ret; rd_bytes += ret; @@ -135,9 +169,9 @@ do_str(void *ctx, struct msg_data *msg, unsigned long arg, } static inline __attribute__((always_inline)) size_t data_event( - void *ctx, struct data_event_desc *desc, unsigned long uptr, + void *ctx, int mem, struct data_event_desc *desc, unsigned long uptr, size_t size, struct bpf_map_def *heap, - long (*do_data_event)(void *, struct msg_data *, unsigned long, size_t)) + long (*do_data_event)(void *, int, struct msg_data *, unsigned long, size_t)) { struct msg_data *msg; int zero = 0, err; @@ -165,7 +199,7 @@ static inline __attribute__((always_inline)) size_t data_event( * Leftover for data_event_str is always 0, because we don't know * how much more was there to copy. */ - err = do_data_event(ctx, msg, uptr, size); + err = do_data_event(ctx, mem, msg, uptr, size); if (err < 0) { desc->error = err; @@ -193,10 +227,10 @@ static inline __attribute__((always_inline)) size_t data_event( * Returns size of struct @desc object or 0 in case of error. */ static inline __attribute__((always_inline)) size_t -data_event_bytes(void *ctx, struct data_event_desc *desc, unsigned long uptr, +data_event_bytes(void *ctx, int mem, struct data_event_desc *desc, unsigned long uptr, size_t size, struct bpf_map_def *heap) { - return data_event(ctx, desc, uptr, size, heap, do_bytes); + return data_event(ctx, mem, desc, uptr, size, heap, do_bytes); } /** @@ -210,10 +244,10 @@ data_event_bytes(void *ctx, struct data_event_desc *desc, unsigned long uptr, * Returns size of struct @desc object or 0 in case of error. */ static inline __attribute__((always_inline)) size_t -data_event_str(void *ctx, struct data_event_desc *desc, unsigned long uptr, +data_event_str(void *ctx, int mem, struct data_event_desc *desc, unsigned long uptr, struct bpf_map_def *heap) { - return data_event(ctx, desc, uptr, -1, heap, do_str); + return data_event(ctx, mem, desc, uptr, -1, heap, do_str); } #endif /* __DATA_EVENT_H__ */ diff --git a/bpf/process/generic_calls.h b/bpf/process/generic_calls.h index d23ac848fbc..fc649df5265 100644 --- a/bpf/process/generic_calls.h +++ b/bpf/process/generic_calls.h @@ -84,7 +84,7 @@ generic_setup_32bit_syscall(struct msg_generic_kprobe *e, u8 op) case MSG_OP_GENERIC_TRACEPOINT: case MSG_OP_GENERIC_KPROBE: info = (struct thread_info *)get_current_task(); - probe_read(&status, sizeof(status), _(&info->status)); + probe_read_kernel(&status, sizeof(status), _(&info->status)); e->sel.is32BitSyscall = status & TS_COMPAT; default: break; diff --git a/bpf/process/types/basic.h b/bpf/process/types/basic.h index f4a57438603..e541edc66ca 100644 --- a/bpf/process/types/basic.h +++ b/bpf/process/types/basic.h @@ -22,6 +22,7 @@ #include "common.h" #include "process/data_event.h" #include "process/bpf_enforcer.h" +#include "probe_read_kernel_or_user.h" /* Type IDs form API with user space generickprobe.go */ enum { @@ -284,7 +285,7 @@ parse_iovec_array(long off, unsigned long arg, int i, unsigned long max, __u64 size; int err; - err = probe_read(&iov, sizeof(iov), (struct iovec *)(arg + index)); + err = probe_read_kernel_or_user(&iov, sizeof(iov), (struct iovec *)(arg + index)); if (err < 0) return char_buf_pagefault; size = iov.iov_len; @@ -294,7 +295,7 @@ parse_iovec_array(long off, unsigned long arg, int i, unsigned long max, return char_buf_toolarge; asm volatile("%[size] &= 0xfff;\n" ::[size] "+r"(size) :); - err = probe_read(args_off(e, off), size, (char *)iov.iov_base); + err = probe_read_kernel_or_user(args_off(e, off), size, (char *)iov.iov_base); if (err < 0) return char_buf_pagefault; return size; @@ -452,7 +453,7 @@ copy_path(char *args, const struct path *arg) asm volatile("%[size] &= 0xff;\n" ::[size] "+r"(size) :); - probe_read(curr, size, buffer); + probe_read_kernel_or_user(curr, size, buffer); *s = size; size += 4; @@ -488,10 +489,10 @@ copy_strings(char *args, char *arg, int max_size) int *s = (int *)args; long size; - // probe_read_str() always nul-terminates the string. + // probe_read_kernel_or_user_str() always nul-terminates the string. // So add one to the length to allow for it. This should // result in us honouring our max_size correctly. - size = probe_read_str(&args[4], max_size + 1, arg); + size = probe_read_kernel_or_user_str(&args[4], max_size + 1, arg); if (size <= 1) return invalid_ty; // Remove the nul character from end. @@ -508,10 +509,10 @@ static inline __attribute__((always_inline)) long copy_skb(char *args, struct skb_type *skb_event = (struct skb_type *)args; /* struct values */ - probe_read(&skb_event->hash, sizeof(__u32), _(&skb->hash)); - probe_read(&skb_event->len, sizeof(__u32), _(&skb->len)); - probe_read(&skb_event->priority, sizeof(__u32), _(&skb->priority)); - probe_read(&skb_event->mark, sizeof(__u32), _(&skb->mark)); + probe_read_kernel_or_user(&skb_event->hash, sizeof(__u32), _(&skb->hash)); + probe_read_kernel_or_user(&skb_event->len, sizeof(__u32), _(&skb->len)); + probe_read_kernel_or_user(&skb_event->priority, sizeof(__u32), _(&skb->priority)); + probe_read_kernel_or_user(&skb_event->mark, sizeof(__u32), _(&skb->mark)); /* socket data */ set_event_from_skb(skb_event, skb); @@ -537,10 +538,10 @@ copy_user_ns(char *args, unsigned long arg) struct msg_user_namespace *u_ns_info = (struct msg_user_namespace *)args; - probe_read(&u_ns_info->level, sizeof(__s32), _(&ns->level)); - probe_read(&u_ns_info->uid, sizeof(__u32), _(&ns->owner)); - probe_read(&u_ns_info->gid, sizeof(__u32), _(&ns->group)); - probe_read(&u_ns_info->ns_inum, sizeof(__u32), _(&ns->ns.inum)); + probe_read_kernel_or_user(&u_ns_info->level, sizeof(__s32), _(&ns->level)); + probe_read_kernel_or_user(&u_ns_info->uid, sizeof(__u32), _(&ns->owner)); + probe_read_kernel_or_user(&u_ns_info->gid, sizeof(__u32), _(&ns->group)); + probe_read_kernel_or_user(&u_ns_info->ns_inum, sizeof(__u32), _(&ns->ns.inum)); return sizeof(struct msg_user_namespace); } @@ -554,20 +555,20 @@ static inline __attribute__((always_inline)) long copy_cred(char *args, struct msg_capabilities *caps = &info->caps; struct msg_user_namespace *user_ns_info = &info->user_ns; - probe_read(&info->uid, sizeof(__u32), _(&cred->uid)); - probe_read(&info->gid, sizeof(__u32), _(&cred->gid)); - probe_read(&info->euid, sizeof(__u32), _(&cred->euid)); - probe_read(&info->egid, sizeof(__u32), _(&cred->egid)); - probe_read(&info->suid, sizeof(__u32), _(&cred->suid)); - probe_read(&info->sgid, sizeof(__u32), _(&cred->sgid)); - probe_read(&info->fsuid, sizeof(__u32), _(&cred->fsuid)); - probe_read(&info->fsgid, sizeof(__u32), _(&cred->fsgid)); + probe_read_kernel_or_user(&info->uid, sizeof(__u32), _(&cred->uid)); + probe_read_kernel_or_user(&info->gid, sizeof(__u32), _(&cred->gid)); + probe_read_kernel_or_user(&info->euid, sizeof(__u32), _(&cred->euid)); + probe_read_kernel_or_user(&info->egid, sizeof(__u32), _(&cred->egid)); + probe_read_kernel_or_user(&info->suid, sizeof(__u32), _(&cred->suid)); + probe_read_kernel_or_user(&info->sgid, sizeof(__u32), _(&cred->sgid)); + probe_read_kernel_or_user(&info->fsuid, sizeof(__u32), _(&cred->fsuid)); + probe_read_kernel_or_user(&info->fsgid, sizeof(__u32), _(&cred->fsgid)); info->pad = 0; - probe_read(&info->securebits, sizeof(__u32), _(&cred->securebits)); + probe_read_kernel_or_user(&info->securebits, sizeof(__u32), _(&cred->securebits)); __get_caps(caps, cred); - probe_read(&ns, sizeof(ns), _(&cred->user_ns)); + probe_read_kernel_or_user(&ns, sizeof(ns), _(&cred->user_ns)); copy_user_ns((char *)user_ns_info, (unsigned long)ns); return sizeof(struct msg_cred); @@ -595,15 +596,15 @@ copy_load_module(char *args, unsigned long arg) memset(info, 0, sizeof(struct tg_kernel_module)); - if (BPF_CORE_READ_INTO(&name, mod, name) != 0) + if (BPF_CORE_READ_KERNEL_OR_USER_INTO(&name, mod, name) != 0) return 0; - if (probe_read_str(&info->name, TG_MODULE_NAME_LEN - 1, name) < 0) + if (probe_read_kernel_or_user_str(&info->name, TG_MODULE_NAME_LEN - 1, name) < 0) return 0; - BPF_CORE_READ_INTO(&info->taints, mod, mod, taints); + BPF_CORE_READ_KERNEL_OR_USER_INTO(&info->taints, mod, mod, taints); - if (BPF_CORE_READ_INTO(&ok, mod, sig_ok) == 0) + if (BPF_CORE_READ_KERNEL_OR_USER_INTO(&ok, mod, sig_ok) == 0) info->sig_ok = !!ok; return sizeof(struct tg_kernel_module); @@ -617,10 +618,10 @@ copy_kernel_module(char *args, unsigned long arg) memset(info, 0, sizeof(struct tg_kernel_module)); - if (probe_read_str(&info->name, TG_MODULE_NAME_LEN - 1, mod->name) < 0) + if (probe_read_kernel_or_user_str(&info->name, TG_MODULE_NAME_LEN - 1, mod->name) < 0) return 0; - BPF_CORE_READ_INTO(&info->taints, mod, taints); + BPF_CORE_READ_KERNEL_OR_USER_INTO(&info->taints, mod, taints); /* * Todo: allow to check if module is signed here too. @@ -681,7 +682,7 @@ __copy_char_buf(void *ctx, long off, unsigned long arg, unsigned long bytes, */ if (bytes >= 0x1000) { s[0] = 1; - return data_event_bytes(ctx, + return data_event_bytes(ctx, DATA_EVENT_MEM_KERNEL_OR_USER, (struct data_event_desc *)&s[1], arg, bytes, data_heap) + 4; @@ -696,7 +697,7 @@ __copy_char_buf(void *ctx, long off, unsigned long arg, unsigned long bytes, rd_bytes = bytes < 0x1000 ? bytes : 0xfff; asm volatile("%[rd_bytes] &= 0xfff;\n" ::[rd_bytes] "+r"(rd_bytes) :); - err = probe_read(&s[2], rd_bytes, (char *)arg); + err = probe_read_kernel_or_user(&s[2], rd_bytes, (char *)arg); if (err < 0) return return_error(s, char_buf_pagefault); s[0] = (int)bytes; @@ -720,7 +721,7 @@ copy_char_buf(void *ctx, long off, unsigned long arg, int argm, return return_error(s, char_buf_saved_for_retprobe); } meta = get_arg_meta(argm, e); - probe_read(&bytes, sizeof(bytes), &meta); + probe_read_kernel_or_user(&bytes, sizeof(bytes), &meta); return __copy_char_buf(ctx, off, arg, bytes, has_max_data(argm), e, data_heap); } @@ -873,11 +874,11 @@ filter_char_buf_equal(struct selector_arg_filter *filter, char *arg_str, uint or : "i"(STRING_MAPS_HEAP_MASK)); #ifdef __LARGE_BPF_PROG if (index <= 5) - probe_read(&heap[1], len, arg_str); + probe_read_kernel(&heap[1], len, arg_str); else - probe_read(&heap[2], len, arg_str); + probe_read_kernel(&heap[2], len, arg_str); #else - probe_read(&heap[1], len, arg_str); + probe_read_kernel(&heap[1], len, arg_str); #endif // Pad string to multiple of key increment size @@ -887,11 +888,11 @@ filter_char_buf_equal(struct selector_arg_filter *filter, char *arg_str, uint or : "i"(STRING_MAPS_HEAP_MASK)); #ifdef __LARGE_BPF_PROG if (index <= 5) - probe_read(heap + len + 1, (padded_len - len) & STRING_MAPS_COPY_MASK, zero_heap); + probe_read_kernel(heap + len + 1, (padded_len - len) & STRING_MAPS_COPY_MASK, zero_heap); else - probe_read(heap + len + 2, (padded_len - len) & STRING_MAPS_COPY_MASK, zero_heap); + probe_read_kernel(heap + len + 2, (padded_len - len) & STRING_MAPS_COPY_MASK, zero_heap); #else - probe_read(heap + len + 1, (padded_len - len) & STRING_MAPS_COPY_MASK, zero_heap); + probe_read_kernel(heap + len + 1, (padded_len - len) & STRING_MAPS_COPY_MASK, zero_heap); #endif } @@ -933,7 +934,7 @@ filter_char_buf_prefix(struct selector_arg_filter *filter, char *arg_str, uint a : [arg_len] "+r"(arg_len) : [mask] "i"(STRING_PREFIX_MAX_LENGTH - 1)); - probe_read(arg->data, arg_len & (STRING_PREFIX_MAX_LENGTH - 1), arg_str); + probe_read_kernel(arg->data, arg_len & (STRING_PREFIX_MAX_LENGTH - 1), arg_str); __u8 *pass = map_lookup_elem(addrmap, arg); @@ -1254,9 +1255,9 @@ copy_bpf_attr(char *args, unsigned long arg) struct bpf_info_type *bpf_info = (struct bpf_info_type *)args; /* struct values */ - probe_read(&bpf_info->prog_type, sizeof(__u32), _(&ba->prog_type)); - probe_read(&bpf_info->insn_cnt, sizeof(__u32), _(&ba->insn_cnt)); - probe_read(&bpf_info->prog_name, BPF_OBJ_NAME_LEN, _(&ba->prog_name)); + probe_read_kernel_or_user(&bpf_info->prog_type, sizeof(__u32), _(&ba->prog_type)); + probe_read_kernel_or_user(&bpf_info->insn_cnt, sizeof(__u32), _(&ba->insn_cnt)); + probe_read_kernel_or_user(&bpf_info->prog_name, BPF_OBJ_NAME_LEN, _(&ba->prog_name)); return sizeof(struct bpf_info_type); } @@ -1271,16 +1272,16 @@ copy_perf_event(char *args, unsigned long arg) /* struct values */ __u64 kprobe_func_addr = 0; - probe_read(&kprobe_func_addr, sizeof(__u64), - _(&p_event->attr.kprobe_func)); - probe_read_str(&event_info->kprobe_func, KSYM_NAME_LEN, - (char *)kprobe_func_addr); + probe_read_kernel_or_user(&kprobe_func_addr, sizeof(__u64), + _(&p_event->attr.kprobe_func)); + probe_read_kernel_or_user_str(&event_info->kprobe_func, KSYM_NAME_LEN, + (char *)kprobe_func_addr); - probe_read(&event_info->type, sizeof(__u32), _(&p_event->attr.type)); - probe_read(&event_info->config, sizeof(__u64), - _(&p_event->attr.config)); - probe_read(&event_info->probe_offset, sizeof(__u64), - _(&p_event->attr.probe_offset)); + probe_read_kernel_or_user(&event_info->type, sizeof(__u32), _(&p_event->attr.type)); + probe_read_kernel_or_user(&event_info->config, sizeof(__u64), + _(&p_event->attr.config)); + probe_read_kernel_or_user(&event_info->probe_offset, sizeof(__u64), + _(&p_event->attr.probe_offset)); return sizeof(struct perf_event_info_type); } @@ -1292,13 +1293,13 @@ copy_bpf_map(char *args, unsigned long arg) struct bpf_map_info_type *map_info = (struct bpf_map_info_type *)args; /* struct values */ - probe_read(&map_info->map_type, sizeof(__u32), _(&bpfmap->map_type)); - probe_read(&map_info->key_size, sizeof(__u32), _(&bpfmap->key_size)); - probe_read(&map_info->value_size, sizeof(__u32), - _(&bpfmap->value_size)); - probe_read(&map_info->max_entries, sizeof(__u32), - _(&bpfmap->max_entries)); - probe_read(&map_info->map_name, BPF_OBJ_NAME_LEN, _(&bpfmap->name)); + probe_read_kernel_or_user(&map_info->map_type, sizeof(__u32), _(&bpfmap->map_type)); + probe_read_kernel_or_user(&map_info->key_size, sizeof(__u32), _(&bpfmap->key_size)); + probe_read_kernel_or_user(&map_info->value_size, sizeof(__u32), + _(&bpfmap->value_size)); + probe_read_kernel_or_user(&map_info->max_entries, sizeof(__u32), + _(&bpfmap->max_entries)); + probe_read_kernel_or_user(&map_info->map_name, BPF_OBJ_NAME_LEN, _(&bpfmap->name)); return sizeof(struct bpf_map_info_type); } @@ -1321,7 +1322,7 @@ copy_iov_iter(void *ctx, long off, unsigned long arg, int argm, struct msg_gener goto nodata; tmp = _(&iov_iter->iter_type); - probe_read(&iter_type, sizeof(iter_type), tmp); + probe_read_kernel_or_user(&iter_type, sizeof(iter_type), tmp); if (bpf_core_enum_value_exists(enum iter_type, ITER_IOVEC)) iter_iovec = bpf_core_enum_value(enum iter_type, ITER_IOVEC); @@ -1333,13 +1334,13 @@ copy_iov_iter(void *ctx, long off, unsigned long arg, int argm, struct msg_gener if (iter_type == iter_iovec) { tmp = _(&iov_iter->kvec); - probe_read(&kvec, sizeof(kvec), tmp); + probe_read_kernel_or_user(&kvec, sizeof(kvec), tmp); tmp = _(&kvec->iov_base); - probe_read(&buf, sizeof(buf), tmp); + probe_read_kernel_or_user(&buf, sizeof(buf), tmp); tmp = _(&kvec->iov_len); - probe_read(&count, sizeof(count), tmp); + probe_read_kernel_or_user(&count, sizeof(count), tmp); return __copy_char_buf(ctx, off, (unsigned long)buf, count, has_max_data(argm), e, data_heap); @@ -1348,10 +1349,10 @@ copy_iov_iter(void *ctx, long off, unsigned long arg, int argm, struct msg_gener #ifdef __V61_BPF_PROG if (iter_type == iter_ubuf) { tmp = _(&iov_iter->ubuf); - probe_read(&buf, sizeof(buf), tmp); + probe_read_kernel_or_user(&buf, sizeof(buf), tmp); tmp = _(&iov_iter->count); - probe_read(&count, sizeof(count), tmp); + probe_read_kernel_or_user(&count, sizeof(count), tmp); return __copy_char_buf(ctx, off, (unsigned long)buf, count, has_max_data(argm), e, data_heap); @@ -1713,7 +1714,7 @@ static inline __attribute__((always_inline)) int match_binaries(__u32 selidx) // prepare the key on the stack to perform lookup in the LPM_TRIE memset(&prefix_key, 0, sizeof(prefix_key)); prefix_key.prefixlen = current->bin.path_length * 8; // prefixlen is in bits - ret = probe_read(prefix_key.data, current->bin.path_length & (STRING_PREFIX_MAX_LENGTH - 1), current->bin.path); + ret = probe_read_kernel(prefix_key.data, current->bin.path_length & (STRING_PREFIX_MAX_LENGTH - 1), current->bin.path); if (ret < 0) return 0; found_key = map_lookup_elem(path_map, &prefix_key); @@ -1944,8 +1945,8 @@ installfd(struct msg_generic_kprobe *e, int fd, int name, bool follow) : [size] "+r"(size) :); - probe_read(&val.file[0], size + 4 /* size */ + 4 /* flags */, - &e->args[nameoff]); + probe_read_kernel(&val.file[0], size + 4 /* size */ + 4 /* flags */, + &e->args[nameoff]); map_update_elem(&fdinstall_map, &key, &val, BPF_ANY); } else { err = map_delete_elem(&fdinstall_map, &key); @@ -2092,7 +2093,7 @@ rate_limit(__u64 ratelimit_interval, __u64 ratelimit_scope, struct msg_generic_k } // Clean the heap - probe_read(key->data, MAX_POSSIBLE_ARGS * KEY_BYTES_PER_ARG, ro_heap); + probe_read_kernel(key->data, MAX_POSSIBLE_ARGS * KEY_BYTES_PER_ARG, ro_heap); dst = key->data; for (i = 0; i < MAX_POSSIBLE_ARGS; i++) { @@ -2109,7 +2110,7 @@ rate_limit(__u64 ratelimit_interval, __u64 ratelimit_scope, struct msg_generic_k asm volatile("%[arg_size] &= 0x3f;\n" // ensure this mask is greater than KEY_BYTES_PER_ARG : [arg_size] "+r"(arg_size) :); - probe_read(&dst[index], arg_size, &e->args[key_index]); + probe_read_kernel(&dst[index], arg_size, &e->args[key_index]); index += arg_size; } } @@ -2529,19 +2530,19 @@ read_call_arg(void *ctx, struct msg_generic_kprobe *e, int index, int type, struct file *file; arg = (unsigned long)_(&kiocb->ki_filp); - probe_read(&file, sizeof(file), (const void *)arg); + probe_read_kernel_or_user(&file, sizeof(file), (const void *)arg); arg = (unsigned long)file; } // fallthrough to file_ty case file_ty: { struct file *file; - probe_read(&file, sizeof(file), &arg); + probe_read_kernel_or_user(&file, sizeof(file), &arg); path_arg = _(&file->f_path); } // fallthrough to copy_path case path_ty: { if (!path_arg) - probe_read(&path_arg, sizeof(path_arg), &arg); + probe_read_kernel_or_user(&path_arg, sizeof(path_arg), &arg); size = copy_path(args, path_arg); break; } @@ -2551,23 +2552,23 @@ read_call_arg(void *ctx, struct msg_generic_kprobe *e, int index, int type, __u32 fd; key.tid = get_current_pid_tgid() >> 32; - probe_read(&fd, sizeof(__u32), &arg); + probe_read_kernel_or_user(&fd, sizeof(__u32), &arg); key.fd = fd; val = map_lookup_elem(&fdinstall_map, &key); if (val) { __u32 bytes = (__u32)val->file[0]; - probe_read(&args[0], sizeof(__u32), &fd); + probe_read_kernel_or_user(&args[0], sizeof(__u32), &fd); asm volatile("%[bytes] &= 0xff;\n" : [bytes] "+r"(bytes) :); - probe_read(&args[4], bytes + 4, (char *)&val->file[0]); + probe_read_kernel_or_user(&args[4], bytes + 4, (char *)&val->file[0]); size = bytes + 4 + 4; // flags - probe_read(&args[size], 4, - (char *)&val->file[size - 4]); + probe_read_kernel_or_user(&args[size], 4, + (char *)&val->file[size - 4]); size += 4; } else { /* If filter specification is fd type then we @@ -2583,15 +2584,15 @@ read_call_arg(void *ctx, struct msg_generic_kprobe *e, int index, int type, struct file *file; arg = (unsigned long)_(&bprm->file); - probe_read(&file, sizeof(file), (const void *)arg); + probe_read_kernel_or_user(&file, sizeof(file), (const void *)arg); path_arg = _(&file->f_path); size = copy_path(args, path_arg); } break; #endif case filename_ty: { struct filename *file; - probe_read(&file, sizeof(file), &arg); - probe_read(&arg, sizeof(arg), &file->name); + probe_read_kernel_or_user(&file, sizeof(file), &arg); + probe_read_kernel_or_user(&arg, sizeof(arg), &file->name); } // fallthrough to copy_string case string_type: @@ -2613,26 +2614,26 @@ read_call_arg(void *ctx, struct msg_generic_kprobe *e, int index, int type, case size_type: case s64_ty: case u64_ty: - probe_read(args, sizeof(__u64), &arg); + probe_read_kernel_or_user(args, sizeof(__u64), &arg); size = sizeof(__u64); break; /* Consolidate all the types to save instructions */ case int_type: case s32_ty: case u32_ty: - probe_read(args, sizeof(__u32), &arg); + probe_read_kernel_or_user(args, sizeof(__u32), &arg); size = sizeof(__u32); break; case s16_ty: case u16_ty: /* read 2 bytes, but send 4 to keep alignment */ - probe_read(args, sizeof(__u16), &arg); + probe_read_kernel_or_user(args, sizeof(__u16), &arg); size = sizeof(__u32); break; case s8_ty: case u8_ty: /* read 1 byte, but send 4 to keep alignment */ - probe_read(args, sizeof(__u8), &arg); + probe_read_kernel_or_user(args, sizeof(__u8), &arg); size = sizeof(__u32); break; case skb_type: @@ -2655,7 +2656,7 @@ read_call_arg(void *ctx, struct msg_generic_kprobe *e, int index, int type, case const_buf_type: { // bound size to 1023 to help the verifier out size = argm & 0x03ff; - probe_read(args, size, (char *)arg); + probe_read_kernel_or_user(args, size, (char *)arg); break; } case bpf_attr_type: { @@ -2690,7 +2691,7 @@ read_call_arg(void *ctx, struct msg_generic_kprobe *e, int index, int type, case cap_inh_ty: case cap_prm_ty: case cap_eff_ty: - probe_read(args, sizeof(__u64), (char *)arg); + probe_read_kernel_or_user(args, sizeof(__u64), (char *)arg); size = sizeof(__u64); break; default: diff --git a/bpf/process/types/probe_read_kernel_or_user.h b/bpf/process/types/probe_read_kernel_or_user.h new file mode 100644 index 00000000000..2abdc6b9cbd --- /dev/null +++ b/bpf/process/types/probe_read_kernel_or_user.h @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright Authors of Cilium */ + +#ifndef __PROBE_READ_KERNEL_OR_USER_H__ +#define __PROBE_READ_KERNEL_OR_USER_H__ + +#ifdef GENERIC_UPROBE +#define probe_read_kernel_or_user probe_read_user +#define probe_read_kernel_or_user_str probe_read_user_str +#define BPF_CORE_READ_KERNEL_OR_USER_INTO BPF_CORE_READ_USER_INTO +#else +#define probe_read_kernel_or_user probe_read_kernel +#define probe_read_kernel_or_user_str probe_read_user_str +#define BPF_CORE_READ_KERNEL_OR_USER_INTO BPF_CORE_READ_KERNEL_INTO +#endif + +#endif diff --git a/bpf/process/types/skb.h b/bpf/process/types/skb.h index e26e6400973..6b2903afda3 100644 --- a/bpf/process/types/skb.h +++ b/bpf/process/types/skb.h @@ -5,6 +5,7 @@ #define __SKB_H__ #include "tuple.h" +#include "probe_read_kernel_or_user.h" struct skb_type { struct tuple_type tuple; @@ -70,7 +71,7 @@ get_ip6_protocol(u16 *payload_off, struct ipv6hdr *ip, u16 network_header_off, e->ip_off = network_header_off; e->curr = 255; e->len = 0; - if (probe_read(&e->next, sizeof(e->next), _(&ip->nexthdr)) < 0) + if (probe_read_kernel_or_user(&e->next, sizeof(e->next), _(&ip->nexthdr)) < 0) return 0; // Maximum 7 valid extensions. @@ -105,8 +106,8 @@ get_ip6_protocol(u16 *payload_off, struct ipv6hdr *ip, u16 network_header_off, } e->curr = e->next; // Read next header and current length. - if (probe_read(&e->next, 2, - skb_head + e->ip_off) < 0) { + if (probe_read_kernel_or_user(&e->next, 2, + skb_head + e->ip_off) < 0) { return 0; } } @@ -127,33 +128,33 @@ set_event_from_skb(struct skb_type *event, struct sk_buff *skb) typeof(skb->transport_header) l4_off; u8 protocol; - probe_read(&skb_head, sizeof(skb_head), _(&skb->head)); - probe_read(&l3_off, sizeof(l3_off), _(&skb->network_header)); + probe_read_kernel_or_user(&skb_head, sizeof(skb_head), _(&skb->head)); + probe_read_kernel_or_user(&l3_off, sizeof(l3_off), _(&skb->network_header)); struct iphdr *ip = (struct iphdr *)(skb_head + l3_off); u8 iphdr_byte0; - probe_read(&iphdr_byte0, 1, _(ip)); + probe_read_kernel_or_user(&iphdr_byte0, 1, _(ip)); u8 ip_ver = iphdr_byte0 >> 4; if (ip_ver == 4) { // IPv4 - probe_read(&protocol, 1, _(&ip->protocol)); + probe_read_kernel_or_user(&protocol, 1, _(&ip->protocol)); event->tuple.protocol = protocol; event->tuple.family = AF_INET; event->tuple.saddr[0] = 0; event->tuple.saddr[1] = 0; event->tuple.daddr[0] = 0; event->tuple.daddr[1] = 0; - probe_read(&event->tuple.saddr, IPV4LEN, _(&ip->saddr)); - probe_read(&event->tuple.daddr, IPV4LEN, _(&ip->daddr)); - probe_read(&l4_off, sizeof(l4_off), _(&skb->transport_header)); + probe_read_kernel_or_user(&event->tuple.saddr, IPV4LEN, _(&ip->saddr)); + probe_read_kernel_or_user(&event->tuple.daddr, IPV4LEN, _(&ip->daddr)); + probe_read_kernel_or_user(&l4_off, sizeof(l4_off), _(&skb->transport_header)); } else if (ip_ver == 6) { struct ipv6hdr *ip6 = (struct ipv6hdr *)(skb_head + l3_off); protocol = get_ip6_protocol(&l4_off, ip6, l3_off, skb_head); event->tuple.protocol = protocol; event->tuple.family = AF_INET6; - probe_read(&event->tuple.saddr, IPV6LEN, _(&ip6->saddr)); - probe_read(&event->tuple.daddr, IPV6LEN, _(&ip6->daddr)); + probe_read_kernel_or_user(&event->tuple.saddr, IPV6LEN, _(&ip6->saddr)); + probe_read_kernel_or_user(&event->tuple.daddr, IPV6LEN, _(&ip6->daddr)); } else { // This is not IP, so we don't know how to parse further. return -22; @@ -162,17 +163,17 @@ set_event_from_skb(struct skb_type *event, struct sk_buff *skb) if (protocol == IPPROTO_TCP) { // TCP struct tcphdr *tcp = (struct tcphdr *)(skb_head + l4_off); - probe_read(&event->tuple.sport, sizeof(event->tuple.sport), - _(&tcp->source)); - probe_read(&event->tuple.dport, sizeof(event->tuple.dport), - _(&tcp->dest)); + probe_read_kernel_or_user(&event->tuple.sport, sizeof(event->tuple.sport), + _(&tcp->source)); + probe_read_kernel_or_user(&event->tuple.dport, sizeof(event->tuple.dport), + _(&tcp->dest)); } else if (protocol == IPPROTO_UDP) { // UDP struct udphdr *udp = (struct udphdr *)(skb_head + l4_off); - probe_read(&event->tuple.sport, sizeof(event->tuple.sport), - _(&udp->source)); - probe_read(&event->tuple.dport, sizeof(event->tuple.dport), - _(&udp->dest)); + probe_read_kernel_or_user(&event->tuple.sport, sizeof(event->tuple.sport), + _(&udp->source)); + probe_read_kernel_or_user(&event->tuple.dport, sizeof(event->tuple.dport), + _(&udp->dest)); } else { event->tuple.sport = 0; event->tuple.dport = 0; @@ -186,18 +187,18 @@ set_event_from_skb(struct skb_type *event, struct sk_buff *skb) u64 offset; #define SKB_EXT_SEC_PATH 1 // TBD do this with BTF - probe_read(&ext, sizeof(ext), _(&skb->extensions)); + probe_read_kernel_or_user(&ext, sizeof(ext), _(&skb->extensions)); if (ext) { - probe_read(&offset, sizeof(offset), - _(&ext->offset[SKB_EXT_SEC_PATH])); + probe_read_kernel_or_user(&offset, sizeof(offset), + _(&ext->offset[SKB_EXT_SEC_PATH])); sp = (void *)ext + (offset << 3); - probe_read(&event->secpath_len, - sizeof(event->secpath_len), - _(&sp->len)); - probe_read(&event->secpath_olen, - sizeof(event->secpath_olen), - _(&sp->olen)); + probe_read_kernel_or_user(&event->secpath_len, + sizeof(event->secpath_len), + _(&sp->len)); + probe_read_kernel_or_user(&event->secpath_olen, + sizeof(event->secpath_olen), + _(&sp->olen)); } } return 0; diff --git a/bpf/process/types/sock.h b/bpf/process/types/sock.h index d11307189b0..d8057b4cf6d 100644 --- a/bpf/process/types/sock.h +++ b/bpf/process/types/sock.h @@ -5,6 +5,7 @@ #define __SOCK_H__ #include "tuple.h" +#include "probe_read_kernel_or_user.h" // The sockaddr field is specifically a __u64 to deter from trying to dereference it. // If an application needs more fields from the sock then they should be added to @@ -30,22 +31,22 @@ set_event_from_sock(struct sk_type *event, struct sock *sk) event->sockaddr = (__u64)sk; - probe_read(&event->tuple.family, sizeof(event->tuple.family), - _(&common->skc_family)); - probe_read(&event->state, sizeof(event->state), - _((const void *)&common->skc_state)); - probe_read(&event->type, sizeof(event->type), _(&sk->sk_type)); - probe_read(&event->tuple.protocol, sizeof(event->tuple.protocol), - _(&sk->sk_protocol)); + probe_read_kernel_or_user(&event->tuple.family, sizeof(event->tuple.family), + _(&common->skc_family)); + probe_read_kernel_or_user(&event->state, sizeof(event->state), + _((const void *)&common->skc_state)); + probe_read_kernel_or_user(&event->type, sizeof(event->type), _(&sk->sk_type)); + probe_read_kernel_or_user(&event->tuple.protocol, sizeof(event->tuple.protocol), + _(&sk->sk_protocol)); if (bpf_core_field_size(sk->sk_protocol) == 4) { // In the BTF, the protocol field in kernels tuple.protocol = event->tuple.protocol >> 8; } - probe_read(&event->mark, sizeof(event->mark), _(&sk->sk_mark)); - probe_read(&event->priority, sizeof(event->priority), - _(&sk->sk_priority)); + probe_read_kernel_or_user(&event->mark, sizeof(event->mark), _(&sk->sk_mark)); + probe_read_kernel_or_user(&event->priority, sizeof(event->priority), + _(&sk->sk_priority)); event->tuple.saddr[0] = 0; event->tuple.saddr[1] = 0; @@ -53,16 +54,16 @@ set_event_from_sock(struct sk_type *event, struct sock *sk) event->tuple.daddr[1] = 0; switch (event->tuple.family) { case AF_INET: - probe_read(&event->tuple.saddr, IPV4LEN, _(&common->skc_rcv_saddr)); - probe_read(&event->tuple.daddr, IPV4LEN, _(&common->skc_daddr)); + probe_read_kernel_or_user(&event->tuple.saddr, IPV4LEN, _(&common->skc_rcv_saddr)); + probe_read_kernel_or_user(&event->tuple.daddr, IPV4LEN, _(&common->skc_daddr)); break; case AF_INET6: - probe_read(&event->tuple.saddr, IPV6LEN, _(&common->skc_v6_rcv_saddr)); - probe_read(&event->tuple.daddr, IPV6LEN, _(&common->skc_v6_daddr)); + probe_read_kernel_or_user(&event->tuple.saddr, IPV6LEN, _(&common->skc_v6_rcv_saddr)); + probe_read_kernel_or_user(&event->tuple.daddr, IPV6LEN, _(&common->skc_v6_daddr)); } - probe_read(&event->tuple.sport, sizeof(event->tuple.sport), _(&common->skc_num)); - probe_read(&event->tuple.dport, sizeof(event->tuple.dport), _(&common->skc_dport)); + probe_read_kernel_or_user(&event->tuple.sport, sizeof(event->tuple.sport), _(&common->skc_num)); + probe_read_kernel_or_user(&event->tuple.dport, sizeof(event->tuple.dport), _(&common->skc_dport)); event->tuple.dport = bpf_ntohs(event->tuple.dport); } #endif // __SOCK_H__