// Copyright (c) 2019 Dropbox, Inc. // Full license can be found in the LICENSE file. // BPF helpers // Set of defines / prototypes to use from eBPF programs as well as from regular // linux/mac "cross" compilation. #ifndef __BPF_HELPERS_H #define __BPF_HELPERS_H // Standard types. // Due to tooons of dependencies in standard linux kernel headers // Define types explicitly. typedef unsigned short __u16; // NOLINT typedef unsigned char __u8; typedef unsigned int __u32; typedef unsigned long long __u64; typedef int __s32; typedef unsigned long size_t; typedef __u32 __be32; typedef __u16 __be16; // BPF map types enum bpf_map_type { BPF_MAP_TYPE_UNSPEC = 0, BPF_MAP_TYPE_HASH, BPF_MAP_TYPE_ARRAY, BPF_MAP_TYPE_PROG_ARRAY, BPF_MAP_TYPE_PERF_EVENT_ARRAY, BPF_MAP_TYPE_PERCPU_HASH, BPF_MAP_TYPE_PERCPU_ARRAY, BPF_MAP_TYPE_STACK_TRACE, BPF_MAP_TYPE_CGROUP_ARRAY, BPF_MAP_TYPE_LRU_HASH, BPF_MAP_TYPE_LRU_PERCPU_HASH, BPF_MAP_TYPE_LPM_TRIE, BPF_MAP_TYPE_ARRAY_OF_MAPS, BPF_MAP_TYPE_HASH_OF_MAPS, BPF_MAP_TYPE_DEVMAP, BPF_MAP_TYPE_SOCKMAP, BPF_MAP_TYPE_CPUMAP, BPF_MAP_TYPE_XSKMAP, BPF_MAP_TYPE_SOCKHASH, BPF_MAP_TYPE_CGROUP_STORAGE, BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, BPF_MAP_TYPE_QUEUE, BPF_MAP_TYPE_STACK, BPF_MAP_TYPE_SK_STORAGE, }; /* BPF_FUNC_skb_store_bytes flags. */ enum { BPF_F_RECOMPUTE_CSUM = (1ULL << 0), BPF_F_INVALIDATE_HASH = (1ULL << 1), }; /* BPF_FUNC_l3_csum_replace and BPF_FUNC_l4_csum_replace flags. * First 4 bits are for passing the header field size. */ enum { BPF_F_HDR_FIELD_MASK = 0xfULL, }; /* BPF_FUNC_l4_csum_replace flags. */ enum { BPF_F_PSEUDO_HDR = (1ULL << 4), BPF_F_MARK_MANGLED_0 = (1ULL << 5), BPF_F_MARK_ENFORCE = (1ULL << 6), }; /* BPF_FUNC_clone_redirect and BPF_FUNC_redirect flags. */ enum { BPF_F_INGRESS = (1ULL << 0), }; /* BPF_FUNC_skb_set_tunnel_key flags. */ enum { BPF_F_ZERO_CSUM_TX = (1ULL << 1), BPF_F_DONT_FRAGMENT = (1ULL << 2), BPF_F_SEQ_NUMBER = (1ULL << 3), }; /* BPF_FUNC_skb_adjust_room flags. */ enum { BPF_F_ADJ_ROOM_FIXED_GSO = (1ULL << 0), BPF_F_ADJ_ROOM_ENCAP_L3_IPV4 = (1ULL << 1), BPF_F_ADJ_ROOM_ENCAP_L3_IPV6 = (1ULL << 2), BPF_F_ADJ_ROOM_ENCAP_L4_GRE = (1ULL << 3), BPF_F_ADJ_ROOM_ENCAP_L4_UDP = (1ULL << 4), BPF_F_ADJ_ROOM_NO_CSUM_RESET = (1ULL << 5), BPF_F_ADJ_ROOM_ENCAP_L2_ETH = (1ULL << 6), }; /* flags for BPF_MAP_UPDATE_ELEM command */ #define BPF_ANY 0 /* create new element or update existing */ #define BPF_NOEXIST 1 /* create new element if it didn't exist */ #define BPF_EXIST 2 /* update existing element */ #define BPF_F_LOCK 4 /* spin_lock-ed map_lookup/map_update */ /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and * BPF_FUNC_perf_event_read_value flags. */ #define BPF_F_INDEX_MASK 0xffffffffULL #define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK // A helper structure used by eBPF C program // to describe map attributes to BPF program loader struct bpf_map_def { __u32 map_type; __u32 key_size; __u32 value_size; __u32 max_entries; __u32 map_flags; // Array/Hash of maps use case: pointer to inner map template void *inner_map_def; // Define this to make map system wide ("object pinning") // path could be anything, like '/sys/fs/bpf/foo' // WARN: You must have BPF filesystem mounted on provided location const char *persistent_path; }; #define BPF_MAP_DEF_SIZE sizeof(struct bpf_map_def) #define BPF_MAP_OFFSET_PERSISTENT offsetof(struct bpf_map_def, persistent_path) #define BPF_MAP_OFFSET_INNER_MAP offsetof(struct bpf_map_def, inner_map_def) /* Generic BPF return codes which all BPF program types may support. * The values are binary compatible with their TC_ACT_* counter-part to * provide backwards compatibility with existing SCHED_CLS and SCHED_ACT * programs. * * XDP is handled seprately, see XDP_*. */ enum bpf_ret_code { BPF_OK = 0, /* 1 reserved */ BPF_DROP = 2, /* 3-6 reserved */ BPF_REDIRECT = 7, /* >127 are reserved for prog type specific return codes. * * BPF_LWT_REROUTE: used by BPF_PROG_TYPE_LWT_IN and * BPF_PROG_TYPE_LWT_XMIT to indicate that skb had been * changed and should be routed based on its new L3 header. * (This is an L3 redirect, as opposed to L2 redirect * represented by BPF_REDIRECT above). */ BPF_LWT_REROUTE = 128, }; // XDP related constants enum xdp_action { XDP_ABORTED = 0, XDP_DROP, XDP_PASS, XDP_TX, XDP_REDIRECT, }; // Socket Filter programs return code enum socket_filter_action { SOCKET_FILTER_DENY = 0, SOCKET_FILTER_ALLOW, }; // Kprobe required constants / structs // (arch/x86/include/asm/ptrace.h) #define PT_REGS_PARM1(x) ((x)->di) #define PT_REGS_PARM2(x) ((x)->si) #define PT_REGS_PARM3(x) ((x)->dx) #define PT_REGS_PARM4(x) ((x)->r10) #define PT_REGS_PARM5(x) ((x)->r8) #define PT_REGS_PARM6(x) ((x)->r9) #define PT_REGS_RET(x) ((x)->sp) #define PT_REGS_FP(x) ((x)->bp) #define PT_REGS_RC(x) ((x)->ax) #define PT_REGS_SP(x) ((x)->sp) #define PT_REGS_IP(x) ((x)->ip) struct pt_regs { unsigned long r15; unsigned long r14; unsigned long r13; unsigned long r12; unsigned long bp; unsigned long bx; unsigned long r11; unsigned long r10; unsigned long r9; unsigned long r8; unsigned long ax; unsigned long cx; unsigned long dx; unsigned long si; unsigned long di; unsigned long orig_ax; unsigned long ip; unsigned long cs; unsigned long flags; unsigned long sp; unsigned long ss; }; #define bpf_likely(X) __builtin_expect(!!(X), 1) #define bpf_unlikely(X) __builtin_expect(!!(X), 0) #define UNUSED __attribute__((unused)) // In order to cross compile BPF program for BPF / Linux / Mac // we need to define platform specific things like: // 1. Custom (non kernel) implementation for bpf_map_* functions // 2. For BPF we need to put programs into special sections, but, for // regular linux target (mostly for tests) we don't. // 3. BPF does not support function calls, so __always_inline__ is must have. // However, for testing it doesn't make sense. // 4. Debug prints - for BPF it is done by calling helper, for linux just // regular printf() #ifdef __BPF__ // Clang for eBPF missed static_assert declaration because programs are C, not // CPP #define static_assert _Static_assert // Helper macro to place programs, maps, license in // different sections in ELF file. #define SEC(NAME) __attribute__((section(NAME), used)) // eBPF does not support functions (yet), so, all functions MUST be inlined. // Starting from kernel 4.16 it is not required to always inline functions // since support has been added #define INLINE __attribute__((__always_inline__)) // XDP metadata - basically data packet // P.S. for some reason XDP programs uses 32bit pointers struct xdp_md { __u32 data; __u32 data_end; __u32 data_meta; /* Below access go through struct xdp_rxq_info */ __u32 ingress_ifindex; /* rxq->dev->ifindex */ __u32 rx_queue_index; /* rxq->queue_index */ __u32 egress_ifindex; /* txq->dev->ifindex */ }; /* user accessible mirror of in-kernel sk_buff. * new fields can only be added to the end of this structure */ struct __sk_buff { __u32 len; __u32 pkt_type; __u32 mark; __u32 queue_mapping; __u32 protocol; __u32 vlan_present; __u32 vlan_tci; __u32 vlan_proto; __u32 priority; __u32 ingress_ifindex; __u32 ifindex; __u32 tc_index; __u32 cb[5]; __u32 hash; __u32 tc_classid; __u32 data; __u32 data_end; __u32 napi_id; /* Accessed by BPF_PROG_TYPE_sk_skb types from here to ... */ __u32 family; __u32 remote_ip4; /* Stored in network byte order */ __u32 local_ip4; /* Stored in network byte order */ __u32 remote_ip6[4]; /* Stored in network byte order */ __u32 local_ip6[4]; /* Stored in network byte order */ __u32 remote_port; /* Stored in network byte order */ __u32 local_port; /* stored in host byte order */ /* ... here. */ __u32 data_meta; }; struct bpf_sock_tuple { union { struct { __be32 saddr; __be32 daddr; __be16 sport; __be16 dport; } ipv4; struct { __be32 saddr[4]; __be32 daddr[4]; __be16 sport; __be16 dport; } ipv6; }; }; struct bpf_spin_lock { __u32 val; }; struct bpf_sysctl { __u32 write; /* Sysctl is being read (= 0) or written (= 1). * Allows 1,2,4-byte read, but no write. */ __u32 file_pos; /* Sysctl file position to read from, write to. * Allows 1,2,4-byte read an 4-byte write. */ }; // BPF helper functions supported on linux kernel 5.2+ // clang-format off #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ FN(map_lookup_elem), \ FN(map_update_elem), \ FN(map_delete_elem), \ FN(probe_read), \ FN(ktime_get_ns), \ FN(trace_printk), \ FN(get_prandom_u32), \ FN(get_smp_processor_id), \ FN(skb_store_bytes), \ FN(l3_csum_replace), \ FN(l4_csum_replace), \ FN(tail_call), \ FN(clone_redirect), \ FN(get_current_pid_tgid), \ FN(get_current_uid_gid), \ FN(get_current_comm), \ FN(get_cgroup_classid), \ FN(skb_vlan_push), \ FN(skb_vlan_pop), \ FN(skb_get_tunnel_key), \ FN(skb_set_tunnel_key), \ FN(perf_event_read), \ FN(redirect), \ FN(get_route_realm), \ FN(perf_event_output), \ FN(skb_load_bytes), \ FN(get_stackid), \ FN(csum_diff), \ FN(skb_get_tunnel_opt), \ FN(skb_set_tunnel_opt), \ FN(skb_change_proto), \ FN(skb_change_type), \ FN(skb_under_cgroup), \ FN(get_hash_recalc), \ FN(get_current_task), \ FN(probe_write_user), \ FN(current_task_under_cgroup), \ FN(skb_change_tail), \ FN(skb_pull_data), \ FN(csum_update), \ FN(set_hash_invalid), \ FN(get_numa_node_id), \ FN(skb_change_head), \ FN(xdp_adjust_head), \ FN(probe_read_str), \ FN(get_socket_cookie), \ FN(get_socket_uid), \ FN(set_hash), \ FN(setsockopt), \ FN(skb_adjust_room), \ FN(redirect_map), \ FN(sk_redirect_map), \ FN(sock_map_update), \ FN(xdp_adjust_meta), \ FN(perf_event_read_value), \ FN(perf_prog_read_value), \ FN(getsockopt), \ FN(override_return), \ FN(sock_ops_cb_flags_set), \ FN(msg_redirect_map), \ FN(msg_apply_bytes), \ FN(msg_cork_bytes), \ FN(msg_pull_data), \ FN(bind), \ FN(xdp_adjust_tail), \ FN(skb_get_xfrm_state), \ FN(get_stack), \ FN(skb_load_bytes_relative), \ FN(fib_lookup), \ FN(sock_hash_update), \ FN(msg_redirect_hash), \ FN(sk_redirect_hash), \ FN(lwt_push_encap), \ FN(lwt_seg6_store_bytes), \ FN(lwt_seg6_adjust_srh), \ FN(lwt_seg6_action), \ FN(rc_repeat), \ FN(rc_keydown), \ FN(skb_cgroup_id), \ FN(get_current_cgroup_id), \ FN(get_local_storage), \ FN(sk_select_reuseport), \ FN(skb_ancestor_cgroup_id), \ FN(sk_lookup_tcp), \ FN(sk_lookup_udp), \ FN(sk_release), \ FN(map_push_elem), \ FN(map_pop_elem), \ FN(map_peek_elem), \ FN(msg_push_data), \ FN(msg_pop_data), \ FN(rc_pointer_rel), \ FN(spin_lock), \ FN(spin_unlock), \ FN(sk_fullsock), \ FN(tcp_sock), \ FN(skb_ecn_set_ce), \ FN(get_listener_sock), \ FN(skc_lookup_tcp), \ FN(tcp_check_syncookie), \ FN(sysctl_get_name), \ FN(sysctl_get_current_value), \ FN(sysctl_get_new_value), \ FN(sysctl_set_new_value), \ FN(strtol), \ FN(strtoul), \ FN(sk_storage_get), \ FN(sk_storage_delete), \ FN(send_signal), #define __BPF_ENUM_FN(x) BPF_FUNC_ ## x enum bpf_func_id { __BPF_FUNC_MAPPER(__BPF_ENUM_FN) __BPF_FUNC_MAX_ID, }; #undef __BPF_ENUM_FN // clang-format on // BPF helper functions - this construction looks complicated, but actually // it explained to just: // static void* bpf_map_lookup_elem(void *map, void *key) = 1 // In other words bpf_map_lookup_elem points to memory address 0x1 - which is // BPF function number 1. // More details about helper functions at: http://docs.cilium.io/en/v1.1/bpf/ // Search for "Helper Functions" // clang-format off // Lookup bpf map element by key. // Return: Map value or NULL static void *(*bpf_map_lookup_elem)(const void *map, const void *key) = (void *) // NOLINT BPF_FUNC_map_lookup_elem; // Update bpf map element by key to value // Return: 0 on success or negative error static int (*bpf_map_update_elem)(const void *map, const void *key, const void *value, __u64 flags) = (void *) // NOLINT BPF_FUNC_map_update_elem; // Delete element. Actually applicable on HASH maps // Return: 0 on success or negative error static int (*bpf_map_delete_elem)(const void *map, void *key) = (void *) // NOLINT BPF_FUNC_map_delete_elem; static int (*bpf_probe_read)(void *dst, __u64 size, const void *unsafe_ptr) = (void *) // NOLINT BPF_FUNC_probe_read; static __u64 (*bpf_ktime_get_ns)(void) = (void *) // NOLINT BPF_FUNC_ktime_get_ns; static __u32 (*bpf_get_prandom_u32)(void) = (void *) // NOLINT BPF_FUNC_get_prandom_u32; // Like printf() for BPF // Return: length of buffer written or negative error static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = (void *) // NOLINT BPF_FUNC_trace_printk; static int (*bpf_probe_read_str)(void *dst, __u64 size, const void *unsafe_ptr) = (void *) // NOLINT BPF_FUNC_probe_read_str; // Jump into another BPF program // prog_array_map: pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY // index: 32-bit index inside array that selects specific program to run // Return: 0 on success or negative error static void (*bpf_tail_call)(const void *ctx, void *map, int index) = (void *) // NOLINT BPF_FUNC_tail_call; static int (*bpf_clone_redirect)(void *ctx, int ifindex, __u32 flags) = (void*) // NOLINT BPF_FUNC_clone_redirect; static __u64 (*bpf_get_smp_processor_id)(void) = (void*) // NOLINT BPF_FUNC_get_smp_processor_id; static __u64 (*bpf_get_current_pid_tgid)(void) = (void*) // NOLINT BPF_FUNC_get_current_pid_tgid; static __u64 (*bpf_get_current_uid_gid)(void) = (void*) // NOLINT BPF_FUNC_get_current_uid_gid; static int (*bpf_get_current_comm)(void *buf, int buf_size) = (void*) // NOLINT BPF_FUNC_get_current_comm; static __u64 (*bpf_get_cgroup_classid)(void *ctx) = (void*) // NOLINT BPF_FUNC_get_cgroup_classid; static __u64 (*bpf_skb_vlan_push)(void *ctx, __u16 proto, __u16 vlan_tci) = (void*) // NOLINT BPF_FUNC_skb_vlan_push; static __u64 (*bpf_skb_vlan_pop)(void *ctx) = (void*) // NOLINT BPF_FUNC_skb_vlan_pop; static int (*bpf_skb_get_tunnel_key)(void *ctx, void *to, __u32 size, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_get_tunnel_key; static int (*bpf_skb_set_tunnel_key)(void *ctx, void *from, __u32 size, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_set_tunnel_key; static __u64 (*bpf_perf_event_read)(void *map, __u64 flags) = (void*) // NOLINT BPF_FUNC_perf_event_read; static int (*bpf_redirect)(int ifindex, __u32 flags) = (void*) // NOLINT BPF_FUNC_redirect; static __u32 (*bpf_get_route_realm)(void *ctx) = (void*) // NOLINT BPF_FUNC_get_route_realm; static int (*bpf_perf_event_output)(void *ctx, void *map, __u64 index, void *data, __u32 size) = (void*) // NOLINT BPF_FUNC_perf_event_output; static int (*bpf_l3_csum_replace)(void *ctx, int offset, __u64 from, __u64 to, __u64 size) = (void *) // NOLINT BPF_FUNC_l3_csum_replace; static int (*bpf_l4_csum_replace)(void *ctx, int offset, __u64 from, __u64 to, __u64 flags) = (void *) // NOLINT BPF_FUNC_l4_csum_replace; static int (*bpf_skb_load_bytes)(void *ctx, int offset, void *to, __u32 len) = (void*) // NOLINT BPF_FUNC_skb_load_bytes; static int (*bpf_skb_store_bytes)(void *ctx, int offset, const void *from, __u32 len, __u64 flags) = (void *) // NOLINT BPF_FUNC_skb_store_bytes; static int (*bpf_perf_event_read_value)(void *map, __u64 flags, void *buf, __u32 buf_size) = (void*) // NOLINT BPF_FUNC_perf_event_read_value; static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, __u32 buf_size) = (void*) // NOLINT BPF_FUNC_perf_prog_read_value; static int (*bpf_current_task_under_cgroup)(void *map, int index) = (void*) // NOLINT BPF_FUNC_current_task_under_cgroup; static __u32 (*bpf_get_socket_cookie)(void *ctx) = (void*) // NOLINT BPF_FUNC_get_socket_cookie; static __u64 (*bpf_get_socket_uid)(void *ctx) = (void*) // NOLINT BPF_FUNC_get_socket_uid; static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, int optlen) = (void*) // NOLINT BPF_FUNC_getsockopt; static int (*bpf_redirect_map)(void *map, __u32 key, __u64 flags) = (void*) // NOLINT BPF_FUNC_redirect_map; static int (*bpf_set_hash)(void *ctx, __u32 hash) = (void*) // NOLINT BPF_FUNC_set_hash; static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, int optlen) = (void*) // NOLINT BPF_FUNC_setsockopt; static int (*bpf_skb_adjust_room)(void *ctx, int len_diff, __u32 mode, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_adjust_room; static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = (void*) // NOLINT BPF_FUNC_skb_under_cgroup; static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, int size, unsigned long long netns_id, unsigned long long flags) = (void*) // NOLINT BPF_FUNC_skc_lookup_tcp; static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = (void*) // NOLINT BPF_FUNC_sk_redirect_map; static int (*bpf_sock_map_update)(void *map, void *key, void *value, unsigned long long flags) = (void*) // NOLINT BPF_FUNC_sock_map_update; static int (*bpf_strtol)(const char *buf, size_t buf_len, __u64 flags, long *res) = (void*) // NOLINT BPF_FUNC_strtol; static int (*bpf_strtoul)(const char *buf, size_t buf_len, __u64 flags, unsigned long *res) = (void*) // NOLINT BPF_FUNC_strtoul; static int (*bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, size_t buf_len) = (void*) // NOLINT BPF_FUNC_sysctl_get_current_value; static int (*bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, size_t buf_len, __u64 flags) = (void*) // NOLINT BPF_FUNC_sysctl_get_name; static int (*bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, size_t buf_len) = (void*) // NOLINT BPF_FUNC_sysctl_get_new_value; static int (*bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, size_t buf_len) = (void*) // NOLINT BPF_FUNC_sysctl_set_new_value; static int (*bpf_tcp_check_syncookie)(struct bpf_sock *sk, void *ip, int ip_len, void *tcp, int tcp_len) = (void*) // NOLINT BPF_FUNC_tcp_check_syncookie; // Adjust the xdp_md.data_meta by delta // ctx: pointer to xdp_md // delta: An positive/negative integer to be added to ctx.data_meta // Return: 0 on success or negative on error static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = (void*) // NOLINT BPF_FUNC_xdp_adjust_meta; static int (*bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void*) // NOLINT BPF_FUNC_get_stackid; static int (*bpf_csum_diff)(void *from, __u64 from_size, void *to, __u64 to_size, __u64 seed) = (void*) // NOLINT BPF_FUNC_csum_diff; static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, __u32 size) = (void*) // NOLINT BPF_FUNC_skb_get_tunnel_opt; static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, __u32 size) = (void*) // NOLINT BPF_FUNC_skb_set_tunnel_opt; static int (*bpf_skb_change_proto)(void *ctx, __u16 proto, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_change_proto; static int (*bpf_skb_change_type)(void *ctx, __u32 type) = (void*) // NOLINT BPF_FUNC_skb_change_type; static __u32 (*bpf_get_hash_recalc)(void *ctx) = (void*) // NOLINT BPF_FUNC_get_hash_recalc; static __u64 (*bpf_get_current_task)(void) = (void*) // NOLINT BPF_FUNC_get_current_task; static int (*bpf_probe_write_user)(void *dst, void *src, __u32 size) = (void*) // NOLINT BPF_FUNC_probe_write_user; static int (*bpf_skb_change_tail)(void *ctx, __u32 new_len, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_change_tail; static int (*bpf_skb_pull_data)(void *ctx, __u32 len) = (void*) // NOLINT BPF_FUNC_skb_pull_data; static int (*bpf_csum_update)(void *ctx, __u16 csum) = (void*) // NOLINT BPF_FUNC_csum_update; static int (*bpf_set_hash_invalid)(void *ctx) = (void*) // NOLINT BPF_FUNC_set_hash_invalid; static int (*bpf_get_numa_node_id)(void) = (void*) // NOLINT BPF_FUNC_get_numa_node_id; static int (*bpf_skb_change_head)(void *ctx, __u32 len, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_change_head; static int (*bpf_override_return)(void *pt_regs, unsigned long rc) = (void*) // NOLINT BPF_FUNC_override_return; static int (*bpf_sock_ops_cb_flags_set)(void *skops, int flags) = (void*) // NOLINT BPF_FUNC_sock_ops_cb_flags_set; static int (*bpf_msg_redirect_map)(void *msg, void *map, __u32 key, __u64 flags) = (void*) // NOLINT BPF_FUNC_msg_redirect_map; static int (*bpf_msg_apply_bytes)(void *msg, __u32 bytes) = (void*) // NOLINT BPF_FUNC_msg_apply_bytes; static int (*bpf_msg_cork_bytes)(void *msg, __u32 bytes) = (void*) // NOLINT BPF_FUNC_msg_cork_bytes; static int (*bpf_msg_pull_data)(void *msg, __u32 start, __u32 end, __u64 flags) = (void*) // NOLINT BPF_FUNC_msg_pull_data; static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = (void*) // NOLINT BPF_FUNC_bind; static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = (void*) // NOLINT BPF_FUNC_xdp_adjust_tail; static int (*bpf_skb_get_xfrm_state)(void *ctx, __u32 index, void *xfrm_state, __u32 size, __u64 flags) = (void*) // NOLINT BPF_FUNC_skb_get_xfrm_state; static int (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void*) // NOLINT BPF_FUNC_get_stack; static int (*bpf_skb_load_bytes_relative)(void *ctx, __u32 offset, void *to, __u32 len, __u32 start_header) = (void*) // NOLINT BPF_FUNC_skb_load_bytes_relative; static int (*bpf_fib_lookup)(void *ctx, void *params, int plen, __u32 flags) = (void*) // NOLINT BPF_FUNC_fib_lookup; static int (*bpf_sock_hash_update)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT BPF_FUNC_sock_hash_update; static int (*bpf_msg_redirect_hash)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT BPF_FUNC_msg_redirect_hash; static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, __u64 flags) = (void*) // NOLINT BPF_FUNC_sk_redirect_hash; static int (*bpf_lwt_push_encap)(void *skb, __u32 type, void *hdr, __u32 len) = (void*) // NOLINT BPF_FUNC_lwt_push_encap; static int (*bpf_lwt_seg6_store_bytes)(void *ctx, __u32 offset, const void *from, __u32 len) = (void*) // NOLINT BPF_FUNC_lwt_seg6_store_bytes; static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, __u32 offset, __s32 delta) = (void*) // NOLINT BPF_FUNC_lwt_seg6_adjust_srh; static int (*bpf_lwt_seg6_action)(void *ctx, __u32 action, void *param, __u32 param_len) = (void*) // NOLINT BPF_FUNC_lwt_seg6_action; static int (*bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void*) // NOLINT BPF_FUNC_rc_keydown; static int (*bpf_rc_repeat)(void *ctx) = (void*) // NOLINT BPF_FUNC_rc_repeat; static __u64 (*bpf_skb_cgroup_id)(void *skb) = (void*) // NOLINT BPF_FUNC_skb_cgroup_id; static __u64 (*bpf_get_current_cgroup_id)(void) = (void*) // NOLINT BPF_FUNC_get_current_cgroup_id; static __u64 (*bpf_skb_ancestor_cgroup_id)(void *skb, int ancestor_level) = (void*) // NOLINT BPF_FUNC_skb_ancestor_cgroup_id; static void * (*bpf_get_local_storage)(void *map, __u64 flags) = (void*) // NOLINT BPF_FUNC_get_local_storage; static int (*bpf_sk_select_reuseport)(void *reuse, void *map, void *key, __u64 flags) = (void*) // NOLINT BPF_FUNC_sk_select_reuseport; static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, int size, unsigned int netns_id, unsigned long long flags) = (void*) // NOLINT BPF_FUNC_sk_lookup_tcp; static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *tuple, int size, unsigned int netns_id, unsigned long long flags) = (void*) // NOLINT BPF_FUNC_sk_lookup_udp; static int (*bpf_sk_release)(struct bpf_sock *sk) = (void*) // NOLINT BPF_FUNC_sk_release; static int (*bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void*) // NOLINT BPF_FUNC_map_push_elem; static int (*bpf_map_pop_elem)(void *map, void *value) = (void*) // NOLINT BPF_FUNC_map_pop_elem; static int (*bpf_map_peek_elem)(void *map, void *value) = (void*) // NOLINT BPF_FUNC_map_peek_elem; static int (*bpf_msg_push_data)(void *skb, __u32 start, __u32 len, __u64 flags) = (void*) // NOLINT BPF_FUNC_msg_push_data; static int (*bpf_msg_pop_data)(void *msg, __u32 start, __u32 pop, __u64 flags) = (void*) // NOLINT BPF_FUNC_msg_pop_data; static int (*bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void*) // NOLINT BPF_FUNC_rc_pointer_rel; static void (*bpf_spin_lock)(struct bpf_spin_lock *lock) = (void*) // NOLINT BPF_FUNC_spin_lock; static void (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void*) // NOLINT BPF_FUNC_spin_unlock; static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = (void*) // NOLINT BPF_FUNC_sk_fullsock; static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = (void*) // NOLINT BPF_FUNC_tcp_sock; static int (*bpf_skb_ecn_set_ce)(void *ctx) = (void*) // NOLINT BPF_FUNC_skb_ecn_set_ce; static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = (void*) // NOLINT BPF_FUNC_get_listener_sock; static void *(*bpf_sk_storage_get)(void *map, struct bpf_sock *sk, void *value, __u64 flags) = (void*) // NOLINT BPF_FUNC_sk_storage_get; static int (*bpf_sk_storage_delete)(void *map, struct bpf_sock *sk) = (void*) // NOLINT BPF_FUNC_sk_storage_delete; static int (*bpf_send_signal)(unsigned sig) = (void *) // NOLINT BPF_FUNC_send_signal; // Adjust the xdp_md.data by delta // ctx: pointer to xdp_md // delta: An positive/negative integer to be added to ctx.data // Return: 0 on success or negative on error static int (*bpf_xdp_adjust_head)(const void *ctx, int delta) = (void *) // NOLINT BPF_FUNC_xdp_adjust_head; // clang-format on // printk() - kernel trace mechanism, like printf() // To get trace (debug) messages: // - Add #define DEBUG into your eBPF program before includes // - $ sudo cat /sys/kernel/debug/tracing/trace #ifdef DEBUG #define bpf_printk(fmt, ...) \ ({ \ char ____fmt[] = fmt; \ bpf_trace_printk(____fmt, sizeof(____fmt), ##__VA_ARGS__); \ }) #else #define bpf_printk(fmt, ...) #endif // Since BPF programs cannot perform any function calls other than // those to BPF helpers, common library code needs to be implemented // as inline functions. In addition, also LLVM provides some built-ins // that can be used for constant sizes. #define memset(dest, chr, n) __builtin_memset((dest), (chr), (n)) #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n)) #define memmove(dest, src, n) __builtin_memmove((dest), (src), (n)) // Do not allow use printf() #define printf(fmt, ...) do_not_use_printf_use_bpf_printk // Macro to define BPF Map #define BPF_MAP_DEF(name) struct bpf_map_def SEC("maps") name #define BPF_MAP_ADD(x) /* https://elixir.bootlin.com/linux/latest/source/include/uapi/linux/bpf.h#L4283 */ /* DIRECT: Skip the FIB rules and go to FIB table associated with device * OUTPUT: Do lookup from egress perspective; default is ingress */ enum { BPF_FIB_LOOKUP_DIRECT = (1U << 0), BPF_FIB_LOOKUP_OUTPUT = (1U << 1), }; enum { BPF_FIB_LKUP_RET_SUCCESS, /* lookup successful */ BPF_FIB_LKUP_RET_BLACKHOLE, /* dest is blackholed; can be dropped */ BPF_FIB_LKUP_RET_UNREACHABLE, /* dest is unreachable; can be dropped */ BPF_FIB_LKUP_RET_PROHIBIT, /* dest not allowed; can be dropped */ BPF_FIB_LKUP_RET_NOT_FWDED, /* packet is not forwarded */ BPF_FIB_LKUP_RET_FWD_DISABLED, /* fwding is not enabled on ingress */ BPF_FIB_LKUP_RET_UNSUPP_LWT, /* fwd requires encapsulation */ BPF_FIB_LKUP_RET_NO_NEIGH, /* no neighbor entry for nh */ BPF_FIB_LKUP_RET_FRAG_NEEDED, /* fragmentation required to fwd */ }; struct bpf_fib_lookup { /* input: network family for lookup (AF_INET, AF_INET6) * output: network family of egress nexthop */ __u8 family; /* set if lookup is to consider L4 data - e.g., FIB rules */ __u8 l4_protocol; __be16 sport; __be16 dport; /* total length of packet from network header - used for MTU check */ __u16 tot_len; /* input: L3 device index for lookup * output: device index from FIB lookup */ __u32 ifindex; union { /* inputs to lookup */ __u8 tos; /* AF_INET */ __be32 flowinfo; /* AF_INET6, flow_label + priority */ /* output: metric of fib result (IPv4/IPv6 only) */ __u32 rt_metric; }; union { __be32 ipv4_src; __u32 ipv6_src[4]; /* in6_addr; network order */ }; /* input to bpf_fib_lookup, ipv{4,6}_dst is destination address in * network header. output: bpf_fib_lookup sets to gateway address * if FIB lookup returns gateway route */ union { __be32 ipv4_dst; __u32 ipv6_dst[4]; /* in6_addr; network order */ }; /* output */ __be16 h_vlan_proto; __be16 h_vlan_TCI; __u8 smac[6]; /* ETH_ALEN */ __u8 dmac[6]; /* ETH_ALEN */ }; // offsetof gets the offset of a struct member #ifndef offsetof #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) #endif ///// end of __BPF__ ///// #else //// All other platforms //// // SEC() is useless for non eBPF - so just dummy #define SEC(NAME) // Functions must be inlined only for eBPF, so don't enforce it for *nix/mac. // Also disable "unused function" warning - // since eBPF programs define functions mostly in headers. #define INLINE static __attribute__((unused)) // Disable warnings for "pragma unroll(all)" #pragma GCC diagnostic ignored "-Wunknown-pragmas" #include #include #include #include // XDP metadata - defined twice because of real eBPF uses 32 bit pointers // which are not acceptable for cross platform compilation. struct xdp_md { void *data; void *data_end; void *data_meta; /* Below access go through struct xdp_rxq_info */ __u32 ingress_ifindex; /* rxq->dev->ifindex */ __u32 rx_queue_index; /* rxq->queue_index */ __u32 egress_ifindex; /* txq->dev->ifindex */ }; // Mock BPF map support: // In order to automatically find all defined BPF maps from GO program we need // to // maintain linked list of maps (to be able to iterate and create them all) // This could be easily and nicely done using __attribute__ ((constructor)) // Which is logically close to func init() int GO. struct __create_map_def { const char *name; void *map_data; // Mock version only: holds head to single linked list of map // items struct bpf_map_def *map_def; SLIST_ENTRY(__create_map_def) next; }; // Declaration only. Definition held in mock_map package. SLIST_HEAD(__maps_head_def, __create_map_def); extern struct __maps_head_def *__maps_head; #define BPF_MAP_DEF(x) static struct bpf_map_def x #define BPF_MAP_ADD(x) \ static __attribute__((constructor)) void __bpf_map_##x() { \ static struct __create_map_def __bpf_map_entry_##x; \ __bpf_map_entry_##x.name = #x; \ __bpf_map_entry_##x.map_data = NULL; \ __bpf_map_entry_##x.map_def = &x; \ SLIST_INSERT_HEAD(__maps_head, &__bpf_map_entry_##x, next); \ } // BPF helper prototypes - definition is up to mac/linux host program void *bpf_map_lookup_elem(const void *map, const void *key); int bpf_map_update_elem(const void *map, const void *key, const void *value, __u64 flags); int bpf_map_delete_elem(const void *map, const void *key); // bpf_printk() is just printf() #define bpf_printk(fmt, ...) \ printf(fmt, ##__VA_ARGS__); \ fflush(stdout); // bpf_tail_call() is nothing: only relevant for BPF arch #define bpf_tail_call(ctx, map, index) // adjust_meta / ajdust_header are simple functions to move pointer UNUSED static int bpf_xdp_adjust_meta(struct xdp_md *ctx, int offset) { // For unittests only - function returns error if data_meta points to data_end // which never the case in real world if (ctx->data_meta == ctx->data_end) { return 1; } ctx->data_meta = (__u8 *)ctx->data_meta + offset; // NOLINT return 0; } UNUSED static int bpf_xdp_adjust_head(struct xdp_md *ctx, int offset) { ctx->data = (__u8 *)ctx->data + offset; // NOLINT return 0; } UNUSED static int bpf_perf_event_output(void *ctx, void *map, __u64 index, void *data, __u32 size) { return 0; } #endif // of other than __BPF__ // Finally make sure that all types have expected size regardless of platform static_assert(sizeof(__u8) == 1, "wrong_u8_size"); static_assert(sizeof(__u16) == 2, "wrong_u16_size"); static_assert(sizeof(__u32) == 4, "wrong_u32_size"); static_assert(sizeof(__u64) == 8, "wrong_u64_size"); #endif