How to store packet offset in BPF Map or skb->cb field? - ebpf

Question

Example Code
I want to pass the offset to the following tail calls. But when I try to store it in BPF MAP(Method 1) or skb->cb field(Method 2), I get an error offset is outside of the packet.
If Method 1/2 code is removed, the bpf program can be loaded sucessfully.
#include <vmlinux.h>
#include <bpf/bpf_endian.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__type(key, __u32);
__type(value, __u32);
__uint(max_entries, 100);
} state_vars SEC(".maps");
SEC("tc")
int tc_ingress(struct __sk_buff *ctx) {
void *data_end = (void *)(__u64)ctx->data_end;
void *data = (void *)(__u64)ctx->data;
u32 data_len = data_end - data;
u32 rn = 0;
u32 rn_idx = 0;
for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
char c = *(char*)(data + rn);
if (c == '\r') {
rn_idx = rn;
break;
}
}
/// Method 1
// u32 var_idx = 0;
// bpf_map_update_elem(&state_vars, &var_idx, &rn_idx, BPF_ANY);
/// Method 2
// ctx->cb[0] = rn_idx;
return TC_ACT_OK;
}
Error Message
Method 1
; void *data = (void *)(__u64)ctx->data;
0: (61) r2 = *(u32 *)(r1 +76)
; void *data_end = (void *)(__u64)ctx->data_end;
1: (61) r3 = *(u32 *)(r1 +80)
2: (b7) r1 = 0
; u32 rn_idx = 0;
3: (63) *(u32 *)(r10 -4) = r1
last_idx 3 first_idx 0
regs=2 stack=0 before 2: (b7) r1 = 0
; u32 data_len = data_end - data;
4: (bf) r4 = r3
5: (1f) r4 -= r2
6: (67) r4 <<= 32
7: (77) r4 >>= 32
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
8: (15) if r4 == 0x0 goto pc+10
R1_w=invP0 R2_w=pkt(id=0,off=0,r=0,imm=0) R3_w=pkt_end(id=0,off=0,imm=0) R4_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R10=fp0 fp-8=0000????
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
9: (bf) r5 = r2
10: (0f) r5 += r1
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
11: (3d) if r5 >= r3 goto pc+7
R1_w=invP0 R2_w=pkt(id=0,off=0,r=0,imm=0) R3_w=pkt_end(id=0,off=0,imm=0) R4_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R5_w=pkt(id=0,off=0,r=0,imm=0) R10=fp0 fp-8=0000????
; char c = *(char*)(data + rn);
12: (71) r5 = *(u8 *)(r5 +0)
invalid access to packet, off=0 size=1, R5(id=0,off=0,r=0)
R5 offset is outside of the packet
processed 13 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0
Method 2
; void *data = (void *)(__u64)ctx->data;
1: (61) r3 = *(u32 *)(r1 +76)
; void *data_end = (void *)(__u64)ctx->data_end;
2: (61) r4 = *(u32 *)(r1 +80)
; u32 data_len = data_end - data;
3: (bf) r5 = r4
4: (1f) r5 -= r3
5: (bf) r0 = r5
6: (67) r0 <<= 32
7: (77) r0 >>= 32
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
8: (15) if r0 == 0x0 goto pc+21
R0_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) R2_w=inv0 R3_w=pkt(id=0,off=0,r=0,imm=0) R4_w=pkt_end(id=0,off=0,imm=0) R5_w=inv(id=0) R10=fp0
9: (b7) r2 = 0
10: (b7) r0 = 0
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
11: (bf) r6 = r3
12: (0f) r6 += r0
last_idx 12 first_idx 0
regs=1 stack=0 before 11: (bf) r6 = r3
regs=1 stack=0 before 10: (b7) r0 = 0
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
13: (3d) if r6 >= r4 goto pc+16
R0_w=invP0 R1=ctx(id=0,off=0,imm=0) R2_w=inv0 R3_w=pkt(id=0,off=0,r=0,imm=0) R4_w=pkt_end(id=0,off=0,imm=0) R5_w=inv(id=0) R6_w=pkt(id=0,off=0,r=0,imm=0) R10=fp0
; char c = *(char*)(data + rn);
14: (71) r6 = *(u8 *)(r6 +0)
invalid access to packet, off=0 size=1, R6(id=0,off=0,r=0)
R6 offset is outside of the packet
processed 15 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0
Question
Why does the error appear when Method 1/2 code is added?
How could I store the offset value in BPF MAP or skb->cb field?
Update 2022.11.1
follow #pchaigno's advice, add +1 in the condition, I get similar error:
; int tc_ingress(struct __sk_buff *ctx)
0: (b7) r2 = 0
; void *data = (void *)(__u64)ctx->data;
1: (61) r3 = *(u32 *)(r1 +76)
; void *data_end = (void *)(__u64)ctx->data_end;
2: (61) r4 = *(u32 *)(r1 +80)
; u32 data_len = data_end - data;
3: (bf) r5 = r4
4: (1f) r5 -= r3
5: (bf) r0 = r5
6: (67) r0 <<= 32
7: (77) r0 >>= 32
; for (rn = 0; rn < 1000 && rn < data_len && data + rn + 1 < data_end; rn++) {
8: (15) if r0 == 0x0 goto pc+23
R0_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R1=ctx(id=0,off=0,imm=0) R2_w=inv0 R3_w=pkt(id=0,off=0,r=0,imm=0) R4_w=pkt_end(id=0,off=0,imm=0) R5_w=inv(id=0) R10=fp0
9: (b7) r2 = 0
10: (b7) r0 = 0
; for (rn = 0; rn < 1000 && rn < data_len && data + rn + 1 < data_end; rn++) {
11: (bf) r6 = r3
12: (0f) r6 += r0
last_idx 12 first_idx 0
regs=1 stack=0 before 11: (bf) r6 = r3
regs=1 stack=0 before 10: (b7) r0 = 0
13: (bf) r7 = r6
14: (07) r7 += 1
; for (rn = 0; rn < 1000 && rn < data_len && data + rn + 1 < data_end; rn++) {
15: (3d) if r7 >= r4 goto pc+16
R0_w=invP0 R1=ctx(id=0,off=0,imm=0) R2_w=inv0 R3_w=pkt(id=0,off=0,r=0,imm=0) R4_w=pkt_end(id=0,off=0,imm=0) R5_w=inv(id=0) R6_w=pkt(id=0,off=0,r=0,imm=0) R7_w=pkt(id=0,off=1,r=0,imm=0) R10=fp0
; char c = *(char*)(data + rn);
16: (71) r6 = *(u8 *)(r6 +0)
invalid access to packet, off=0 size=1, R6(id=0,off=0,r=0)
R6 offset is outside of the packet
processed 17 insns (limit 1000000) max_states_per_insn 0 total_states 0 peak_states 0 mark_read 0

Answer 1

TL;DR. The issue is when you read the packet data, not when you write it. You have an off-by-one issue on the bounds check. The issue only appears once you use the read data because otherwise the compiler optimizes out the code.
Verifier Error Explanation
; for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
11: (3d) if r5 >= r3 goto pc+7
R1_w=invP0 R2_w=pkt(id=0,off=0,r=0,imm=0) R3_w=pkt_end(id=0,off=0,imm=0) R4_w=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R5_w=pkt(id=0,off=0,r=0,imm=0) R10=fp0 fp-8=0000????
; char c = *(char*)(data + rn);
12: (71) r5 = *(u8 *)(r5 +0)
invalid access to packet, off=0 size=1, R5(id=0,off=0,r=0)
The verifier says that that the packet access is out of bounds by one: R5's offset + size > R3's offset. That is, both offsets are 0 and the access size is 1.
Root Cause
Your bounds check is off by one:
for (rn = 0; rn < 1000 && rn < data_len && data + rn < data_end; rn++) {
char c = *(char*)(data + rn);
To account for the access size, it should be:
for (rn = 0; rn < 1000 && rn < data_len && data + rn + 1 < data_end; rn++) {
char c = *(char*)(data + rn);
Why does it only happen once you write the packet data?
If you don't write rn_idx anywhere, then the compiler understands that rn_idx and c are not needed. All code for those variables is compiled out and the out-of-bound packet access is removed.