8000 fix xdp-acl demo · lx1036/code@20b17dc · GitHub
[go: up one dir, main page]
Skip to content

Commit 20b17dc

Browse files
author
shenming
committed
fix xdp-acl demo
1 parent 93876f4 commit 20b17dc

File tree

3 files changed

+169
-27
lines changed

3 files changed

+169
-27
lines changed

‎go/k8s/bpf/xdp-l4lb/xdp-cilium-l4lb/cilium/test/xdp-acl/acl.c

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -103,18 +103,18 @@ static __always_inline int xdp_acl_ipv4_port(struct xdp_md *ctx) {
103103

104104
struct iphdr *ipv4h = (data + sizeof(struct ethhdr));
105105
if ((void *) (ipv4h + 1) > data_end) {
106-
// bpf_printk("fail to lookup from servers map. (void *) (ipv4h + 1) > data_end"); // 这里不能打印日志
106+
bpf_printk("fail to lookup from servers map. (void *) (ipv4h + 1) > data_end"); // 这里不能打印日志
107107
return XDP_DROP;
108108
}
109109
if (ipv4h->ihl != 5) { // 必然是 5，感觉这种检查意义不大
110-
// bpf_printk("fail to lookup from servers map. ipv4h->ihl != 5");
110+
bpf_printk("fail to lookup from servers map. ipv4h->ihl != 5");
111111
return XDP_PASS;
112112
}
113113

114114
// 这个逻辑是为了调试，因为 ecs eth0 网卡一直都有流量，这里限定另一台 ecs saddr 发 tcp 包
115-
if (ipv4h->saddr != bpf_htonl(0xac100a02)) { /* 172.16.10.2 */
116-
return XDP_PASS;
117-
}
115+
// if (ipv4h->saddr != bpf_htonl(0xac100a02)) { /* 172.16.10.2 */
116+
// return XDP_PASS;
117+
// }
118118

119119
// 1.因为 eth0 可能绑定多个 ip 地址，dstIP 必须是指定的 ip，必须做检查过滤
120120
__u32 key = 0;
@@ -133,48 +133,55 @@ static __always_inline int xdp_acl_ipv4_port(struct xdp_md *ctx) {
133133
break; // break 起作用的
134134
}
135135

136-
bpf_printk("target_ip not found %d", i);
136+
// bpf_printk("target_ip not found %d", i);
137137
}
138138

139139
// bpf_printk("target_ip 0x%x", server->target_ips[0]);
140140

141141
if (!found) {
142-
bpf_printk("dst ip: 0x%x is not target ip, skip it. target_ip 0x%x", ipv4h->daddr,
143-
server->target_ips[0]); // 使用 u32toIP() 报错
142+
// 使用 u32toIP() 报错
143+
// bpf_printk("dst ip: 0x%x is not target ip, skip it. target_ip 0x%x", ipv4h->daddr, server->target_ips[0]);
144144
return XDP_PASS;
145145
}
146146

147-
bpf_debug_printk("dst ip: 0x%x is a target ip, acl it.", ipv4h->daddr); // XDPACL_DEBUG 参数起作用的, %pI4 不行
147+
// bpf_debug_printk("dst ip: 0x%x is a target ip, acl it.", ipv4h->daddr); // XDPACL_DEBUG 参数起作用的, %pI4 不行
148148

149149
if (ipv4h->protocol != IPPROTO_TCP && ipv4h->protocol != IPPROTO_UDP) {
150150
bpf_printk("protocol: %x is not tcp or udp, skip it.", ipv4h->protocol);
151151
return XDP_PASS;
152152
}
153153

154+
// 这样可以不用区分 tcphdr 和 udphdr
154155
struct ports *port;
155156
port = data + sizeof(struct ethhdr) + sizeof(struct iphdr);
156157
if ((void *) (port + 1) > data_end) {
157158
bpf_printk("fail to fetch tcp/udp ports, skip it.");
158159
return XDP_PASS;
159160
}
160161

162+
// 这个逻辑是为了调试，因为 lo 网卡一直都有 tcp 包，把 ->9090 包过滤出来
163+
if (bpf_ntohs(port->dest) != 9090) {
164+
return XDP_PASS;
165+
}
166+
161167
// unsigned short, __u16 -> unsigned int, __u32
162-
// __u32 dst_port = bpf_ntohs(port->dest); // network to host shorts
163168
struct endpoint endpoint = {}; // 对象初始化
164-
endpoint.dport = port->dest;
169+
// network to host shorts, 不加 bpf_ntohs() dport 为 0x8223, 而是加上 hex(9090)=0x2382, 需要注意!!!
170+
endpoint.dport = bpf_ntohs(port->dest);
165171
endpoint.protocol = ipv4h->protocol;
166172
// bpftool map dump name endpoints | jq
167173
struct action *action;
168174
action = bpf_map_lookup_elem(&endpoints, &endpoint); // 这里报错一直查找不到对应的 endpoint???
169175
if (!action) {
176+
bpf_printk("dport: %x, protocol: %x", endpoint.dport, endpoint.protocol);
170177
// fail to lookup endpoints map, dport: 9090, protocol: 6, action:0
171178
bpf_printk("fail to lookup endpoints map, dport: %d, protocol: %x, action:%x", bpf_ntohs(port->dest),
172179
ipv4h->protocol, action);
173180
return XDP_PASS;
174181
}
175182

176183
if (action->action == 0) { // deny
177-
bpf_printk("action of protocol:%x port:%d is deny, drop it.", ipv4h->protocol, port->dest);
184+
bpf_printk("action of protocol:%x port:%d is deny, drop it.", ipv4h->protocol, bpf_ntohs(port->dest));
178185
return XDP_DROP;
179186
}
180187

@@ -186,6 +193,9 @@ static __always_inline int xdp_acl_ipv4_port(struct xdp_md *ctx) {
186193
* 根据 protocol/svcPort 来判断 action(XDP_PASS/XDP_DROP)
187194
*
188195
* 但是没有验证通过!!! 这里报错一直查找不到对应的 endpoint???
196+
* 答案已经找到：bpf_ntohs(port->dest) 需要加上 bpf_ntohs(), 验证通过!!!
197+
*
198+
* TODO: 根据 bitmap 来寻找 action rule, 减少内存
189199
*/
190200

191201
SEC("xdp_acl")

‎go/k8s/bpf/xdp-l4lb/xdp-cilium-l4lb/cilium/test/xdp-acl/bpf.sh

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
#!/bin/bash
22

3-
4-
apt-get update -y
5-
apt-get install -y gcc-multilib libbpf-dev clang linux-tools-`uname -r` jq
6-
7-
# 这里安装 libbpf-dev 包后，代码里可以直接 include linux 头文件
8-
clang -O2 -Wall -target bpf -c acl.c -o acl.o
9-
3+
# 本地运行时，不会走 eth0-acl 网卡，而是 lo 网卡 127.0.0.1.9091 > 127.0.0.1.9090
104

115
ip link add dev eth0-acl type dummy
126
ip link set dev eth0-acl up

‎go/k8s/bpf/xdp-l4lb/xdp-cilium-l4lb/cilium/test/xdp-acl/main.go

Lines changed: 146 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"encoding/binary"
5+
"flag"
56
"github.com/cilium/ebpf"
67
"github.com/cilium/ebpf/btf"
78
"github.com/cilium/ebpf/link"
@@ -16,14 +17,47 @@ import (
1617

1718
//go:generate go run github.com/cilium/ebpf/cmd/bpf2go bpf acl.c -- -I.
1819

20+
const (
21+
INADDR_TEST = "127.0.0.1"
22+
BindPort = 9090
23+
24+
DATA = "testing"
25+
)
26+
1927
// go generate .
20-
// CGO_ENABLED=0 go run .
28+
// CGO_ENABLED=0 go run . --action=1
29+
// CGO_ENABLED=0 go run . --action=0
2130
func main() {
2231
logrus.SetReportCaller(true)
2332

33+
actionArg := flag.Int("action", 1, "for xdp")
34+
iface := flag.String("iface", "lo", "the interface to attach this program to")
35+
flag.Parse()
36+
2437
stopCh := make(chan os.Signal, 1)
2538
signal.Notify(stopCh, os.Interrupt, syscall.SIGTERM)
2639

40+
serverFd := startServer()
41+
defer unix.Close(serverFd)
42+
clientFd := connectToFd(serverFd)
43+
defer unix.Close(clientFd)
44+
// connect 已经建立 tcp connection，从全队列(accept)里取出一个 connection socket
45+
clientServerFd, _, err := unix.Accept(serverFd)
46+
if err != nil {
47+
logrus.Fatalf("Accept err: %v", err)
48+
}
49+
defer unix.Close(clientServerFd)
50+
if _, err = unix.Write(clientFd, []byte(DATA)); err != nil {
51+
logrus.Fatalf("unix.Write err: %v", err)
52+
}
53+
cbuf := make([]byte, 1024)
54+
n, _, _, _, err := unix.Recvmsg(clientServerFd, cbuf, nil, 0)
55+
if err != nil {
56+
logrus.Fatalf("unix.Recvmsg err: %v", err)
57+
}
58+
cbuf = cbuf[:n]
59+
logrus.Infof("server recvmsg from client: %s", string(cbuf))
60+
2761
// Load pre-compiled programs and maps into the kernel.
2862
btfSpec, err := btf.LoadKernelSpec()
2963
if err != nil {
@@ -58,12 +92,11 @@ func main() {
5892
logrus.Error(err)
5993
}
6094

61-
ip1 := "172.16.10.3"
6295
var addr uint32
6396
if IsLittleEndian() {
64-
addr = binary.LittleEndian.Uint32(net.ParseIP(ip1).To4()) // byte[]{a,b,c,d} -> dcba
97+
addr = binary.LittleEndian.Uint32(net.ParseIP(INADDR_TEST).To4()) // byte[]{a,b,c,d} -> dcba
6598
} else {
66-
addr = binary.BigEndian.Uint32(net.ParseIP(ip1).To4()) // byte[]{a,b,c,d} -> abcd
99+
addr = binary.BigEndian.Uint32(net.ParseIP(INADDR_TEST).To4()) // byte[]{a,b,c,d} -> abcd
67100
}
68101
// serverIPs := bpfServerIps{
69102
// TargetIps: [4]uint32{
@@ -80,10 +113,10 @@ func main() {
80113

81114
endpoint := bpfEndpoint{
82115
Protocol: unix.IPPROTO_TCP,
83-
Dport: uint16(9090),
116+
Dport: uint16(BindPort),
84117
}
85118
action := bpfAction{
86-
Action: uint8(0),
119+
Action: uint8(*actionArg),
87120
}
88121
if err := objs.bpfMaps.Endpoints.Put(endpoint, action); err != nil {
89122
logrus.Error(err)
@@ -95,7 +128,7 @@ func main() {
95128
}
96129
logrus.Infof("%+v", action1)
97130

98-
ifaceObj, err := net.InterfaceByName("eth0")
131+
ifaceObj, err := net.InterfaceByName(*iface)
99132
if err != nil {
100133
logrus.Fatalf("loading objects: %v", err)
101134
}
@@ -109,12 +142,117 @@ func main() {
109142
}
110143
defer l.Close()
111144

145+
if _, err = unix.Write(clientFd, []byte(DATA)); err != nil {
146+
logrus.Fatalf("unix.Write err: %v", err)
147+
}
148+
cbuf2 := make([]byte, 1024)
149+
n2, _, _, _, err := unix.Recvmsg(clientServerFd, cbuf2, nil, 0)
150+
if err != nil {
151+
logrus.Fatalf("unix.Recvmsg err: %v", err)
152+
}
153+
cbuf2 = cbuf2[:n2]
154+
logrus.Infof("server recvmsg from client: %s", string(cbuf2))
155+
112156
// Wait
113157
<-stopCh
114158
}
115159

160+
func connectToFd(serverFd int) int {
161+
socketType, err := unix.GetsockoptInt(serverFd, unix.SOL_SOCKET, unix.SO_TYPE)
162+
if err != nil {
163+
logrus.Fatal(err)
164+
}
165+
166+
//tcpSaveSyn, err := unix.GetsockoptInt(serverFd, unix.SOL_TCP, unix.TCP_SAVE_SYN)
167+
168+
serverSockAddr, err := unix.Getsockname(serverFd)
169+
if err != nil {
170+
logrus.Fatal(err)
171+
}
172+
173+
clientFd, err := unix.Socket(unix.AF_INET, socketType, 0)
174+
if err != nil {
175+
logrus.Fatal(err)
176+
}
177+
setSocketTimeout(clientFd, 5000)
178+
179+
ip := net.ParseIP(INADDR_TEST)
180+
sa := &unix.SockaddrInet4{
181+
Port: BindPort + 1,
182+
Addr: [4]byte{},
183+
}
184+
copy(sa.Addr[:], ip)
185+
err = unix.Bind(clientFd, sa)
186+
if err != nil {
187+
logrus.Fatal(err)
188+
}
189+
190+
// 非阻塞的
191+
err = unix.Connect(clientFd, serverSockAddr)
192+
if err != nil {
193+
logrus.Fatal(err)
194+
}
195+
196+
return clientFd
197+
}
198+
199+
func startServer() int {
200+
serverFd, err := unix.Socket(unix.AF_INET, unix.SOCK_STREAM, 0)
201+
if err != nil {
202+
logrus.Fatal(err)
203+
}
204+
setSocketTimeout(serverFd, 5000)
205+
206+
err = unix.SetsockoptInt(serverFd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1)
207+
if err != nil {
208+
logrus.Fatalf("unix.SO_REUSEADDR error: %v", err)
209+
}
210+
211+
ip := net.ParseIP(INADDR_TEST)
212+
sa := &unix.SockaddrInet4{
213+
Port: BindPort,
214+
Addr: [4]byte{},
215+
}
216+
copy(sa.Addr[:], ip)
217+
err = unix.Bind(serverFd, sa)
218+
if err != nil {
219+
logrus.Fatal(err)
220+
}
221+
222+
err = unix.Listen(serverFd, 1)
223+
if err != nil {
224+
logrus.Fatal(err)
225+
}
226+
227+
return serverFd
228+
}
229+
230+
func setSocketTimeout(fd, timeoutMs int) {
231+
var timeVal *unix.Timeval
232+
if timeoutMs > 0 {
233+
timeVal = &unix.Timeval{
234+
Sec: int64(timeoutMs / 1000),
235+
Usec: int64(timeoutMs % 1000 * 1000),
236+
}
237+
} else {
238+
timeVal = &unix.Timeval{
239+
Sec: 3,
240+
}
241+
}
242+
243+
err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, timeVal)
244+
if err != nil {
245+
logrus.Fatal(err)
246+
}
247+
248+
err = unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_SNDTIMEO, timeVal)
249+
if err != nil {
250+
logrus.Fatal(err)
251+
}
252+
}
253+
116254
func IsLittleEndian() bool {
117-
var val int32 = 0x1
255+
val := int32(0x1)
118256

119257
return *(*byte)(unsafe.Pointer(&val)) == 1
120258
}

0 commit comments

Comments
 (0)
0