[go: up one dir, main page]
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the AF_INET socket handler.
8 *
9 * Version: @(#)sock.h 1.0.4 05/13/93
10 *
11 * Authors: Ross Biro
12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13 * Corey Minyard <wf-rch!minyard@relay.EU.net>
14 * Florian La Roche <flla@stud.uni-sb.de>
15 *
16 * Fixes:
17 * Alan Cox : Volatiles in skbuff pointers. See
18 * skbuff comments. May be overdone,
19 * better to prove they can be removed
20 * than the reverse.
21 * Alan Cox : Added a zapped field for tcp to note
22 * a socket is reset and must stay shut up
23 * Alan Cox : New fields for options
24 * Pauline Middelink : identd support
25 * Alan Cox : Eliminate low level recv/recvfrom
26 * David S. Miller : New socket lookup architecture.
27 * Steve Whitehouse: Default routines for sock_ops
28 * Arnaldo C. Melo : removed net_pinfo, tp_pinfo and made
29 * protinfo be just a void pointer, as the
30 * protocol specific parts were moved to
31 * respective headers and ipv4/v6, etc now
32 * use private slabcaches for its socks
33 * Pedro Hortas : New flags field for socket options
34 */
35#ifndef _SOCK_H
36#define _SOCK_H
37
38#include <linux/hardirq.h>
39#include <linux/kernel.h>
40#include <linux/list.h>
41#include <linux/list_nulls.h>
42#include <linux/timer.h>
43#include <linux/cache.h>
44#include <linux/bitops.h>
45#include <linux/lockdep.h>
46#include <linux/netdevice.h>
47#include <linux/skbuff.h> /* struct sk_buff */
48#include <linux/mm.h>
49#include <linux/security.h>
50#include <linux/slab.h>
51#include <linux/uaccess.h>
52#include <linux/page_counter.h>
53#include <linux/memcontrol.h>
54#include <linux/static_key.h>
55#include <linux/sched.h>
56#include <linux/wait.h>
57#include <linux/cgroup-defs.h>
58#include <linux/rbtree.h>
59#include <linux/rculist_nulls.h>
60#include <linux/poll.h>
61#include <linux/sockptr.h>
62#include <linux/indirect_call_wrapper.h>
63#include <linux/atomic.h>
64#include <linux/refcount.h>
65#include <linux/llist.h>
66#include <net/dst.h>
67#include <net/checksum.h>
68#include <net/tcp_states.h>
69#include <linux/net_tstamp.h>
70#include <net/l3mdev.h>
71#include <uapi/linux/socket.h>
72
73/*
74 * This structure really needs to be cleaned up.
75 * Most of it is for TCP, and not used by any of
76 * the other protocols.
77 */
78
79/* This is the per-socket lock. The spinlock provides a synchronization
80 * between user contexts and software interrupt processing, whereas the
81 * mini-semaphore synchronizes multiple users amongst themselves.
82 */
83typedef struct {
84 spinlock_t slock;
85 int owned;
86 wait_queue_head_t wq;
87 /*
88 * We express the mutex-alike socket_lock semantics
89 * to the lock validator by explicitly managing
90 * the slock as a lock variant (in addition to
91 * the slock itself):
92 */
93#ifdef CONFIG_DEBUG_LOCK_ALLOC
94 struct lockdep_map dep_map;
95#endif
96} socket_lock_t;
97
98struct sock;
99struct proto;
100struct net;
101
102typedef __u32 __bitwise __portpair;
103typedef __u64 __bitwise __addrpair;
104
105/**
106 * struct sock_common - minimal network layer representation of sockets
107 * @skc_daddr: Foreign IPv4 addr
108 * @skc_rcv_saddr: Bound local IPv4 addr
109 * @skc_addrpair: 8-byte-aligned __u64 union of @skc_daddr & @skc_rcv_saddr
110 * @skc_hash: hash value used with various protocol lookup tables
111 * @skc_u16hashes: two u16 hash values used by UDP lookup tables
112 * @skc_dport: placeholder for inet_dport/tw_dport
113 * @skc_num: placeholder for inet_num/tw_num
114 * @skc_portpair: __u32 union of @skc_dport & @skc_num
115 * @skc_family: network address family
116 * @skc_state: Connection state
117 * @skc_reuse: %SO_REUSEADDR setting
118 * @skc_reuseport: %SO_REUSEPORT setting
119 * @skc_ipv6only: socket is IPV6 only
120 * @skc_net_refcnt: socket is using net ref counting
121 * @skc_bypass_prot_mem: bypass the per-protocol memory accounting for skb
122 * @skc_bound_dev_if: bound device index if != 0
123 * @skc_bind_node: bind hash linkage for various protocol lookup tables
124 * @skc_portaddr_node: second hash linkage for UDP/UDP-Lite protocol
125 * @skc_prot: protocol handlers inside a network family
126 * @skc_net: reference to the network namespace of this socket
127 * @skc_v6_daddr: IPV6 destination address
128 * @skc_v6_rcv_saddr: IPV6 source address
129 * @skc_cookie: socket's cookie value
130 * @skc_node: main hash linkage for various protocol lookup tables
131 * @skc_nulls_node: main hash linkage for TCP/UDP/UDP-Lite protocol
132 * @skc_tx_queue_mapping: tx queue number for this connection
133 * @skc_rx_queue_mapping: rx queue number for this connection
134 * @skc_flags: place holder for sk_flags
135 * %SO_LINGER (l_onoff), %SO_BROADCAST, %SO_KEEPALIVE,
136 * %SO_OOBINLINE settings, %SO_TIMESTAMPING settings
137 * @skc_listener: connection request listener socket (aka rsk_listener)
138 * [union with @skc_flags]
139 * @skc_tw_dr: (aka tw_dr) ptr to &struct inet_timewait_death_row
140 * [union with @skc_flags]
141 * @skc_incoming_cpu: record/match cpu processing incoming packets
142 * @skc_rcv_wnd: (aka rsk_rcv_wnd) TCP receive window size (possibly scaled)
143 * [union with @skc_incoming_cpu]
144 * @skc_tw_rcv_nxt: (aka tw_rcv_nxt) TCP window next expected seq number
145 * [union with @skc_incoming_cpu]
146 * @skc_refcnt: reference count
147 *
148 * This is the minimal network layer representation of sockets, the header
149 * for struct sock and struct inet_timewait_sock.
150 */
151struct sock_common {
152 union {
153 __addrpair skc_addrpair;
154 struct {
155 __be32 skc_daddr;
156 __be32 skc_rcv_saddr;
157 };
158 };
159 union {
160 unsigned int skc_hash;
161 __u16 skc_u16hashes[2];
162 };
163 /* skc_dport && skc_num must be grouped as well */
164 union {
165 __portpair skc_portpair;
166 struct {
167 __be16 skc_dport;
168 __u16 skc_num;
169 };
170 };
171
172 unsigned short skc_family;
173 volatile unsigned char skc_state;
174 unsigned char skc_reuse:4;
175 unsigned char skc_reuseport:1;
176 unsigned char skc_ipv6only:1;
177 unsigned char skc_net_refcnt:1;
178 unsigned char skc_bypass_prot_mem:1;
179 int skc_bound_dev_if;
180 union {
181 struct hlist_node skc_bind_node;
182 struct hlist_node skc_portaddr_node;
183 };
184 struct proto *skc_prot;
185 possible_net_t skc_net;
186
187#if IS_ENABLED(CONFIG_IPV6)
188 struct in6_addr skc_v6_daddr;
189 struct in6_addr skc_v6_rcv_saddr;
190#endif
191
192 atomic64_t skc_cookie;
193
194 /* following fields are padding to force
195 * offset(struct sock, sk_refcnt) == 128 on 64bit arches
196 * assuming IPV6 is enabled. We use this padding differently
197 * for different kind of 'sockets'
198 */
199 union {
200 unsigned long skc_flags;
201 struct sock *skc_listener; /* request_sock */
202 struct inet_timewait_death_row *skc_tw_dr; /* inet_timewait_sock */
203 };
204 /*
205 * fields between dontcopy_begin/dontcopy_end
206 * are not copied in sock_copy()
207 */
208 /* private: */
209 int skc_dontcopy_begin[0];
210 /* public: */
211 union {
212 struct hlist_node skc_node;
213 struct hlist_nulls_node skc_nulls_node;
214 };
215 unsigned short skc_tx_queue_mapping;
216#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
217 unsigned short skc_rx_queue_mapping;
218#endif
219 union {
220 int skc_incoming_cpu;
221 u32 skc_rcv_wnd;
222 u32 skc_tw_rcv_nxt; /* struct tcp_timewait_sock */
223 };
224
225 refcount_t skc_refcnt;
226 /* private: */
227 int skc_dontcopy_end[0];
228 union {
229 u32 skc_rxhash;
230 u32 skc_window_clamp;
231 u32 skc_tw_snd_nxt; /* struct tcp_timewait_sock */
232 };
233 /* public: */
234};
235
236struct bpf_local_storage;
237struct sk_filter;
238
239/**
240 * struct sock - network layer representation of sockets
241 * @__sk_common: shared layout with inet_timewait_sock
242 * @sk_shutdown: mask of %SEND_SHUTDOWN and/or %RCV_SHUTDOWN
243 * @sk_userlocks: %SO_SNDBUF and %SO_RCVBUF settings
244 * @sk_lock: synchronizer
245 * @sk_kern_sock: True if sock is using kernel lock classes
246 * @sk_rcvbuf: size of receive buffer in bytes
247 * @sk_wq: sock wait queue and async head
248 * @sk_rx_dst: receive input route used by early demux
249 * @sk_rx_dst_ifindex: ifindex for @sk_rx_dst
250 * @sk_rx_dst_cookie: cookie for @sk_rx_dst
251 * @sk_dst_cache: destination cache
252 * @sk_dst_pending_confirm: need to confirm neighbour
253 * @sk_policy: flow policy
254 * @psp_assoc: PSP association, if socket is PSP-secured
255 * @sk_receive_queue: incoming packets
256 * @sk_wmem_alloc: transmit queue bytes committed
257 * @sk_tsq_flags: TCP Small Queues flags
258 * @sk_write_queue: Packet sending queue
259 * @sk_omem_alloc: "o" is "option" or "other"
260 * @sk_wmem_queued: persistent queue size
261 * @sk_forward_alloc: space allocated forward
262 * @sk_reserved_mem: space reserved and non-reclaimable for the socket
263 * @sk_napi_id: id of the last napi context to receive data for sk
264 * @sk_ll_usec: usecs to busypoll when there is no data
265 * @sk_allocation: allocation mode
266 * @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler)
267 * @sk_pacing_status: Pacing status (requested, handled by sch_fq)
268 * @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE)
269 * @sk_sndbuf: size of send buffer in bytes
270 * @sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets
271 * @sk_no_check_rx: allow zero checksum in RX packets
272 * @sk_route_caps: route capabilities (e.g. %NETIF_F_TSO)
273 * @sk_gso_disabled: if set, NETIF_F_GSO_MASK is forbidden.
274 * @sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4)
275 * @sk_gso_max_size: Maximum GSO segment size to build
276 * @sk_gso_max_segs: Maximum number of GSO segments
277 * @sk_pacing_shift: scaling factor for TCP Small Queues
278 * @sk_lingertime: %SO_LINGER l_linger setting
279 * @sk_backlog: always used with the per-socket spinlock held
280 * @sk_callback_lock: used with the callbacks in the end of this struct
281 * @sk_error_queue: rarely used
282 * @sk_prot_creator: sk_prot of original sock creator (see ipv6_setsockopt,
283 * IPV6_ADDRFORM for instance)
284 * @sk_err: last error
285 * @sk_err_soft: errors that don't cause failure but are the cause of a
286 * persistent failure not just 'timed out'
287 * @sk_drops: raw/udp drops counter
288 * @sk_drop_counters: optional pointer to numa_drop_counters
289 * @sk_ack_backlog: current listen backlog
290 * @sk_max_ack_backlog: listen backlog set in listen()
291 * @sk_uid: user id of owner
292 * @sk_ino: inode number (zero if orphaned)
293 * @sk_prefer_busy_poll: prefer busypolling over softirq processing
294 * @sk_busy_poll_budget: napi processing budget when busypolling
295 * @sk_priority: %SO_PRIORITY setting
296 * @sk_type: socket type (%SOCK_STREAM, etc)
297 * @sk_protocol: which protocol this socket belongs in this network family
298 * @sk_peer_lock: lock protecting @sk_peer_pid and @sk_peer_cred
299 * @sk_peer_pid: &struct pid for this socket's peer
300 * @sk_peer_cred: %SO_PEERCRED setting
301 * @sk_rcvlowat: %SO_RCVLOWAT setting
302 * @sk_rcvtimeo: %SO_RCVTIMEO setting
303 * @sk_sndtimeo: %SO_SNDTIMEO setting
304 * @sk_txhash: computed flow hash for use on transmit
305 * @sk_txrehash: enable TX hash rethink
306 * @sk_filter: socket filtering instructions
307 * @sk_timer: sock cleanup timer
308 * @tcp_retransmit_timer: tcp retransmit timer
309 * @mptcp_retransmit_timer: mptcp retransmit timer
310 * @sk_stamp: time stamp of last packet received
311 * @sk_stamp_seq: lock for accessing sk_stamp on 32 bit architectures only
312 * @sk_tsflags: SO_TIMESTAMPING flags
313 * @sk_bpf_cb_flags: used in bpf_setsockopt()
314 * @sk_use_task_frag: allow sk_page_frag() to use current->task_frag.
315 * Sockets that can be used under memory reclaim should
316 * set this to false.
317 * @sk_bind_phc: SO_TIMESTAMPING bind PHC index of PTP virtual clock
318 * for timestamping
319 * @sk_tskey: counter to disambiguate concurrent tstamp requests
320 * @sk_tx_queue_mapping_jiffies: time in jiffies of last @sk_tx_queue_mapping refresh.
321 * @sk_zckey: counter to order MSG_ZEROCOPY notifications
322 * @sk_socket: Identd and reporting IO signals
323 * @sk_user_data: RPC layer private data. Write-protected by @sk_callback_lock.
324 * @sk_frag: cached page frag
325 * @sk_peek_off: current peek_offset value
326 * @sk_send_head: front of stuff to transmit
327 * @tcp_rtx_queue: TCP re-transmit queue [union with @sk_send_head]
328 * @sk_security: used by security modules
329 * @sk_mark: generic packet mark
330 * @sk_cgrp_data: cgroup data for this cgroup
331 * @sk_memcg: this socket's memory cgroup association
332 * @sk_write_pending: a write to stream socket waits to start
333 * @sk_disconnects: number of disconnect operations performed on this sock
334 * @sk_state_change: callback to indicate change in the state of the sock
335 * @sk_data_ready: callback to indicate there is data to be processed
336 * @sk_write_space: callback to indicate there is bf sending space available
337 * @sk_error_report: callback to indicate errors (e.g. %MSG_ERRQUEUE)
338 * @sk_backlog_rcv: callback to process the backlog
339 * @sk_validate_xmit_skb: ptr to an optional validate function
340 * @sk_destruct: called at sock freeing time, i.e. when all refcnt == 0
341 * @sk_reuseport_cb: reuseport group container
342 * @sk_bpf_storage: ptr to cache and control for bpf_sk_storage
343 * @sk_rcu: used during RCU grace period
344 * @sk_clockid: clockid used by time-based scheduling (SO_TXTIME)
345 * @sk_txtime_deadline_mode: set deadline mode for SO_TXTIME
346 * @sk_txtime_report_errors: set report errors mode for SO_TXTIME
347 * @sk_txtime_unused: unused txtime flags
348 * @sk_scm_recv_flags: all flags used by scm_recv()
349 * @sk_scm_credentials: flagged by SO_PASSCRED to recv SCM_CREDENTIALS
350 * @sk_scm_security: flagged by SO_PASSSEC to recv SCM_SECURITY
351 * @sk_scm_pidfd: flagged by SO_PASSPIDFD to recv SCM_PIDFD
352 * @sk_scm_rights: flagged by SO_PASSRIGHTS to recv SCM_RIGHTS
353 * @sk_scm_unused: unused flags for scm_recv()
354 * @ns_tracker: tracker for netns reference
355 * @sk_user_frags: xarray of pages the user is holding a reference on.
356 * @sk_owner: reference to the real owner of the socket that calls
357 * sock_lock_init_class_and_name().
358 */
359struct sock {
360 /*
361 * Now struct inet_timewait_sock also uses sock_common, so please just
362 * don't add nothing before this first member (__sk_common) --acme
363 */
364 struct sock_common __sk_common;
365#define sk_node __sk_common.skc_node
366#define sk_nulls_node __sk_common.skc_nulls_node
367#define sk_refcnt __sk_common.skc_refcnt
368#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
369#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
370#define sk_rx_queue_mapping __sk_common.skc_rx_queue_mapping
371#endif
372
373#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
374#define sk_dontcopy_end __sk_common.skc_dontcopy_end
375#define sk_hash __sk_common.skc_hash
376#define sk_portpair __sk_common.skc_portpair
377#define sk_num __sk_common.skc_num
378#define sk_dport __sk_common.skc_dport
379#define sk_addrpair __sk_common.skc_addrpair
380#define sk_daddr __sk_common.skc_daddr
381#define sk_rcv_saddr __sk_common.skc_rcv_saddr
382#define sk_family __sk_common.skc_family
383#define sk_state __sk_common.skc_state
384#define sk_reuse __sk_common.skc_reuse
385#define sk_reuseport __sk_common.skc_reuseport
386#define sk_ipv6only __sk_common.skc_ipv6only
387#define sk_net_refcnt __sk_common.skc_net_refcnt
388#define sk_bypass_prot_mem __sk_common.skc_bypass_prot_mem
389#define sk_bound_dev_if __sk_common.skc_bound_dev_if
390#define sk_bind_node __sk_common.skc_bind_node
391#define sk_prot __sk_common.skc_prot
392#define sk_net __sk_common.skc_net
393#define sk_v6_daddr __sk_common.skc_v6_daddr
394#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
395#define sk_cookie __sk_common.skc_cookie
396#define sk_incoming_cpu __sk_common.skc_incoming_cpu
397#define sk_flags __sk_common.skc_flags
398#define sk_rxhash __sk_common.skc_rxhash
399
400 __cacheline_group_begin(sock_write_rx);
401
402 atomic_t sk_drops;
403 __s32 sk_peek_off;
404 struct sk_buff_head sk_error_queue;
405 struct sk_buff_head sk_receive_queue;
406 /*
407 * The backlog queue is special, it is always used with
408 * the per-socket spinlock held and requires low latency
409 * access. Therefore we special case it's implementation.
410 * Note : rmem_alloc is in this structure to fill a hole
411 * on 64bit arches, not because its logically part of
412 * backlog.
413 */
414 struct {
415 atomic_t rmem_alloc;
416 int len;
417 struct sk_buff *head;
418 struct sk_buff *tail;
419 } sk_backlog;
420#define sk_rmem_alloc sk_backlog.rmem_alloc
421
422 __cacheline_group_end(sock_write_rx);
423
424 __cacheline_group_begin(sock_read_rx);
425 /* early demux fields */
426 struct dst_entry __rcu *sk_rx_dst;
427 int sk_rx_dst_ifindex;
428 u32 sk_rx_dst_cookie;
429
430#ifdef CONFIG_NET_RX_BUSY_POLL
431 unsigned int sk_ll_usec;
432 unsigned int sk_napi_id;
433 u16 sk_busy_poll_budget;
434 u8 sk_prefer_busy_poll;
435#endif
436 u8 sk_userlocks;
437 int sk_rcvbuf;
438
439 struct sk_filter __rcu *sk_filter;
440 union {
441 struct socket_wq __rcu *sk_wq;
442 /* private: */
443 struct socket_wq *sk_wq_raw;
444 /* public: */
445 };
446
447 void (*sk_data_ready)(struct sock *sk);
448 long sk_rcvtimeo;
449 int sk_rcvlowat;
450 __cacheline_group_end(sock_read_rx);
451
452 __cacheline_group_begin(sock_read_rxtx);
453 int sk_err;
454 struct socket *sk_socket;
455#ifdef CONFIG_MEMCG
456 struct mem_cgroup *sk_memcg;
457#endif
458#ifdef CONFIG_XFRM
459 struct xfrm_policy __rcu *sk_policy[2];
460#endif
461#if IS_ENABLED(CONFIG_INET_PSP)
462 struct psp_assoc __rcu *psp_assoc;
463#endif
464 __cacheline_group_end(sock_read_rxtx);
465
466 __cacheline_group_begin(sock_write_rxtx);
467 socket_lock_t sk_lock;
468 u32 sk_reserved_mem;
469 int sk_forward_alloc;
470 u32 sk_tsflags;
471 __cacheline_group_end(sock_write_rxtx);
472
473 __cacheline_group_begin(sock_write_tx);
474 int sk_write_pending;
475 atomic_t sk_omem_alloc;
476 int sk_err_soft;
477
478 int sk_wmem_queued;
479 refcount_t sk_wmem_alloc;
480 unsigned long sk_tsq_flags;
481 union {
482 struct sk_buff *sk_send_head;
483 struct rb_root tcp_rtx_queue;
484 };
485 struct sk_buff_head sk_write_queue;
486 struct page_frag sk_frag;
487 union {
488 struct timer_list sk_timer;
489 struct timer_list tcp_retransmit_timer;
490 struct timer_list mptcp_retransmit_timer;
491 };
492 unsigned long sk_pacing_rate; /* bytes per second */
493 atomic_t sk_zckey;
494 atomic_t sk_tskey;
495 unsigned long sk_tx_queue_mapping_jiffies;
496 __cacheline_group_end(sock_write_tx);
497
498 __cacheline_group_begin(sock_read_tx);
499 u32 sk_dst_pending_confirm;
500 u32 sk_pacing_status; /* see enum sk_pacing */
501 unsigned long sk_max_pacing_rate;
502 long sk_sndtimeo;
503 u32 sk_priority;
504 u32 sk_mark;
505 kuid_t sk_uid;
506 u16 sk_protocol;
507 u16 sk_type;
508 struct dst_entry __rcu *sk_dst_cache;
509 netdev_features_t sk_route_caps;
510#ifdef CONFIG_SOCK_VALIDATE_XMIT
511 struct sk_buff* (*sk_validate_xmit_skb)(struct sock *sk,
512 struct net_device *dev,
513 struct sk_buff *skb);
514#endif
515 u16 sk_gso_type;
516 u16 sk_gso_max_segs;
517 unsigned int sk_gso_max_size;
518 gfp_t sk_allocation;
519 u32 sk_txhash;
520 int sk_sndbuf;
521 u8 sk_pacing_shift;
522 bool sk_use_task_frag;
523 __cacheline_group_end(sock_read_tx);
524
525 /*
526 * Because of non atomicity rules, all
527 * changes are protected by socket lock.
528 */
529 u8 sk_gso_disabled : 1,
530 sk_kern_sock : 1,
531 sk_no_check_tx : 1,
532 sk_no_check_rx : 1;
533 u8 sk_shutdown;
534 unsigned long sk_lingertime;
535 struct proto *sk_prot_creator;
536 rwlock_t sk_callback_lock;
537 u32 sk_ack_backlog;
538 u32 sk_max_ack_backlog;
539 unsigned long sk_ino;
540 spinlock_t sk_peer_lock;
541 int sk_bind_phc;
542 struct pid *sk_peer_pid;
543 const struct cred *sk_peer_cred;
544
545 ktime_t sk_stamp;
546#if BITS_PER_LONG==32
547 seqlock_t sk_stamp_seq;
548#endif
549 int sk_disconnects;
550
551 union {
552 u8 sk_txrehash;
553 u8 sk_scm_recv_flags;
554 struct {
555 u8 sk_scm_credentials : 1,
556 sk_scm_security : 1,
557 sk_scm_pidfd : 1,
558 sk_scm_rights : 1,
559 sk_scm_unused : 4;
560 };
561 };
562 u8 sk_clockid;
563 u8 sk_txtime_deadline_mode : 1,
564 sk_txtime_report_errors : 1,
565 sk_txtime_unused : 6;
566#define SK_BPF_CB_FLAG_TEST(SK, FLAG) ((SK)->sk_bpf_cb_flags & (FLAG))
567 u8 sk_bpf_cb_flags;
568
569 void *sk_user_data;
570#ifdef CONFIG_SECURITY
571 void *sk_security;
572#endif
573 struct sock_cgroup_data sk_cgrp_data;
574 void (*sk_state_change)(struct sock *sk);
575 void (*sk_write_space)(struct sock *sk);
576 void (*sk_error_report)(struct sock *sk);
577 int (*sk_backlog_rcv)(struct sock *sk,
578 struct sk_buff *skb);
579 void (*sk_destruct)(struct sock *sk);
580 struct sock_reuseport __rcu *sk_reuseport_cb;
581#ifdef CONFIG_BPF_SYSCALL
582 struct bpf_local_storage __rcu *sk_bpf_storage;
583#endif
584 struct numa_drop_counters *sk_drop_counters;
585 struct rcu_head sk_rcu;
586 netns_tracker ns_tracker;
587 struct xarray sk_user_frags;
588
589#if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES)
590 struct module *sk_owner;
591#endif
592};
593
594struct sock_bh_locked {
595 struct sock *sock;
596 local_lock_t bh_lock;
597};
598
599enum sk_pacing {
600 SK_PACING_NONE = 0,
601 SK_PACING_NEEDED = 1,
602 SK_PACING_FQ = 2,
603};
604
605/* flag bits in sk_user_data
606 *
607 * - SK_USER_DATA_NOCOPY: Pointer stored in sk_user_data might
608 * not be suitable for copying when cloning the socket. For instance,
609 * it can point to a reference counted object. sk_user_data bottom
610 * bit is set if pointer must not be copied.
611 *
612 * - SK_USER_DATA_BPF: Mark whether sk_user_data field is
613 * managed/owned by a BPF reuseport array. This bit should be set
614 * when sk_user_data's sk is added to the bpf's reuseport_array.
615 *
616 * - SK_USER_DATA_PSOCK: Mark whether pointer stored in
617 * sk_user_data points to psock type. This bit should be set
618 * when sk_user_data is assigned to a psock object.
619 */
620#define SK_USER_DATA_NOCOPY 1UL
621#define SK_USER_DATA_BPF 2UL
622#define SK_USER_DATA_PSOCK 4UL
623#define SK_USER_DATA_PTRMASK ~(SK_USER_DATA_NOCOPY | SK_USER_DATA_BPF |\
624 SK_USER_DATA_PSOCK)
625
626/**
627 * sk_user_data_is_nocopy - Test if sk_user_data pointer must not be copied
628 * @sk: socket
629 */
630static inline bool sk_user_data_is_nocopy(const struct sock *sk)
631{
632 return ((uintptr_t)sk->sk_user_data & SK_USER_DATA_NOCOPY);
633}
634
635#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
636
637/**
638 * __locked_read_sk_user_data_with_flags - return the pointer
639 * only if argument flags all has been set in sk_user_data. Otherwise
640 * return NULL
641 *
642 * @sk: socket
643 * @flags: flag bits
644 *
645 * The caller must be holding sk->sk_callback_lock.
646 */
647static inline void *
648__locked_read_sk_user_data_with_flags(const struct sock *sk,
649 uintptr_t flags)
650{
651 uintptr_t sk_user_data =
652 (uintptr_t)rcu_dereference_check(__sk_user_data(sk),
653 lockdep_is_held(&sk->sk_callback_lock));
654
655 WARN_ON_ONCE(flags & SK_USER_DATA_PTRMASK);
656
657 if ((sk_user_data & flags) == flags)
658 return (void *)(sk_user_data & SK_USER_DATA_PTRMASK);
659 return NULL;
660}
661
662/**
663 * __rcu_dereference_sk_user_data_with_flags - return the pointer
664 * only if argument flags all has been set in sk_user_data. Otherwise
665 * return NULL
666 *
667 * @sk: socket
668 * @flags: flag bits
669 */
670static inline void *
671__rcu_dereference_sk_user_data_with_flags(const struct sock *sk,
672 uintptr_t flags)
673{
674 uintptr_t sk_user_data = (uintptr_t)rcu_dereference(__sk_user_data(sk));
675
676 WARN_ON_ONCE(flags & SK_USER_DATA_PTRMASK);
677
678 if ((sk_user_data & flags) == flags)
679 return (void *)(sk_user_data & SK_USER_DATA_PTRMASK);
680 return NULL;
681}
682
683#define rcu_dereference_sk_user_data(sk) \
684 __rcu_dereference_sk_user_data_with_flags(sk, 0)
685#define __rcu_assign_sk_user_data_with_flags(sk, ptr, flags) \
686({ \
687 uintptr_t __tmp1 = (uintptr_t)(ptr), \
688 __tmp2 = (uintptr_t)(flags); \
689 WARN_ON_ONCE(__tmp1 & ~SK_USER_DATA_PTRMASK); \
690 WARN_ON_ONCE(__tmp2 & SK_USER_DATA_PTRMASK); \
691 rcu_assign_pointer(__sk_user_data((sk)), \
692 __tmp1 | __tmp2); \
693})
694#define rcu_assign_sk_user_data(sk, ptr) \
695 __rcu_assign_sk_user_data_with_flags(sk, ptr, 0)
696
697static inline
698struct net *sock_net(const struct sock *sk)
699{
700 return read_pnet(pnet: &sk->sk_net);
701}
702
703static inline
704void sock_net_set(struct sock *sk, struct net *net)
705{
706 write_pnet(pnet: &sk->sk_net, net);
707}
708
709/*
710 * SK_CAN_REUSE and SK_NO_REUSE on a socket mean that the socket is OK
711 * or not whether his port will be reused by someone else. SK_FORCE_REUSE
712 * on a socket means that the socket will reuse everybody else's port
713 * without looking at the other's sk_reuse value.
714 */
715
716#define SK_NO_REUSE 0
717#define SK_CAN_REUSE 1
718#define SK_FORCE_REUSE 2
719
720int sk_set_peek_off(struct sock *sk, int val);
721
722static inline int sk_peek_offset(const struct sock *sk, int flags)
723{
724 if (unlikely(flags & MSG_PEEK)) {
725 return READ_ONCE(sk->sk_peek_off);
726 }
727
728 return 0;
729}
730
731static inline void sk_peek_offset_bwd(struct sock *sk, int val)
732{
733 s32 off = READ_ONCE(sk->sk_peek_off);
734
735 if (unlikely(off >= 0)) {
736 off = max_t(s32, off - val, 0);
737 WRITE_ONCE(sk->sk_peek_off, off);
738 }
739}
740
741static inline void sk_peek_offset_fwd(struct sock *sk, int val)
742{
743 sk_peek_offset_bwd(sk, val: -val);
744}
745
746/*
747 * Hashed lists helper routines
748 */
749static inline struct sock *sk_entry(const struct hlist_node *node)
750{
751 return hlist_entry(node, struct sock, sk_node);
752}
753
754static inline struct sock *__sk_head(const struct hlist_head *head)
755{
756 return hlist_entry(head->first, struct sock, sk_node);
757}
758
759static inline struct sock *sk_head(const struct hlist_head *head)
760{
761 return hlist_empty(h: head) ? NULL : __sk_head(head);
762}
763
764static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
765{
766 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
767}
768
769static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
770{
771 return hlist_nulls_empty(h: head) ? NULL : __sk_nulls_head(head);
772}
773
774static inline struct sock *sk_next(const struct sock *sk)
775{
776 return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
777}
778
779static inline struct sock *sk_nulls_next(const struct sock *sk)
780{
781 return (!is_a_nulls(ptr: sk->sk_nulls_node.next)) ?
782 hlist_nulls_entry(sk->sk_nulls_node.next,
783 struct sock, sk_nulls_node) :
784 NULL;
785}
786
787static inline bool sk_unhashed(const struct sock *sk)
788{
789 return hlist_unhashed(h: &sk->sk_node);
790}
791
792static inline bool sk_hashed(const struct sock *sk)
793{
794 return !sk_unhashed(sk);
795}
796
797static inline void sk_node_init(struct hlist_node *node)
798{
799 node->pprev = NULL;
800}
801
802static inline void __sk_del_node(struct sock *sk)
803{
804 __hlist_del(n: &sk->sk_node);
805}
806
807/* NB: equivalent to hlist_del_init_rcu */
808static inline bool __sk_del_node_init(struct sock *sk)
809{
810 if (sk_hashed(sk)) {
811 __sk_del_node(sk);
812 sk_node_init(node: &sk->sk_node);
813 return true;
814 }
815 return false;
816}
817
818/* Grab socket reference count. This operation is valid only
819 when sk is ALREADY grabbed f.e. it is found in hash table
820 or a list and the lookup is made under lock preventing hash table
821 modifications.
822 */
823
824static __always_inline void sock_hold(struct sock *sk)
825{
826 refcount_inc(r: &sk->sk_refcnt);
827}
828
829/* Ungrab socket in the context, which assumes that socket refcnt
830 cannot hit zero, f.e. it is true in context of any socketcall.
831 */
832static __always_inline void __sock_put(struct sock *sk)
833{
834 refcount_dec(r: &sk->sk_refcnt);
835}
836
837static inline bool sk_del_node_init(struct sock *sk)
838{
839 bool rc = __sk_del_node_init(sk);
840
841 if (rc)
842 __sock_put(sk);
843
844 return rc;
845}
846#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
847
848static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
849{
850 if (sk_hashed(sk)) {
851 hlist_nulls_del_init_rcu(n: &sk->sk_nulls_node);
852 return true;
853 }
854 return false;
855}
856
857static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
858{
859 bool rc = __sk_nulls_del_node_init_rcu(sk);
860
861 if (rc)
862 __sock_put(sk);
863
864 return rc;
865}
866
867static inline bool sk_nulls_replace_node_init_rcu(struct sock *old,
868 struct sock *new)
869{
870 if (sk_hashed(sk: old)) {
871 hlist_nulls_replace_init_rcu(old: &old->sk_nulls_node,
872 new: &new->sk_nulls_node);
873 __sock_put(sk: old);
874 return true;
875 }
876
877 return false;
878}
879
880static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
881{
882 hlist_add_head(n: &sk->sk_node, h: list);
883}
884
885static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
886{
887 sock_hold(sk);
888 __sk_add_node(sk, list);
889}
890
891static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
892{
893 sock_hold(sk);
894 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
895 sk->sk_family == AF_INET6)
896 hlist_add_tail_rcu(n: &sk->sk_node, h: list);
897 else
898 hlist_add_head_rcu(n: &sk->sk_node, h: list);
899}
900
901static inline void sk_add_node_tail_rcu(struct sock *sk, struct hlist_head *list)
902{
903 sock_hold(sk);
904 hlist_add_tail_rcu(n: &sk->sk_node, h: list);
905}
906
907static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
908{
909 hlist_nulls_add_head_rcu(n: &sk->sk_nulls_node, h: list);
910}
911
912static inline void __sk_nulls_add_node_tail_rcu(struct sock *sk, struct hlist_nulls_head *list)
913{
914 hlist_nulls_add_tail_rcu(n: &sk->sk_nulls_node, h: list);
915}
916
917static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
918{
919 sock_hold(sk);
920 __sk_nulls_add_node_rcu(sk, list);
921}
922
923static inline void __sk_del_bind_node(struct sock *sk)
924{
925 __hlist_del(n: &sk->sk_bind_node);
926}
927
928static inline void sk_add_bind_node(struct sock *sk,
929 struct hlist_head *list)
930{
931 hlist_add_head(n: &sk->sk_bind_node, h: list);
932}
933
934#define sk_for_each(__sk, list) \
935 hlist_for_each_entry(__sk, list, sk_node)
936#define sk_for_each_rcu(__sk, list) \
937 hlist_for_each_entry_rcu(__sk, list, sk_node)
938#define sk_nulls_for_each(__sk, node, list) \
939 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
940#define sk_nulls_for_each_rcu(__sk, node, list) \
941 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
942#define sk_for_each_from(__sk) \
943 hlist_for_each_entry_from(__sk, sk_node)
944#define sk_nulls_for_each_from(__sk, node) \
945 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
946 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
947#define sk_for_each_safe(__sk, tmp, list) \
948 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
949#define sk_for_each_bound(__sk, list) \
950 hlist_for_each_entry(__sk, list, sk_bind_node)
951#define sk_for_each_bound_safe(__sk, tmp, list) \
952 hlist_for_each_entry_safe(__sk, tmp, list, sk_bind_node)
953
954/**
955 * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
956 * @tpos: the type * to use as a loop cursor.
957 * @pos: the &struct hlist_node to use as a loop cursor.
958 * @head: the head for your list.
959 * @offset: offset of hlist_node within the struct.
960 *
961 */
962#define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
963 for (pos = rcu_dereference(hlist_first_rcu(head)); \
964 pos != NULL && \
965 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
966 pos = rcu_dereference(hlist_next_rcu(pos)))
967
968static inline struct user_namespace *sk_user_ns(const struct sock *sk)
969{
970 /* Careful only use this in a context where these parameters
971 * can not change and must all be valid, such as recvmsg from
972 * userspace.
973 */
974 return sk->sk_socket->file->f_cred->user_ns;
975}
976
977/* Sock flags */
978enum sock_flags {
979 SOCK_DEAD,
980 SOCK_DONE,
981 SOCK_URGINLINE,
982 SOCK_KEEPOPEN,
983 SOCK_LINGER,
984 SOCK_DESTROY,
985 SOCK_BROADCAST,
986 SOCK_TIMESTAMP,
987 SOCK_ZAPPED,
988 SOCK_USE_WRITE_QUEUE, /* whether to call sk->sk_write_space in sock_wfree */
989 SOCK_DBG, /* %SO_DEBUG setting */
990 SOCK_RCVTSTAMP, /* %SO_TIMESTAMP setting */
991 SOCK_RCVTSTAMPNS, /* %SO_TIMESTAMPNS setting */
992 SOCK_LOCALROUTE, /* route locally only, %SO_DONTROUTE setting */
993 SOCK_MEMALLOC, /* VM depends on this socket for swapping */
994 SOCK_TIMESTAMPING_RX_SOFTWARE, /* %SOF_TIMESTAMPING_RX_SOFTWARE */
995 SOCK_FASYNC, /* fasync() active */
996 SOCK_RXQ_OVFL,
997 SOCK_ZEROCOPY, /* buffers from userspace */
998 SOCK_WIFI_STATUS, /* push wifi status to userspace */
999 SOCK_NOFCS, /* Tell NIC not to do the Ethernet FCS.
1000 * Will use last 4 bytes of packet sent from
1001 * user-space instead.
1002 */
1003 SOCK_FILTER_LOCKED, /* Filter cannot be changed anymore */
1004 SOCK_SELECT_ERR_QUEUE, /* Wake select on error queue */
1005 SOCK_RCU_FREE, /* wait rcu grace period in sk_destruct() */
1006 SOCK_TXTIME,
1007 SOCK_XDP, /* XDP is attached */
1008 SOCK_TSTAMP_NEW, /* Indicates 64 bit timestamps always */
1009 SOCK_RCVMARK, /* Receive SO_MARK ancillary data with packet */
1010 SOCK_RCVPRIORITY, /* Receive SO_PRIORITY ancillary data with packet */
1011 SOCK_TIMESTAMPING_ANY, /* Copy of sk_tsflags & TSFLAGS_ANY */
1012};
1013
1014#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
1015/*
1016 * The highest bit of sk_tsflags is reserved for kernel-internal
1017 * SOCKCM_FLAG_TS_OPT_ID. There is a check in core/sock.c to control that
1018 * SOF_TIMESTAMPING* values do not reach this reserved area
1019 */
1020#define SOCKCM_FLAG_TS_OPT_ID BIT(31)
1021
1022static inline void sock_copy_flags(struct sock *nsk, const struct sock *osk)
1023{
1024 nsk->sk_flags = osk->sk_flags;
1025}
1026
1027static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
1028{
1029 __set_bit(flag, &sk->sk_flags);
1030}
1031
1032static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
1033{
1034 __clear_bit(flag, &sk->sk_flags);
1035}
1036
1037static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit,
1038 int valbool)
1039{
1040 if (valbool)
1041 sock_set_flag(sk, flag: bit);
1042 else
1043 sock_reset_flag(sk, flag: bit);
1044}
1045
1046static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
1047{
1048 return test_bit(flag, &sk->sk_flags);
1049}
1050
1051#ifdef CONFIG_NET
1052DECLARE_STATIC_KEY_FALSE(memalloc_socks_key);
1053static inline int sk_memalloc_socks(void)
1054{
1055 return static_branch_unlikely(&memalloc_socks_key);
1056}
1057
1058void __receive_sock(struct file *file);
1059#else
1060
1061static inline int sk_memalloc_socks(void)
1062{
1063 return 0;
1064}
1065
1066static inline void __receive_sock(struct file *file)
1067{ }
1068#endif
1069
1070static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
1071{
1072 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
1073}
1074
1075static inline void sk_acceptq_removed(struct sock *sk)
1076{
1077 WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog - 1);
1078}
1079
1080static inline void sk_acceptq_added(struct sock *sk)
1081{
1082 WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog + 1);
1083}
1084
1085/* Note: If you think the test should be:
1086 * return READ_ONCE(sk->sk_ack_backlog) >= READ_ONCE(sk->sk_max_ack_backlog);
1087 * Then please take a look at commit 64a146513f8f ("[NET]: Revert incorrect accept queue backlog changes.")
1088 */
1089static inline bool sk_acceptq_is_full(const struct sock *sk)
1090{
1091 return READ_ONCE(sk->sk_ack_backlog) > READ_ONCE(sk->sk_max_ack_backlog);
1092}
1093
1094/*
1095 * Compute minimal free write space needed to queue new packets.
1096 */
1097static inline int sk_stream_min_wspace(const struct sock *sk)
1098{
1099 return READ_ONCE(sk->sk_wmem_queued) >> 1;
1100}
1101
1102static inline int sk_stream_wspace(const struct sock *sk)
1103{
1104 return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
1105}
1106
1107static inline void sk_wmem_queued_add(struct sock *sk, int val)
1108{
1109 WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);
1110}
1111
1112static inline void sk_forward_alloc_add(struct sock *sk, int val)
1113{
1114 /* Paired with lockless reads of sk->sk_forward_alloc */
1115 WRITE_ONCE(sk->sk_forward_alloc, sk->sk_forward_alloc + val);
1116}
1117
1118void sk_stream_write_space(struct sock *sk);
1119
1120/* OOB backlog add */
1121static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
1122{
1123 /* dont let skb dst not refcounted, we are going to leave rcu lock */
1124 skb_dst_force(skb);
1125
1126 if (!sk->sk_backlog.tail)
1127 WRITE_ONCE(sk->sk_backlog.head, skb);
1128 else
1129 sk->sk_backlog.tail->next = skb;
1130
1131 WRITE_ONCE(sk->sk_backlog.tail, skb);
1132 skb->next = NULL;
1133}
1134
1135/*
1136 * Take into account size of receive queue and backlog queue
1137 * Do not take into account this skb truesize,
1138 * to allow even a single big packet to come.
1139 */
1140static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
1141{
1142 unsigned int qsize = sk->sk_backlog.len + atomic_read(v: &sk->sk_rmem_alloc);
1143
1144 return qsize > limit;
1145}
1146
1147/* The per-socket spinlock must be held here. */
1148static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
1149 unsigned int limit)
1150{
1151 if (sk_rcvqueues_full(sk, limit))
1152 return -ENOBUFS;
1153
1154 /*
1155 * If the skb was allocated from pfmemalloc reserves, only
1156 * allow SOCK_MEMALLOC sockets to use it as this socket is
1157 * helping free memory
1158 */
1159 if (skb_pfmemalloc(skb) && !sock_flag(sk, flag: SOCK_MEMALLOC))
1160 return -ENOMEM;
1161
1162 __sk_add_backlog(sk, skb);
1163 sk->sk_backlog.len += skb->truesize;
1164 return 0;
1165}
1166
1167int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
1168
1169INDIRECT_CALLABLE_DECLARE(int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb));
1170INDIRECT_CALLABLE_DECLARE(int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb));
1171
1172static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1173{
1174 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
1175 return __sk_backlog_rcv(sk, skb);
1176
1177 return INDIRECT_CALL_INET(sk->sk_backlog_rcv,
1178 tcp_v6_do_rcv,
1179 tcp_v4_do_rcv,
1180 sk, skb);
1181}
1182
1183static inline void sk_incoming_cpu_update(struct sock *sk)
1184{
1185 int cpu = raw_smp_processor_id();
1186
1187 if (unlikely(READ_ONCE(sk->sk_incoming_cpu) != cpu))
1188 WRITE_ONCE(sk->sk_incoming_cpu, cpu);
1189}
1190
1191
1192static inline void sock_rps_save_rxhash(struct sock *sk,
1193 const struct sk_buff *skb)
1194{
1195#ifdef CONFIG_RPS
1196 /* The following WRITE_ONCE() is paired with the READ_ONCE()
1197 * here, and another one in sock_rps_record_flow().
1198 */
1199 if (unlikely(READ_ONCE(sk->sk_rxhash) != skb->hash))
1200 WRITE_ONCE(sk->sk_rxhash, skb->hash);
1201#endif
1202}
1203
1204static inline void sock_rps_reset_rxhash(struct sock *sk)
1205{
1206#ifdef CONFIG_RPS
1207 /* Paired with READ_ONCE() in sock_rps_record_flow() */
1208 WRITE_ONCE(sk->sk_rxhash, 0);
1209#endif
1210}
1211
1212#define sk_wait_event(__sk, __timeo, __condition, __wait) \
1213 ({ int __rc, __dis = __sk->sk_disconnects; \
1214 release_sock(__sk); \
1215 __rc = __condition; \
1216 if (!__rc) { \
1217 *(__timeo) = wait_woken(__wait, \
1218 TASK_INTERRUPTIBLE, \
1219 *(__timeo)); \
1220 } \
1221 sched_annotate_sleep(); \
1222 lock_sock(__sk); \
1223 __rc = __dis == __sk->sk_disconnects ? __condition : -EPIPE; \
1224 __rc; \
1225 })
1226
1227int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
1228int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
1229void sk_stream_wait_close(struct sock *sk, long timeo_p);
1230int sk_stream_error(struct sock *sk, int flags, int err);
1231void sk_stream_kill_queues(struct sock *sk);
1232void sk_set_memalloc(struct sock *sk);
1233void sk_clear_memalloc(struct sock *sk);
1234
1235void __sk_flush_backlog(struct sock *sk);
1236
1237static inline bool sk_flush_backlog(struct sock *sk)
1238{
1239 if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
1240 __sk_flush_backlog(sk);
1241 return true;
1242 }
1243 return false;
1244}
1245
1246int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
1247
1248struct request_sock_ops;
1249struct timewait_sock_ops;
1250struct inet_hashinfo;
1251struct raw_hashinfo;
1252struct smc_hashinfo;
1253struct module;
1254struct sk_psock;
1255
1256/*
1257 * caches using SLAB_TYPESAFE_BY_RCU should let .next pointer from nulls nodes
1258 * un-modified. Special care is taken when initializing object to zero.
1259 */
1260static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1261{
1262 if (offsetof(struct sock, sk_node.next) != 0)
1263 memset(sk, 0, offsetof(struct sock, sk_node.next));
1264 memset(&sk->sk_node.pprev, 0,
1265 size - offsetof(struct sock, sk_node.pprev));
1266}
1267
1268struct proto_accept_arg {
1269 int flags;
1270 int err;
1271 int is_empty;
1272 bool kern;
1273};
1274
1275/* Networking protocol blocks we attach to sockets.
1276 * socket layer -> transport layer interface
1277 */
1278struct proto {
1279 void (*close)(struct sock *sk,
1280 long timeout);
1281 int (*pre_connect)(struct sock *sk,
1282 struct sockaddr_unsized *uaddr,
1283 int addr_len);
1284 int (*connect)(struct sock *sk,
1285 struct sockaddr_unsized *uaddr,
1286 int addr_len);
1287 int (*disconnect)(struct sock *sk, int flags);
1288
1289 struct sock * (*accept)(struct sock *sk,
1290 struct proto_accept_arg *arg);
1291
1292 int (*ioctl)(struct sock *sk, int cmd,
1293 int *karg);
1294 int (*init)(struct sock *sk);
1295 void (*destroy)(struct sock *sk);
1296 void (*shutdown)(struct sock *sk, int how);
1297 int (*setsockopt)(struct sock *sk, int level,
1298 int optname, sockptr_t optval,
1299 unsigned int optlen);
1300 int (*getsockopt)(struct sock *sk, int level,
1301 int optname, char __user *optval,
1302 int __user *option);
1303 void (*keepalive)(struct sock *sk, int valbool);
1304#ifdef CONFIG_COMPAT
1305 int (*compat_ioctl)(struct sock *sk,
1306 unsigned int cmd, unsigned long arg);
1307#endif
1308 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
1309 size_t len);
1310 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
1311 size_t len, int flags, int *addr_len);
1312 void (*splice_eof)(struct socket *sock);
1313 int (*bind)(struct sock *sk,
1314 struct sockaddr_unsized *addr, int addr_len);
1315 int (*bind_add)(struct sock *sk,
1316 struct sockaddr_unsized *addr, int addr_len);
1317
1318 int (*backlog_rcv) (struct sock *sk,
1319 struct sk_buff *skb);
1320 bool (*bpf_bypass_getsockopt)(int level,
1321 int optname);
1322
1323 void (*release_cb)(struct sock *sk);
1324
1325 /* Keeping track of sk's, looking them up, and port selection methods. */
1326 int (*hash)(struct sock *sk);
1327 void (*unhash)(struct sock *sk);
1328 void (*rehash)(struct sock *sk);
1329 int (*get_port)(struct sock *sk, unsigned short snum);
1330 void (*put_port)(struct sock *sk);
1331#ifdef CONFIG_BPF_SYSCALL
1332 int (*psock_update_sk_prot)(struct sock *sk,
1333 struct sk_psock *psock,
1334 bool restore);
1335#endif
1336
1337 /* Keeping track of sockets in use */
1338#ifdef CONFIG_PROC_FS
1339 unsigned int inuse_idx;
1340#endif
1341
1342 bool (*stream_memory_free)(const struct sock *sk, int wake);
1343 bool (*sock_is_readable)(struct sock *sk);
1344 /* Memory pressure */
1345 void (*enter_memory_pressure)(struct sock *sk);
1346 void (*leave_memory_pressure)(struct sock *sk);
1347 atomic_long_t *memory_allocated; /* Current allocated memory. */
1348 int __percpu *per_cpu_fw_alloc;
1349 struct percpu_counter *sockets_allocated; /* Current number of sockets. */
1350
1351 /*
1352 * Pressure flag: try to collapse.
1353 * Technical note: it is used by multiple contexts non atomically.
1354 * Make sure to use READ_ONCE()/WRITE_ONCE() for all reads/writes.
1355 * All the __sk_mem_schedule() is of this nature: accounting
1356 * is strict, actions are advisory and have some latency.
1357 */
1358 unsigned long *memory_pressure;
1359 long *sysctl_mem;
1360
1361 int *sysctl_wmem;
1362 int *sysctl_rmem;
1363 u32 sysctl_wmem_offset;
1364 u32 sysctl_rmem_offset;
1365
1366 int max_header;
1367 bool no_autobind;
1368
1369 struct kmem_cache *slab;
1370 unsigned int obj_size;
1371 unsigned int ipv6_pinfo_offset;
1372 slab_flags_t slab_flags;
1373 unsigned int useroffset; /* Usercopy region offset */
1374 unsigned int usersize; /* Usercopy region size */
1375
1376 struct request_sock_ops *rsk_prot;
1377 struct timewait_sock_ops *twsk_prot;
1378
1379 union {
1380 struct inet_hashinfo *hashinfo;
1381 struct udp_table *udp_table;
1382 struct raw_hashinfo *raw_hash;
1383 struct smc_hashinfo *smc_hash;
1384 } h;
1385
1386 struct module *owner;
1387
1388 char name[32];
1389
1390 struct list_head node;
1391 int (*diag_destroy)(struct sock *sk, int err);
1392} __randomize_layout;
1393
1394int proto_register(struct proto *prot, int alloc_slab);
1395void proto_unregister(struct proto *prot);
1396int sock_load_diag_module(int family, int protocol);
1397
1398INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));
1399
1400static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
1401{
1402 if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
1403 return false;
1404
1405 return sk->sk_prot->stream_memory_free ?
1406 INDIRECT_CALL_INET_1(sk->sk_prot->stream_memory_free,
1407 tcp_stream_memory_free, sk, wake) : true;
1408}
1409
1410static inline bool sk_stream_memory_free(const struct sock *sk)
1411{
1412 return __sk_stream_memory_free(sk, wake: 0);
1413}
1414
1415static inline bool __sk_stream_is_writeable(const struct sock *sk, int wake)
1416{
1417 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1418 __sk_stream_memory_free(sk, wake);
1419}
1420
1421static inline bool sk_stream_is_writeable(const struct sock *sk)
1422{
1423 return __sk_stream_is_writeable(sk, wake: 0);
1424}
1425
1426static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1427 struct cgroup *ancestor)
1428{
1429#ifdef CONFIG_SOCK_CGROUP_DATA
1430 return cgroup_is_descendant(cgrp: sock_cgroup_ptr(skcd: &sk->sk_cgrp_data),
1431 ancestor);
1432#else
1433 return -ENOTSUPP;
1434#endif
1435}
1436
1437#define SK_ALLOC_PERCPU_COUNTER_BATCH 16
1438
1439static inline void sk_sockets_allocated_dec(struct sock *sk)
1440{
1441 percpu_counter_add_batch(fbc: sk->sk_prot->sockets_allocated, amount: -1,
1442 SK_ALLOC_PERCPU_COUNTER_BATCH);
1443}
1444
1445static inline void sk_sockets_allocated_inc(struct sock *sk)
1446{
1447 percpu_counter_add_batch(fbc: sk->sk_prot->sockets_allocated, amount: 1,
1448 SK_ALLOC_PERCPU_COUNTER_BATCH);
1449}
1450
1451static inline u64
1452sk_sockets_allocated_read_positive(struct sock *sk)
1453{
1454 return percpu_counter_read_positive(fbc: sk->sk_prot->sockets_allocated);
1455}
1456
1457static inline int
1458proto_sockets_allocated_sum_positive(struct proto *prot)
1459{
1460 return percpu_counter_sum_positive(fbc: prot->sockets_allocated);
1461}
1462
1463#ifdef CONFIG_PROC_FS
1464#define PROTO_INUSE_NR 64 /* should be enough for the first time */
1465struct prot_inuse {
1466 int all;
1467 int val[PROTO_INUSE_NR];
1468};
1469
1470static inline void sock_prot_inuse_add(const struct net *net,
1471 const struct proto *prot, int val)
1472{
1473 this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val);
1474}
1475
1476static inline void sock_inuse_add(const struct net *net, int val)
1477{
1478 this_cpu_add(net->core.prot_inuse->all, val);
1479}
1480
1481int sock_prot_inuse_get(struct net *net, struct proto *proto);
1482int sock_inuse_get(struct net *net);
1483#else
1484static inline void sock_prot_inuse_add(const struct net *net,
1485 const struct proto *prot, int val)
1486{
1487}
1488
1489static inline void sock_inuse_add(const struct net *net, int val)
1490{
1491}
1492#endif
1493
1494
1495/* With per-bucket locks this operation is not-atomic, so that
1496 * this version is not worse.
1497 */
1498static inline int __sk_prot_rehash(struct sock *sk)
1499{
1500 sk->sk_prot->unhash(sk);
1501 return sk->sk_prot->hash(sk);
1502}
1503
1504/* About 10 seconds */
1505#define SOCK_DESTROY_TIME (10*HZ)
1506
1507/* Sockets 0-1023 can't be bound to unless you are superuser */
1508#define PROT_SOCK 1024
1509
1510#define SHUTDOWN_MASK 3
1511#define RCV_SHUTDOWN 1
1512#define SEND_SHUTDOWN 2
1513
1514#define SOCK_BINDADDR_LOCK 4
1515#define SOCK_BINDPORT_LOCK 8
1516/**
1517 * define SOCK_CONNECT_BIND - &sock->sk_userlocks flag for auto-bind at connect() time
1518 */
1519#define SOCK_CONNECT_BIND 16
1520
1521struct socket_alloc {
1522 struct socket socket;
1523 struct inode vfs_inode;
1524};
1525
1526static inline struct socket *SOCKET_I(struct inode *inode)
1527{
1528 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1529}
1530
1531static inline struct inode *SOCK_INODE(struct socket *socket)
1532{
1533 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1534}
1535
1536/*
1537 * Functions for memory accounting
1538 */
1539int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1540int __sk_mem_schedule(struct sock *sk, int size, int kind);
1541void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1542void __sk_mem_reclaim(struct sock *sk, int amount);
1543
1544#define SK_MEM_SEND 0
1545#define SK_MEM_RECV 1
1546
1547/* sysctl_mem values are in pages */
1548static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1549{
1550 return READ_ONCE(sk->sk_prot->sysctl_mem[index]);
1551}
1552
1553static inline int sk_mem_pages(int amt)
1554{
1555 return (amt + PAGE_SIZE - 1) >> PAGE_SHIFT;
1556}
1557
1558static inline bool sk_has_account(struct sock *sk)
1559{
1560 /* return true if protocol supports memory accounting */
1561 return !!sk->sk_prot->memory_allocated;
1562}
1563
1564static inline bool sk_wmem_schedule(struct sock *sk, int size)
1565{
1566 int delta;
1567
1568 if (!sk_has_account(sk))
1569 return true;
1570 delta = size - sk->sk_forward_alloc;
1571 return delta <= 0 || __sk_mem_schedule(sk, size: delta, SK_MEM_SEND);
1572}
1573
1574static inline bool
1575__sk_rmem_schedule(struct sock *sk, int size, bool pfmemalloc)
1576{
1577 int delta;
1578
1579 if (!sk_has_account(sk))
1580 return true;
1581 delta = size - sk->sk_forward_alloc;
1582 return delta <= 0 || __sk_mem_schedule(sk, size: delta, SK_MEM_RECV) ||
1583 pfmemalloc;
1584}
1585
1586static inline bool
1587sk_rmem_schedule(struct sock *sk, const struct sk_buff *skb, int size)
1588{
1589 return __sk_rmem_schedule(sk, size, pfmemalloc: skb_pfmemalloc(skb));
1590}
1591
1592static inline int sk_unused_reserved_mem(const struct sock *sk)
1593{
1594 int unused_mem;
1595
1596 if (likely(!sk->sk_reserved_mem))
1597 return 0;
1598
1599 unused_mem = sk->sk_reserved_mem - sk->sk_wmem_queued -
1600 atomic_read(v: &sk->sk_rmem_alloc);
1601
1602 return unused_mem > 0 ? unused_mem : 0;
1603}
1604
1605static inline void sk_mem_reclaim(struct sock *sk)
1606{
1607 int reclaimable;
1608
1609 if (!sk_has_account(sk))
1610 return;
1611
1612 reclaimable = sk->sk_forward_alloc - sk_unused_reserved_mem(sk);
1613
1614 if (reclaimable >= (int)PAGE_SIZE)
1615 __sk_mem_reclaim(sk, amount: reclaimable);
1616}
1617
1618static inline void sk_mem_reclaim_final(struct sock *sk)
1619{
1620 sk->sk_reserved_mem = 0;
1621 sk_mem_reclaim(sk);
1622}
1623
1624static inline void sk_mem_charge(struct sock *sk, int size)
1625{
1626 if (!sk_has_account(sk))
1627 return;
1628 sk_forward_alloc_add(sk, val: -size);
1629}
1630
1631static inline void sk_mem_uncharge(struct sock *sk, int size)
1632{
1633 if (!sk_has_account(sk))
1634 return;
1635 sk_forward_alloc_add(sk, val: size);
1636 sk_mem_reclaim(sk);
1637}
1638
1639void __sk_charge(struct sock *sk, gfp_t gfp);
1640
1641#if IS_ENABLED(CONFIG_PROVE_LOCKING) && IS_ENABLED(CONFIG_MODULES)
1642static inline void sk_owner_set(struct sock *sk, struct module *owner)
1643{
1644 __module_get(module: owner);
1645 sk->sk_owner = owner;
1646}
1647
1648static inline void sk_owner_clear(struct sock *sk)
1649{
1650 sk->sk_owner = NULL;
1651}
1652
1653static inline void sk_owner_put(struct sock *sk)
1654{
1655 module_put(module: sk->sk_owner);
1656}
1657#else
1658static inline void sk_owner_set(struct sock *sk, struct module *owner)
1659{
1660}
1661
1662static inline void sk_owner_clear(struct sock *sk)
1663{
1664}
1665
1666static inline void sk_owner_put(struct sock *sk)
1667{
1668}
1669#endif
1670/*
1671 * Macro so as to not evaluate some arguments when
1672 * lockdep is not enabled.
1673 *
1674 * Mark both the sk_lock and the sk_lock.slock as a
1675 * per-address-family lock class.
1676 */
1677#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1678do { \
1679 sk_owner_set(sk, THIS_MODULE); \
1680 sk->sk_lock.owned = 0; \
1681 init_waitqueue_head(&sk->sk_lock.wq); \
1682 spin_lock_init(&(sk)->sk_lock.slock); \
1683 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1684 sizeof((sk)->sk_lock)); \
1685 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1686 (skey), (sname)); \
1687 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1688} while (0)
1689
1690static inline bool lockdep_sock_is_held(const struct sock *sk)
1691{
1692 return lockdep_is_held(&sk->sk_lock) ||
1693 lockdep_is_held(&sk->sk_lock.slock);
1694}
1695
1696void lock_sock_nested(struct sock *sk, int subclass);
1697
1698static inline void lock_sock(struct sock *sk)
1699{
1700 lock_sock_nested(sk, subclass: 0);
1701}
1702
1703void __lock_sock(struct sock *sk);
1704void __release_sock(struct sock *sk);
1705void release_sock(struct sock *sk);
1706
1707/* BH context may only use the following locking interface. */
1708#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1709#define bh_lock_sock_nested(__sk) \
1710 spin_lock_nested(&((__sk)->sk_lock.slock), \
1711 SINGLE_DEPTH_NESTING)
1712#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1713
1714bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock);
1715
1716/**
1717 * lock_sock_fast - fast version of lock_sock
1718 * @sk: socket
1719 *
1720 * This version should be used for very small section, where process won't block
1721 * return false if fast path is taken:
1722 *
1723 * sk_lock.slock locked, owned = 0, BH disabled
1724 *
1725 * return true if slow path is taken:
1726 *
1727 * sk_lock.slock unlocked, owned = 1, BH enabled
1728 */
1729static inline bool lock_sock_fast(struct sock *sk)
1730{
1731 /* The sk_lock has mutex_lock() semantics here. */
1732 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
1733
1734 return __lock_sock_fast(sk);
1735}
1736
1737/* fast socket lock variant for caller already holding a [different] socket lock */
1738static inline bool lock_sock_fast_nested(struct sock *sk)
1739{
1740 mutex_acquire(&sk->sk_lock.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
1741
1742 return __lock_sock_fast(sk);
1743}
1744
1745/**
1746 * unlock_sock_fast - complement of lock_sock_fast
1747 * @sk: socket
1748 * @slow: slow mode
1749 *
1750 * fast unlock socket for user context.
1751 * If slow mode is on, we call regular release_sock()
1752 */
1753static inline void unlock_sock_fast(struct sock *sk, bool slow)
1754 __releases(&sk->sk_lock.slock)
1755{
1756 if (slow) {
1757 release_sock(sk);
1758 __release(&sk->sk_lock.slock);
1759 } else {
1760 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1761 spin_unlock_bh(lock: &sk->sk_lock.slock);
1762 }
1763}
1764
1765void sockopt_lock_sock(struct sock *sk);
1766void sockopt_release_sock(struct sock *sk);
1767bool sockopt_ns_capable(struct user_namespace *ns, int cap);
1768bool sockopt_capable(int cap);
1769
1770/* Used by processes to "lock" a socket state, so that
1771 * interrupts and bottom half handlers won't change it
1772 * from under us. It essentially blocks any incoming
1773 * packets, so that we won't get any new data or any
1774 * packets that change the state of the socket.
1775 *
1776 * While locked, BH processing will add new packets to
1777 * the backlog queue. This queue is processed by the
1778 * owner of the socket lock right before it is released.
1779 *
1780 * Since ~2.3.5 it is also exclusive sleep lock serializing
1781 * accesses from user process context.
1782 */
1783
1784static inline void sock_owned_by_me(const struct sock *sk)
1785{
1786#ifdef CONFIG_LOCKDEP
1787 WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1788#endif
1789}
1790
1791static inline void sock_not_owned_by_me(const struct sock *sk)
1792{
1793#ifdef CONFIG_LOCKDEP
1794 WARN_ON_ONCE(lockdep_sock_is_held(sk) && debug_locks);
1795#endif
1796}
1797
1798static inline bool sock_owned_by_user(const struct sock *sk)
1799{
1800 sock_owned_by_me(sk);
1801 return sk->sk_lock.owned;
1802}
1803
1804static inline bool sock_owned_by_user_nocheck(const struct sock *sk)
1805{
1806 return sk->sk_lock.owned;
1807}
1808
1809static inline void sock_release_ownership(struct sock *sk)
1810{
1811 DEBUG_NET_WARN_ON_ONCE(!sock_owned_by_user_nocheck(sk));
1812 sk->sk_lock.owned = 0;
1813
1814 /* The sk_lock has mutex_unlock() semantics: */
1815 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1816}
1817
1818/* no reclassification while locks are held */
1819static inline bool sock_allow_reclassification(const struct sock *csk)
1820{
1821 struct sock *sk = (struct sock *)csk;
1822
1823 return !sock_owned_by_user_nocheck(sk) &&
1824 !spin_is_locked(lock: &sk->sk_lock.slock);
1825}
1826
1827struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1828 struct proto *prot, int kern);
1829void sk_free(struct sock *sk);
1830void sk_net_refcnt_upgrade(struct sock *sk);
1831void sk_destruct(struct sock *sk);
1832struct sock *sk_clone(const struct sock *sk, const gfp_t priority, bool lock);
1833
1834static inline struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
1835{
1836 return sk_clone(sk, priority, lock: true);
1837}
1838
1839struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1840 gfp_t priority);
1841void __sock_wfree(struct sk_buff *skb);
1842void sock_wfree(struct sk_buff *skb);
1843struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1844 gfp_t priority);
1845void skb_orphan_partial(struct sk_buff *skb);
1846void sock_rfree(struct sk_buff *skb);
1847void sock_efree(struct sk_buff *skb);
1848#ifdef CONFIG_INET
1849void sock_edemux(struct sk_buff *skb);
1850void sock_pfree(struct sk_buff *skb);
1851
1852static inline void skb_set_owner_edemux(struct sk_buff *skb, struct sock *sk)
1853{
1854 skb_orphan(skb);
1855 if (refcount_inc_not_zero(r: &sk->sk_refcnt)) {
1856 skb->sk = sk;
1857 skb->destructor = sock_edemux;
1858 }
1859}
1860#else
1861#define sock_edemux sock_efree
1862#endif
1863
1864int sk_setsockopt(struct sock *sk, int level, int optname,
1865 sockptr_t optval, unsigned int optlen);
1866int sock_setsockopt(struct socket *sock, int level, int op,
1867 sockptr_t optval, unsigned int optlen);
1868int do_sock_setsockopt(struct socket *sock, bool compat, int level,
1869 int optname, sockptr_t optval, int optlen);
1870int do_sock_getsockopt(struct socket *sock, bool compat, int level,
1871 int optname, sockptr_t optval, sockptr_t optlen);
1872
1873int sk_getsockopt(struct sock *sk, int level, int optname,
1874 sockptr_t optval, sockptr_t optlen);
1875int sock_gettstamp(struct socket *sock, void __user *userstamp,
1876 bool timeval, bool time32);
1877struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1878 unsigned long data_len, int noblock,
1879 int *errcode, int max_page_order);
1880
1881static inline struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1882 unsigned long size,
1883 int noblock, int *errcode)
1884{
1885 return sock_alloc_send_pskb(sk, header_len: size, data_len: 0, noblock, errcode, max_page_order: 0);
1886}
1887
1888void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1889void *sock_kmemdup(struct sock *sk, const void *src,
1890 int size, gfp_t priority);
1891void sock_kfree_s(struct sock *sk, void *mem, int size);
1892void sock_kzfree_s(struct sock *sk, void *mem, int size);
1893void sk_send_sigurg(struct sock *sk);
1894
1895static inline void sock_replace_proto(struct sock *sk, struct proto *proto)
1896{
1897 if (sk->sk_socket)
1898 clear_bit(nr: SOCK_SUPPORT_ZC, addr: &sk->sk_socket->flags);
1899 WRITE_ONCE(sk->sk_prot, proto);
1900}
1901
1902struct sockcm_cookie {
1903 u64 transmit_time;
1904 u32 mark;
1905 u32 tsflags;
1906 u32 ts_opt_id;
1907 u32 priority;
1908 u32 dmabuf_id;
1909};
1910
1911static inline void sockcm_init(struct sockcm_cookie *sockc,
1912 const struct sock *sk)
1913{
1914 *sockc = (struct sockcm_cookie) {
1915 .mark = READ_ONCE(sk->sk_mark),
1916 .tsflags = READ_ONCE(sk->sk_tsflags),
1917 .priority = READ_ONCE(sk->sk_priority),
1918 };
1919}
1920
1921int __sock_cmsg_send(struct sock *sk, struct cmsghdr *cmsg,
1922 struct sockcm_cookie *sockc);
1923int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1924 struct sockcm_cookie *sockc);
1925
1926/*
1927 * Functions to fill in entries in struct proto_ops when a protocol
1928 * does not implement a particular function.
1929 */
1930int sock_no_bind(struct socket *sock, struct sockaddr_unsized *saddr, int len);
1931int sock_no_connect(struct socket *sock, struct sockaddr_unsized *saddr, int len, int flags);
1932int sock_no_socketpair(struct socket *, struct socket *);
1933int sock_no_accept(struct socket *, struct socket *, struct proto_accept_arg *);
1934int sock_no_getname(struct socket *, struct sockaddr *, int);
1935int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1936int sock_no_listen(struct socket *, int);
1937int sock_no_shutdown(struct socket *, int);
1938int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1939int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
1940int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1941int sock_no_mmap(struct file *file, struct socket *sock,
1942 struct vm_area_struct *vma);
1943
1944/*
1945 * Functions to fill in entries in struct proto_ops when a protocol
1946 * uses the inet style.
1947 */
1948int sock_common_getsockopt(struct socket *sock, int level, int optname,
1949 char __user *optval, int __user *optlen);
1950int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1951 int flags);
1952int sock_common_setsockopt(struct socket *sock, int level, int optname,
1953 sockptr_t optval, unsigned int optlen);
1954
1955void sk_common_release(struct sock *sk);
1956
1957/*
1958 * Default socket callbacks and setup code
1959 */
1960
1961/* Initialise core socket variables using an explicit uid. */
1962void sock_init_data_uid(struct socket *sock, struct sock *sk, kuid_t uid);
1963
1964/* Initialise core socket variables.
1965 * Assumes struct socket *sock is embedded in a struct socket_alloc.
1966 */
1967void sock_init_data(struct socket *sock, struct sock *sk);
1968
1969/*
1970 * Socket reference counting postulates.
1971 *
1972 * * Each user of socket SHOULD hold a reference count.
1973 * * Each access point to socket (an hash table bucket, reference from a list,
1974 * running timer, skb in flight MUST hold a reference count.
1975 * * When reference count hits 0, it means it will never increase back.
1976 * * When reference count hits 0, it means that no references from
1977 * outside exist to this socket and current process on current CPU
1978 * is last user and may/should destroy this socket.
1979 * * sk_free is called from any context: process, BH, IRQ. When
1980 * it is called, socket has no references from outside -> sk_free
1981 * may release descendant resources allocated by the socket, but
1982 * to the time when it is called, socket is NOT referenced by any
1983 * hash tables, lists etc.
1984 * * Packets, delivered from outside (from network or from another process)
1985 * and enqueued on receive/error queues SHOULD NOT grab reference count,
1986 * when they sit in queue. Otherwise, packets will leak to hole, when
1987 * socket is looked up by one cpu and unhasing is made by another CPU.
1988 * It is true for udp/raw, netlink (leak to receive and error queues), tcp
1989 * (leak to backlog). Packet socket does all the processing inside
1990 * BR_NETPROTO_LOCK, so that it has not this race condition. UNIX sockets
1991 * use separate SMP lock, so that they are prone too.
1992 */
1993
1994/* Ungrab socket and destroy it, if it was the last reference. */
1995static inline void sock_put(struct sock *sk)
1996{
1997 if (refcount_dec_and_test(r: &sk->sk_refcnt))
1998 sk_free(sk);
1999}
2000/* Generic version of sock_put(), dealing with all sockets
2001 * (TCP_TIMEWAIT, TCP_NEW_SYN_RECV, ESTABLISHED...)
2002 */
2003void sock_gen_put(struct sock *sk);
2004
2005int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
2006 unsigned int trim_cap, bool refcounted);
2007static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
2008 const int nested)
2009{
2010 return __sk_receive_skb(sk, skb, nested, trim_cap: 1, refcounted: true);
2011}
2012
2013static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
2014{
2015 /* sk_tx_queue_mapping accept only upto a 16-bit value */
2016 if (WARN_ON_ONCE((unsigned short)tx_queue >= USHRT_MAX))
2017 return;
2018 /* Paired with READ_ONCE() in sk_tx_queue_get() and
2019 * other WRITE_ONCE() because socket lock might be not held.
2020 */
2021 if (READ_ONCE(sk->sk_tx_queue_mapping) != tx_queue) {
2022 WRITE_ONCE(sk->sk_tx_queue_mapping, tx_queue);
2023 WRITE_ONCE(sk->sk_tx_queue_mapping_jiffies, jiffies);
2024 return;
2025 }
2026
2027 /* Refresh sk_tx_queue_mapping_jiffies if too old. */
2028 if (time_is_before_jiffies(READ_ONCE(sk->sk_tx_queue_mapping_jiffies) + HZ))
2029 WRITE_ONCE(sk->sk_tx_queue_mapping_jiffies, jiffies);
2030}
2031
2032#define NO_QUEUE_MAPPING USHRT_MAX
2033
2034static inline void sk_tx_queue_clear(struct sock *sk)
2035{
2036 /* Paired with READ_ONCE() in sk_tx_queue_get() and
2037 * other WRITE_ONCE() because socket lock might be not held.
2038 */
2039 WRITE_ONCE(sk->sk_tx_queue_mapping, NO_QUEUE_MAPPING);
2040}
2041
2042int sk_tx_queue_get(const struct sock *sk);
2043
2044static inline void __sk_rx_queue_set(struct sock *sk,
2045 const struct sk_buff *skb,
2046 bool force_set)
2047{
2048#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
2049 if (skb_rx_queue_recorded(skb)) {
2050 u16 rx_queue = skb_get_rx_queue(skb);
2051
2052 if (force_set ||
2053 unlikely(READ_ONCE(sk->sk_rx_queue_mapping) != rx_queue))
2054 WRITE_ONCE(sk->sk_rx_queue_mapping, rx_queue);
2055 }
2056#endif
2057}
2058
2059static inline void sk_rx_queue_set(struct sock *sk, const struct sk_buff *skb)
2060{
2061 __sk_rx_queue_set(sk, skb, force_set: true);
2062}
2063
2064static inline void sk_rx_queue_update(struct sock *sk, const struct sk_buff *skb)
2065{
2066 __sk_rx_queue_set(sk, skb, force_set: false);
2067}
2068
2069static inline void sk_rx_queue_clear(struct sock *sk)
2070{
2071#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
2072 WRITE_ONCE(sk->sk_rx_queue_mapping, NO_QUEUE_MAPPING);
2073#endif
2074}
2075
2076static inline int sk_rx_queue_get(const struct sock *sk)
2077{
2078#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
2079 if (sk) {
2080 int res = READ_ONCE(sk->sk_rx_queue_mapping);
2081
2082 if (res != NO_QUEUE_MAPPING)
2083 return res;
2084 }
2085#endif
2086
2087 return -1;
2088}
2089
2090static inline void sk_set_socket(struct sock *sk, struct socket *sock)
2091{
2092 sk->sk_socket = sock;
2093 if (sock) {
2094 WRITE_ONCE(sk->sk_uid, SOCK_INODE(sock)->i_uid);
2095 WRITE_ONCE(sk->sk_ino, SOCK_INODE(sock)->i_ino);
2096 } else {
2097 /* Note: sk_uid is unchanged. */
2098 WRITE_ONCE(sk->sk_ino, 0);
2099 }
2100}
2101
2102static inline wait_queue_head_t *sk_sleep(struct sock *sk)
2103{
2104 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
2105 return &rcu_dereference_raw(sk->sk_wq)->wait;
2106}
2107/* Detach socket from process context.
2108 * Announce socket dead, detach it from wait queue and inode.
2109 * Note that parent inode held reference count on this struct sock,
2110 * we do not release it in this function, because protocol
2111 * probably wants some additional cleanups or even continuing
2112 * to work with this socket (TCP).
2113 */
2114static inline void sock_orphan(struct sock *sk)
2115{
2116 write_lock_bh(&sk->sk_callback_lock);
2117 sock_set_flag(sk, flag: SOCK_DEAD);
2118 sk_set_socket(sk, NULL);
2119 sk->sk_wq = NULL;
2120 write_unlock_bh(&sk->sk_callback_lock);
2121}
2122
2123static inline void sock_graft(struct sock *sk, struct socket *parent)
2124{
2125 WARN_ON(parent->sk);
2126 write_lock_bh(&sk->sk_callback_lock);
2127 rcu_assign_pointer(sk->sk_wq, &parent->wq);
2128 parent->sk = sk;
2129 sk_set_socket(sk, sock: parent);
2130 security_sock_graft(sk, parent);
2131 write_unlock_bh(&sk->sk_callback_lock);
2132}
2133
2134static inline unsigned long sock_i_ino(const struct sock *sk)
2135{
2136 /* Paired with WRITE_ONCE() in sock_graft() and sock_orphan() */
2137 return READ_ONCE(sk->sk_ino);
2138}
2139
2140static inline kuid_t sk_uid(const struct sock *sk)
2141{
2142 /* Paired with WRITE_ONCE() in sockfs_setattr() */
2143 return READ_ONCE(sk->sk_uid);
2144}
2145
2146static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
2147{
2148 return sk ? sk_uid(sk) : make_kuid(from: net->user_ns, uid: 0);
2149}
2150
2151static inline u32 net_tx_rndhash(void)
2152{
2153 u32 v = get_random_u32();
2154
2155 return v ?: 1;
2156}
2157
2158static inline void sk_set_txhash(struct sock *sk)
2159{
2160 /* This pairs with READ_ONCE() in skb_set_hash_from_sk() */
2161 WRITE_ONCE(sk->sk_txhash, net_tx_rndhash());
2162}
2163
2164static inline bool sk_rethink_txhash(struct sock *sk)
2165{
2166 if (sk->sk_txhash && sk->sk_txrehash == SOCK_TXREHASH_ENABLED) {
2167 sk_set_txhash(sk);
2168 return true;
2169 }
2170 return false;
2171}
2172
2173static inline struct dst_entry *
2174__sk_dst_get(const struct sock *sk)
2175{
2176 return rcu_dereference_check(sk->sk_dst_cache,
2177 lockdep_sock_is_held(sk));
2178}
2179
2180static inline struct dst_entry *
2181sk_dst_get(const struct sock *sk)
2182{
2183 struct dst_entry *dst;
2184
2185 rcu_read_lock();
2186 dst = rcu_dereference(sk->sk_dst_cache);
2187 if (dst && !rcuref_get(ref: &dst->__rcuref))
2188 dst = NULL;
2189 rcu_read_unlock();
2190 return dst;
2191}
2192
2193static inline void __dst_negative_advice(struct sock *sk)
2194{
2195 struct dst_entry *dst = __sk_dst_get(sk);
2196
2197 if (dst && dst->ops->negative_advice)
2198 dst->ops->negative_advice(sk, dst);
2199}
2200
2201static inline void dst_negative_advice(struct sock *sk)
2202{
2203 sk_rethink_txhash(sk);
2204 __dst_negative_advice(sk);
2205}
2206
2207static inline void
2208__sk_dst_set(struct sock *sk, struct dst_entry *dst)
2209{
2210 struct dst_entry *old_dst;
2211
2212 sk_tx_queue_clear(sk);
2213 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2214 old_dst = rcu_dereference_protected(sk->sk_dst_cache,
2215 lockdep_sock_is_held(sk));
2216 rcu_assign_pointer(sk->sk_dst_cache, dst);
2217 dst_release(dst: old_dst);
2218}
2219
2220static inline void
2221sk_dst_set(struct sock *sk, struct dst_entry *dst)
2222{
2223 struct dst_entry *old_dst;
2224
2225 sk_tx_queue_clear(sk);
2226 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2227 old_dst = unrcu_pointer(xchg(&sk->sk_dst_cache, RCU_INITIALIZER(dst)));
2228 dst_release(dst: old_dst);
2229}
2230
2231static inline void
2232__sk_dst_reset(struct sock *sk)
2233{
2234 __sk_dst_set(sk, NULL);
2235}
2236
2237static inline void
2238sk_dst_reset(struct sock *sk)
2239{
2240 sk_dst_set(sk, NULL);
2241}
2242
2243struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
2244
2245struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
2246
2247static inline void sk_dst_confirm(struct sock *sk)
2248{
2249 if (!READ_ONCE(sk->sk_dst_pending_confirm))
2250 WRITE_ONCE(sk->sk_dst_pending_confirm, 1);
2251}
2252
2253static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
2254{
2255 if (skb_get_dst_pending_confirm(skb)) {
2256 struct sock *sk = skb->sk;
2257
2258 if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
2259 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2260 neigh_confirm(n);
2261 }
2262}
2263
2264bool sk_mc_loop(const struct sock *sk);
2265
2266static inline bool sk_can_gso(const struct sock *sk)
2267{
2268 return net_gso_ok(features: sk->sk_route_caps, gso_type: sk->sk_gso_type);
2269}
2270
2271void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
2272
2273static inline void sk_gso_disable(struct sock *sk)
2274{
2275 sk->sk_gso_disabled = 1;
2276 sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2277}
2278
2279static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
2280 struct iov_iter *from, char *to,
2281 int copy, int offset)
2282{
2283 if (skb->ip_summed == CHECKSUM_NONE) {
2284 __wsum csum = 0;
2285 if (!csum_and_copy_from_iter_full(addr: to, bytes: copy, csum: &csum, i: from))
2286 return -EFAULT;
2287 skb->csum = csum_block_add(csum: skb->csum, csum2: csum, offset);
2288 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
2289 if (!copy_from_iter_full_nocache(addr: to, bytes: copy, i: from))
2290 return -EFAULT;
2291 } else if (!copy_from_iter_full(addr: to, bytes: copy, i: from))
2292 return -EFAULT;
2293
2294 return 0;
2295}
2296
2297static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
2298 struct iov_iter *from, int copy)
2299{
2300 int err, offset = skb->len;
2301
2302 err = skb_do_copy_data_nocache(sk, skb, from, to: skb_put(skb, len: copy),
2303 copy, offset);
2304 if (err)
2305 __skb_trim(skb, len: offset);
2306
2307 return err;
2308}
2309
2310static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
2311 struct sk_buff *skb,
2312 struct page *page,
2313 int off, int copy)
2314{
2315 int err;
2316
2317 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
2318 copy, offset: skb->len);
2319 if (err)
2320 return err;
2321
2322 skb_len_add(skb, delta: copy);
2323 sk_wmem_queued_add(sk, val: copy);
2324 sk_mem_charge(sk, size: copy);
2325 return 0;
2326}
2327
2328#define SK_WMEM_ALLOC_BIAS 1
2329/**
2330 * sk_wmem_alloc_get - returns write allocations
2331 * @sk: socket
2332 *
2333 * Return: sk_wmem_alloc minus initial offset of one
2334 */
2335static inline int sk_wmem_alloc_get(const struct sock *sk)
2336{
2337 return refcount_read(r: &sk->sk_wmem_alloc) - SK_WMEM_ALLOC_BIAS;
2338}
2339
2340/**
2341 * sk_rmem_alloc_get - returns read allocations
2342 * @sk: socket
2343 *
2344 * Return: sk_rmem_alloc
2345 */
2346static inline int sk_rmem_alloc_get(const struct sock *sk)
2347{
2348 return atomic_read(v: &sk->sk_rmem_alloc);
2349}
2350
2351/**
2352 * sk_has_allocations - check if allocations are outstanding
2353 * @sk: socket
2354 *
2355 * Return: true if socket has write or read allocations
2356 */
2357static inline bool sk_has_allocations(const struct sock *sk)
2358{
2359 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
2360}
2361
2362/**
2363 * skwq_has_sleeper - check if there are any waiting processes
2364 * @wq: struct socket_wq
2365 *
2366 * Return: true if socket_wq has waiting processes
2367 *
2368 * The purpose of the skwq_has_sleeper and sock_poll_wait is to wrap the memory
2369 * barrier call. They were added due to the race found within the tcp code.
2370 *
2371 * Consider following tcp code paths::
2372 *
2373 * CPU1 CPU2
2374 * sys_select receive packet
2375 * ... ...
2376 * __add_wait_queue update tp->rcv_nxt
2377 * ... ...
2378 * tp->rcv_nxt check sock_def_readable
2379 * ... {
2380 * schedule rcu_read_lock();
2381 * wq = rcu_dereference(sk->sk_wq);
2382 * if (wq && waitqueue_active(&wq->wait))
2383 * wake_up_interruptible(&wq->wait)
2384 * ...
2385 * }
2386 *
2387 * The race for tcp fires when the __add_wait_queue changes done by CPU1 stay
2388 * in its cache, and so does the tp->rcv_nxt update on CPU2 side. The CPU1
2389 * could then endup calling schedule and sleep forever if there are no more
2390 * data on the socket.
2391 *
2392 */
2393static inline bool skwq_has_sleeper(struct socket_wq *wq)
2394{
2395 return wq && wq_has_sleeper(wq_head: &wq->wait);
2396}
2397
2398/**
2399 * sock_poll_wait - wrapper for the poll_wait call.
2400 * @filp: file
2401 * @sock: socket to wait on
2402 * @p: poll_table
2403 *
2404 * See the comments in the wq_has_sleeper function.
2405 */
2406static inline void sock_poll_wait(struct file *filp, struct socket *sock,
2407 poll_table *p)
2408{
2409 /* Provides a barrier we need to be sure we are in sync
2410 * with the socket flags modification.
2411 *
2412 * This memory barrier is paired in the wq_has_sleeper.
2413 */
2414 poll_wait(filp, wait_address: &sock->wq.wait, p);
2415}
2416
2417static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2418{
2419 /* This pairs with WRITE_ONCE() in sk_set_txhash() */
2420 u32 txhash = READ_ONCE(sk->sk_txhash);
2421
2422 if (txhash) {
2423 skb->l4_hash = 1;
2424 skb->hash = txhash;
2425 }
2426}
2427
2428void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2429
2430/*
2431 * Queue a received datagram if it will fit. Stream and sequenced
2432 * protocols can't normally use this as they need to fit buffers in
2433 * and play with them.
2434 *
2435 * Inlined as it's very short and called for pretty much every
2436 * packet ever received.
2437 */
2438static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2439{
2440 skb_orphan(skb);
2441 skb->sk = sk;
2442 skb->destructor = sock_rfree;
2443 atomic_add(i: skb->truesize, v: &sk->sk_rmem_alloc);
2444 sk_mem_charge(sk, size: skb->truesize);
2445}
2446
2447static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struct sock *sk)
2448{
2449 if (sk && refcount_inc_not_zero(r: &sk->sk_refcnt)) {
2450 skb_orphan(skb);
2451 skb->destructor = sock_efree;
2452 skb->sk = sk;
2453 return true;
2454 }
2455 return false;
2456}
2457
2458static inline struct sk_buff *skb_clone_and_charge_r(struct sk_buff *skb, struct sock *sk)
2459{
2460 skb = skb_clone(skb, priority: sk_gfp_mask(sk, GFP_ATOMIC));
2461 if (skb) {
2462 if (sk_rmem_schedule(sk, skb, size: skb->truesize)) {
2463 skb_set_owner_r(skb, sk);
2464 return skb;
2465 }
2466 __kfree_skb(skb);
2467 }
2468 return NULL;
2469}
2470
2471static inline void skb_prepare_for_gro(struct sk_buff *skb)
2472{
2473 if (skb->destructor != sock_wfree) {
2474 skb_orphan(skb);
2475 return;
2476 }
2477 skb->slow_gro = 1;
2478}
2479
2480void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2481 unsigned long expires);
2482
2483void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2484
2485void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer);
2486
2487int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
2488 struct sk_buff *skb, unsigned int flags,
2489 void (*destructor)(struct sock *sk,
2490 struct sk_buff *skb));
2491int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2492
2493int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb,
2494 enum skb_drop_reason *reason);
2495
2496static inline int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
2497{
2498 return sock_queue_rcv_skb_reason(sk, skb, NULL);
2499}
2500
2501int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2502struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2503
2504/*
2505 * Recover an error report and clear atomically
2506 */
2507
2508static inline int sock_error(struct sock *sk)
2509{
2510 int err;
2511
2512 /* Avoid an atomic operation for the common case.
2513 * This is racy since another cpu/thread can change sk_err under us.
2514 */
2515 if (likely(data_race(!sk->sk_err)))
2516 return 0;
2517
2518 err = xchg(&sk->sk_err, 0);
2519 return -err;
2520}
2521
2522void sk_error_report(struct sock *sk);
2523
2524static inline unsigned long sock_wspace(struct sock *sk)
2525{
2526 int amt = 0;
2527
2528 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2529 amt = sk->sk_sndbuf - refcount_read(r: &sk->sk_wmem_alloc);
2530 if (amt < 0)
2531 amt = 0;
2532 }
2533 return amt;
2534}
2535
2536/* Note:
2537 * We use sk->sk_wq_raw, from contexts knowing this
2538 * pointer is not NULL and cannot disappear/change.
2539 */
2540static inline void sk_set_bit(int nr, struct sock *sk)
2541{
2542 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2543 !sock_flag(sk, flag: SOCK_FASYNC))
2544 return;
2545
2546 set_bit(nr, addr: &sk->sk_wq_raw->flags);
2547}
2548
2549static inline void sk_clear_bit(int nr, struct sock *sk)
2550{
2551 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2552 !sock_flag(sk, flag: SOCK_FASYNC))
2553 return;
2554
2555 clear_bit(nr, addr: &sk->sk_wq_raw->flags);
2556}
2557
2558static inline void sk_wake_async(const struct sock *sk, int how, int band)
2559{
2560 if (sock_flag(sk, flag: SOCK_FASYNC)) {
2561 rcu_read_lock();
2562 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2563 rcu_read_unlock();
2564 }
2565}
2566
2567static inline void sk_wake_async_rcu(const struct sock *sk, int how, int band)
2568{
2569 if (unlikely(sock_flag(sk, SOCK_FASYNC)))
2570 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2571}
2572
2573/* Since sk_{r,w}mem_alloc sums skb->truesize, even a small frame might
2574 * need sizeof(sk_buff) + MTU + padding, unless net driver perform copybreak.
2575 * Note: for send buffers, TCP works better if we can build two skbs at
2576 * minimum.
2577 */
2578#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2579
2580#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2581#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2582
2583static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2584{
2585 u32 val;
2586
2587 if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
2588 return;
2589
2590 val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2591 val = max_t(u32, val, sk_unused_reserved_mem(sk));
2592
2593 WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
2594}
2595
2596/**
2597 * sk_page_frag - return an appropriate page_frag
2598 * @sk: socket
2599 *
2600 * Use the per task page_frag instead of the per socket one for
2601 * optimization when we know that we're in process context and own
2602 * everything that's associated with %current.
2603 *
2604 * Both direct reclaim and page faults can nest inside other
2605 * socket operations and end up recursing into sk_page_frag()
2606 * while it's already in use: explicitly avoid task page_frag
2607 * when users disable sk_use_task_frag.
2608 *
2609 * Return: a per task page_frag if context allows that,
2610 * otherwise a per socket one.
2611 */
2612static inline struct page_frag *sk_page_frag(struct sock *sk)
2613{
2614 if (sk->sk_use_task_frag)
2615 return &current->task_frag;
2616
2617 return &sk->sk_frag;
2618}
2619
2620bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2621
2622static inline bool __sock_writeable(const struct sock *sk, int wmem_alloc)
2623{
2624 return wmem_alloc < (READ_ONCE(sk->sk_sndbuf) >> 1);
2625}
2626/*
2627 * Default write policy as shown to user space via poll/select/SIGIO
2628 */
2629static inline bool sock_writeable(const struct sock *sk)
2630{
2631 return __sock_writeable(sk, wmem_alloc: refcount_read(r: &sk->sk_wmem_alloc));
2632}
2633
2634static inline gfp_t gfp_any(void)
2635{
2636 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2637}
2638
2639static inline gfp_t gfp_memcg_charge(void)
2640{
2641 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2642}
2643
2644#ifdef CONFIG_MEMCG
2645static inline struct mem_cgroup *mem_cgroup_from_sk(const struct sock *sk)
2646{
2647 return sk->sk_memcg;
2648}
2649
2650static inline bool mem_cgroup_sk_enabled(const struct sock *sk)
2651{
2652 return mem_cgroup_sockets_enabled && mem_cgroup_from_sk(sk);
2653}
2654
2655static inline bool mem_cgroup_sk_under_memory_pressure(const struct sock *sk)
2656{
2657 struct mem_cgroup *memcg = mem_cgroup_from_sk(sk);
2658
2659#ifdef CONFIG_MEMCG_V1
2660 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2661 return !!memcg->tcpmem_pressure;
2662#endif /* CONFIG_MEMCG_V1 */
2663
2664 do {
2665 if (time_before64(get_jiffies_64(),
2666 mem_cgroup_get_socket_pressure(memcg))) {
2667 memcg_memory_event(memcg: mem_cgroup_from_sk(sk),
2668 event: MEMCG_SOCK_THROTTLED);
2669 return true;
2670 }
2671 } while ((memcg = parent_mem_cgroup(memcg)));
2672
2673 return false;
2674}
2675#else
2676static inline struct mem_cgroup *mem_cgroup_from_sk(const struct sock *sk)
2677{
2678 return NULL;
2679}
2680
2681static inline bool mem_cgroup_sk_enabled(const struct sock *sk)
2682{
2683 return false;
2684}
2685
2686static inline bool mem_cgroup_sk_under_memory_pressure(const struct sock *sk)
2687{
2688 return false;
2689}
2690#endif
2691
2692static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2693{
2694 return noblock ? 0 : READ_ONCE(sk->sk_rcvtimeo);
2695}
2696
2697static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2698{
2699 return noblock ? 0 : READ_ONCE(sk->sk_sndtimeo);
2700}
2701
2702static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2703{
2704 int v = waitall ? len : min_t(int, READ_ONCE(sk->sk_rcvlowat), len);
2705
2706 return v ?: 1;
2707}
2708
2709/* Alas, with timeout socket operations are not restartable.
2710 * Compare this to poll().
2711 */
2712static inline int sock_intr_errno(long timeo)
2713{
2714 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2715}
2716
2717struct sock_skb_cb {
2718 u32 dropcount;
2719};
2720
2721/* Store sock_skb_cb at the end of skb->cb[] so protocol families
2722 * using skb->cb[] would keep using it directly and utilize its
2723 * alignment guarantee.
2724 */
2725#define SOCK_SKB_CB_OFFSET (sizeof_field(struct sk_buff, cb) - \
2726 sizeof(struct sock_skb_cb))
2727
2728#define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2729 SOCK_SKB_CB_OFFSET))
2730
2731#define sock_skb_cb_check_size(size) \
2732 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2733
2734static inline void sk_drops_add(struct sock *sk, int segs)
2735{
2736 struct numa_drop_counters *ndc = sk->sk_drop_counters;
2737
2738 if (ndc)
2739 numa_drop_add(ndc, val: segs);
2740 else
2741 atomic_add(i: segs, v: &sk->sk_drops);
2742}
2743
2744static inline void sk_drops_inc(struct sock *sk)
2745{
2746 sk_drops_add(sk, segs: 1);
2747}
2748
2749static inline int sk_drops_read(const struct sock *sk)
2750{
2751 const struct numa_drop_counters *ndc = sk->sk_drop_counters;
2752
2753 if (ndc) {
2754 DEBUG_NET_WARN_ON_ONCE(atomic_read(&sk->sk_drops));
2755 return numa_drop_read(ndc);
2756 }
2757 return atomic_read(v: &sk->sk_drops);
2758}
2759
2760static inline void sk_drops_reset(struct sock *sk)
2761{
2762 struct numa_drop_counters *ndc = sk->sk_drop_counters;
2763
2764 if (ndc)
2765 numa_drop_reset(ndc);
2766 atomic_set(v: &sk->sk_drops, i: 0);
2767}
2768
2769static inline void
2770sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2771{
2772 SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, flag: SOCK_RXQ_OVFL) ?
2773 sk_drops_read(sk) : 0;
2774}
2775
2776static inline void sk_drops_skbadd(struct sock *sk, const struct sk_buff *skb)
2777{
2778 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2779
2780 sk_drops_add(sk, segs);
2781}
2782
2783static inline ktime_t sock_read_timestamp(struct sock *sk)
2784{
2785#if BITS_PER_LONG==32
2786 unsigned int seq;
2787 ktime_t kt;
2788
2789 do {
2790 seq = read_seqbegin(&sk->sk_stamp_seq);
2791 kt = sk->sk_stamp;
2792 } while (read_seqretry(&sk->sk_stamp_seq, seq));
2793
2794 return kt;
2795#else
2796 return READ_ONCE(sk->sk_stamp);
2797#endif
2798}
2799
2800static inline void sock_write_timestamp(struct sock *sk, ktime_t kt)
2801{
2802#if BITS_PER_LONG==32
2803 write_seqlock(&sk->sk_stamp_seq);
2804 sk->sk_stamp = kt;
2805 write_sequnlock(&sk->sk_stamp_seq);
2806#else
2807 WRITE_ONCE(sk->sk_stamp, kt);
2808#endif
2809}
2810
2811void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2812 struct sk_buff *skb);
2813void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2814 struct sk_buff *skb);
2815
2816bool skb_has_tx_timestamp(struct sk_buff *skb, const struct sock *sk);
2817int skb_get_tx_timestamp(struct sk_buff *skb, struct sock *sk,
2818 struct timespec64 *ts);
2819
2820static inline void
2821sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2822{
2823 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2824 u32 tsflags = READ_ONCE(sk->sk_tsflags);
2825 ktime_t kt = skb->tstamp;
2826 /*
2827 * generate control messages if
2828 * - receive time stamping in software requested
2829 * - software time stamp available and wanted
2830 * - hardware time stamps available and wanted
2831 */
2832 if (sock_flag(sk, flag: SOCK_RCVTSTAMP) ||
2833 (tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2834 (kt && tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2835 (hwtstamps->hwtstamp &&
2836 (tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2837 __sock_recv_timestamp(msg, sk, skb);
2838 else
2839 sock_write_timestamp(sk, kt);
2840
2841 if (sock_flag(sk, flag: SOCK_WIFI_STATUS) && skb_wifi_acked_valid(skb))
2842 __sock_recv_wifi_status(msg, sk, skb);
2843}
2844
2845void __sock_recv_cmsgs(struct msghdr *msg, struct sock *sk,
2846 struct sk_buff *skb);
2847
2848#define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
2849static inline void sock_recv_cmsgs(struct msghdr *msg, struct sock *sk,
2850 struct sk_buff *skb)
2851{
2852#define FLAGS_RECV_CMSGS ((1UL << SOCK_RXQ_OVFL) | \
2853 (1UL << SOCK_RCVTSTAMP) | \
2854 (1UL << SOCK_RCVMARK) | \
2855 (1UL << SOCK_RCVPRIORITY) | \
2856 (1UL << SOCK_TIMESTAMPING_ANY))
2857#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2858 SOF_TIMESTAMPING_RAW_HARDWARE)
2859
2860 if (READ_ONCE(sk->sk_flags) & FLAGS_RECV_CMSGS)
2861 __sock_recv_cmsgs(msg, sk, skb);
2862 else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2863 sock_write_timestamp(sk, kt: skb->tstamp);
2864 else if (unlikely(sock_read_timestamp(sk) == SK_DEFAULT_STAMP))
2865 sock_write_timestamp(sk, kt: 0);
2866}
2867
2868void __sock_tx_timestamp(__u32 tsflags, __u8 *tx_flags);
2869
2870/**
2871 * _sock_tx_timestamp - checks whether the outgoing packet is to be time stamped
2872 * @sk: socket sending this packet
2873 * @sockc: pointer to socket cmsg cookie to get timestamping info
2874 * @tx_flags: completed with instructions for time stamping
2875 * @tskey: filled in with next sk_tskey (not for TCP, which uses seqno)
2876 *
2877 * Note: callers should take care of initial ``*tx_flags`` value (usually 0)
2878 */
2879static inline void _sock_tx_timestamp(struct sock *sk,
2880 const struct sockcm_cookie *sockc,
2881 __u8 *tx_flags, __u32 *tskey)
2882{
2883 __u32 tsflags = sockc->tsflags;
2884
2885 if (unlikely(tsflags)) {
2886 __sock_tx_timestamp(tsflags, tx_flags);
2887 if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
2888 tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK) {
2889 if (tsflags & SOCKCM_FLAG_TS_OPT_ID)
2890 *tskey = sockc->ts_opt_id;
2891 else
2892 *tskey = atomic_inc_return(v: &sk->sk_tskey) - 1;
2893 }
2894 }
2895}
2896
2897static inline void sock_tx_timestamp(struct sock *sk,
2898 const struct sockcm_cookie *sockc,
2899 __u8 *tx_flags)
2900{
2901 _sock_tx_timestamp(sk, sockc, tx_flags, NULL);
2902}
2903
2904static inline void skb_setup_tx_timestamp(struct sk_buff *skb,
2905 const struct sockcm_cookie *sockc)
2906{
2907 _sock_tx_timestamp(sk: skb->sk, sockc, tx_flags: &skb_shinfo(skb)->tx_flags,
2908 tskey: &skb_shinfo(skb)->tskey);
2909}
2910
2911static inline bool sk_is_inet(const struct sock *sk)
2912{
2913 int family = READ_ONCE(sk->sk_family);
2914
2915 return family == AF_INET || family == AF_INET6;
2916}
2917
2918static inline bool sk_is_tcp(const struct sock *sk)
2919{
2920 return sk_is_inet(sk) &&
2921 sk->sk_type == SOCK_STREAM &&
2922 sk->sk_protocol == IPPROTO_TCP;
2923}
2924
2925static inline bool sk_is_udp(const struct sock *sk)
2926{
2927 return sk_is_inet(sk) &&
2928 sk->sk_type == SOCK_DGRAM &&
2929 sk->sk_protocol == IPPROTO_UDP;
2930}
2931
2932static inline bool sk_is_unix(const struct sock *sk)
2933{
2934 return sk->sk_family == AF_UNIX;
2935}
2936
2937static inline bool sk_is_stream_unix(const struct sock *sk)
2938{
2939 return sk_is_unix(sk) && sk->sk_type == SOCK_STREAM;
2940}
2941
2942static inline bool sk_is_vsock(const struct sock *sk)
2943{
2944 return sk->sk_family == AF_VSOCK;
2945}
2946
2947static inline bool sk_may_scm_recv(const struct sock *sk)
2948{
2949 return (IS_ENABLED(CONFIG_UNIX) && sk->sk_family == AF_UNIX) ||
2950 sk->sk_family == AF_NETLINK ||
2951 (IS_ENABLED(CONFIG_BT) && sk->sk_family == AF_BLUETOOTH);
2952}
2953
2954/**
2955 * sk_eat_skb - Release a skb if it is no longer needed
2956 * @sk: socket to eat this skb from
2957 * @skb: socket buffer to eat
2958 *
2959 * This routine must be called with interrupts disabled or with the socket
2960 * locked so that the sk_buff queue operation is ok.
2961*/
2962static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2963{
2964 __skb_unlink(skb, list: &sk->sk_receive_queue);
2965 __kfree_skb(skb);
2966}
2967
2968static inline bool
2969skb_sk_is_prefetched(struct sk_buff *skb)
2970{
2971#ifdef CONFIG_INET
2972 return skb->destructor == sock_pfree;
2973#else
2974 return false;
2975#endif /* CONFIG_INET */
2976}
2977
2978/* This helper checks if a socket is a full socket,
2979 * ie _not_ a timewait or request socket.
2980 */
2981static inline bool sk_fullsock(const struct sock *sk)
2982{
2983 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2984}
2985
2986static inline bool
2987sk_is_refcounted(struct sock *sk)
2988{
2989 /* Only full sockets have sk->sk_flags. */
2990 return !sk_fullsock(sk) || !sock_flag(sk, flag: SOCK_RCU_FREE);
2991}
2992
2993static inline bool
2994sk_requests_wifi_status(struct sock *sk)
2995{
2996 return sk && sk_fullsock(sk) && sock_flag(sk, flag: SOCK_WIFI_STATUS);
2997}
2998
2999/* This helper checks if a socket is a LISTEN or NEW_SYN_RECV
3000 * SYNACK messages can be attached to either ones (depending on SYNCOOKIE)
3001 */
3002static inline bool sk_listener(const struct sock *sk)
3003{
3004 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
3005}
3006
3007/* This helper checks if a socket is a LISTEN or NEW_SYN_RECV or TIME_WAIT
3008 * TCP SYNACK messages can be attached to LISTEN or NEW_SYN_RECV (depending on SYNCOOKIE)
3009 * TCP RST and ACK can be attached to TIME_WAIT.
3010 */
3011static inline bool sk_listener_or_tw(const struct sock *sk)
3012{
3013 return (1 << READ_ONCE(sk->sk_state)) &
3014 (TCPF_LISTEN | TCPF_NEW_SYN_RECV | TCPF_TIME_WAIT);
3015}
3016
3017void sock_enable_timestamp(struct sock *sk, enum sock_flags flag);
3018int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
3019 int type);
3020
3021bool sk_ns_capable(const struct sock *sk,
3022 struct user_namespace *user_ns, int cap);
3023bool sk_capable(const struct sock *sk, int cap);
3024bool sk_net_capable(const struct sock *sk, int cap);
3025
3026void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
3027
3028/* Take into consideration the size of the struct sk_buff overhead in the
3029 * determination of these values, since that is non-constant across
3030 * platforms. This makes socket queueing behavior and performance
3031 * not depend upon such differences.
3032 */
3033#define _SK_MEM_PACKETS 256
3034#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
3035#define SK_WMEM_DEFAULT (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
3036#define SK_RMEM_DEFAULT (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
3037
3038extern __u32 sysctl_wmem_max;
3039extern __u32 sysctl_rmem_max;
3040
3041extern __u32 sysctl_wmem_default;
3042extern __u32 sysctl_rmem_default;
3043
3044#define SKB_FRAG_PAGE_ORDER get_order(32768)
3045DECLARE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
3046
3047static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
3048{
3049 /* Does this proto have per netns sysctl_wmem ? */
3050 if (proto->sysctl_wmem_offset)
3051 return READ_ONCE(*(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset));
3052
3053 return READ_ONCE(*proto->sysctl_wmem);
3054}
3055
3056static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
3057{
3058 /* Does this proto have per netns sysctl_rmem ? */
3059 if (proto->sysctl_rmem_offset)
3060 return READ_ONCE(*(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset));
3061
3062 return READ_ONCE(*proto->sysctl_rmem);
3063}
3064
3065/* Default TCP Small queue budget is ~1 ms of data (1sec >> 10)
3066 * Some wifi drivers need to tweak it to get more chunks.
3067 * They can use this helper from their ndo_start_xmit()
3068 */
3069static inline void sk_pacing_shift_update(struct sock *sk, int val)
3070{
3071 if (!sk || !sk_fullsock(sk) || READ_ONCE(sk->sk_pacing_shift) == val)
3072 return;
3073 WRITE_ONCE(sk->sk_pacing_shift, val);
3074}
3075
3076/* if a socket is bound to a device, check that the given device
3077 * index is either the same or that the socket is bound to an L3
3078 * master device and the given device index is also enslaved to
3079 * that L3 master
3080 */
3081static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
3082{
3083 int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
3084 int mdif;
3085
3086 if (!bound_dev_if || bound_dev_if == dif)
3087 return true;
3088
3089 mdif = l3mdev_master_ifindex_by_index(net: sock_net(sk), ifindex: dif);
3090 if (mdif && mdif == bound_dev_if)
3091 return true;
3092
3093 return false;
3094}
3095
3096void sock_def_readable(struct sock *sk);
3097
3098int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk);
3099void sock_set_timestamp(struct sock *sk, int optname, bool valbool);
3100int sock_set_timestamping(struct sock *sk, int optname,
3101 struct so_timestamping timestamping);
3102
3103#if defined(CONFIG_CGROUP_BPF)
3104void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op);
3105#else
3106static inline void bpf_skops_tx_timestamping(struct sock *sk, struct sk_buff *skb, int op)
3107{
3108}
3109#endif
3110void sock_no_linger(struct sock *sk);
3111void sock_set_keepalive(struct sock *sk);
3112void sock_set_priority(struct sock *sk, u32 priority);
3113void sock_set_rcvbuf(struct sock *sk, int val);
3114void sock_set_mark(struct sock *sk, u32 val);
3115void sock_set_reuseaddr(struct sock *sk);
3116void sock_set_reuseport(struct sock *sk);
3117void sock_set_sndtimeo(struct sock *sk, s64 secs);
3118
3119int sock_bind_add(struct sock *sk, struct sockaddr_unsized *addr, int addr_len);
3120
3121int sock_get_timeout(long timeo, void *optval, bool old_timeval);
3122int sock_copy_user_timeval(struct __kernel_sock_timeval *tv,
3123 sockptr_t optval, int optlen, bool old_timeval);
3124
3125int sock_ioctl_inout(struct sock *sk, unsigned int cmd,
3126 void __user *arg, void *karg, size_t size);
3127int sk_ioctl(struct sock *sk, unsigned int cmd, void __user *arg);
3128static inline bool sk_is_readable(struct sock *sk)
3129{
3130 const struct proto *prot = READ_ONCE(sk->sk_prot);
3131
3132 if (prot->sock_is_readable)
3133 return prot->sock_is_readable(sk);
3134
3135 return false;
3136}
3137#endif /* _SOCK_H */
3138

source code of linux/include/net/sock.h