Donate to e Foundation | Murena handsets with /e/OS | Own a part of Murena! Learn more

Commit 9b1f3d6e authored by Martin KaFai Lau's avatar Martin KaFai Lau Committed by Alexei Starovoitov
Browse files

bpf: Refactor sock_ops_convert_ctx_access 



The next patch will introduce a new "struct bpf_tcp_sock" which
exposes the same tcp_sock's fields already exposed in
"struct bpf_sock_ops".

This patch refactor the existing convert_ctx_access() codes for
"struct bpf_sock_ops" to get them ready to be reused for
"struct bpf_tcp_sock".  The "rtt_min" is not refactored
in this patch because its handling is different from other
fields.

The SOCK_OPS_GET_TCP_SOCK_FIELD is new. All other SOCK_OPS_XXX_FIELD
changes are code move only.

Acked-by: default avatarAlexei Starovoitov <ast@kernel.org>
Signed-off-by: default avatarMartin KaFai Lau <kafai@fb.com>
Signed-off-by: default avatarAlexei Starovoitov <ast@kernel.org>
parent aa65d696

net/core/filter.c

+127 −160
Original line number Diff line number Diff line
@@ -5030,6 +5030,54 @@ static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = { 
};

#endif /* CONFIG_IPV6_SEG6_BPF */



#define CONVERT_COMMON_TCP_SOCK_FIELDS(md_type, CONVERT)		\

do {									\

	switch (si->off) {						\

	case offsetof(md_type, snd_cwnd):				\

		CONVERT(snd_cwnd); break;				\

	case offsetof(md_type, srtt_us):				\

		CONVERT(srtt_us); break;				\

	case offsetof(md_type, snd_ssthresh):				\

		CONVERT(snd_ssthresh); break;				\

	case offsetof(md_type, rcv_nxt):				\

		CONVERT(rcv_nxt); break;				\

	case offsetof(md_type, snd_nxt):				\

		CONVERT(snd_nxt); break;				\

	case offsetof(md_type, snd_una):				\

		CONVERT(snd_una); break;				\

	case offsetof(md_type, mss_cache):				\

		CONVERT(mss_cache); break;				\

	case offsetof(md_type, ecn_flags):				\

		CONVERT(ecn_flags); break;				\

	case offsetof(md_type, rate_delivered):				\

		CONVERT(rate_delivered); break;				\

	case offsetof(md_type, rate_interval_us):			\

		CONVERT(rate_interval_us); break;			\

	case offsetof(md_type, packets_out):				\

		CONVERT(packets_out); break;				\

	case offsetof(md_type, retrans_out):				\

		CONVERT(retrans_out); break;				\

	case offsetof(md_type, total_retrans):				\

		CONVERT(total_retrans); break;				\

	case offsetof(md_type, segs_in):				\

		CONVERT(segs_in); break;				\

	case offsetof(md_type, data_segs_in):				\

		CONVERT(data_segs_in); break;				\

	case offsetof(md_type, segs_out):				\

		CONVERT(segs_out); break;				\

	case offsetof(md_type, data_segs_out):				\

		CONVERT(data_segs_out); break;				\

	case offsetof(md_type, lost_out):				\

		CONVERT(lost_out); break;				\

	case offsetof(md_type, sacked_out):				\

		CONVERT(sacked_out); break;				\

	case offsetof(md_type, bytes_received):				\

		CONVERT(bytes_received); break;				\

	case offsetof(md_type, bytes_acked):				\

		CONVERT(bytes_acked); break;				\

	}								\

} while (0)



#ifdef CONFIG_INET

static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,

			      int dif, int sdif, u8 family, u8 proto)
@@ -7196,6 +7244,85 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, 
	struct bpf_insn *insn = insn_buf;

	int off;



/* Helper macro for adding read access to tcp_sock or sock fields. */

#define SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)			      \

	do {								      \

		BUILD_BUG_ON(FIELD_SIZEOF(OBJ, OBJ_FIELD) >		      \

			     FIELD_SIZEOF(struct bpf_sock_ops, BPF_FIELD));   \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern,     \

						is_fullsock),		      \

				      si->dst_reg, si->src_reg,		      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       is_fullsock));		      \

		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 2);	      \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern, sk),\

				      si->dst_reg, si->src_reg,		      \

				      offsetof(struct bpf_sock_ops_kern, sk));\

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(OBJ,		      \

						       OBJ_FIELD),	      \

				      si->dst_reg, si->dst_reg,		      \

				      offsetof(OBJ, OBJ_FIELD));	      \

	} while (0)



#define SOCK_OPS_GET_TCP_SOCK_FIELD(FIELD) \

		SOCK_OPS_GET_FIELD(FIELD, FIELD, struct tcp_sock)



/* Helper macro for adding write access to tcp_sock or sock fields.

 * The macro is called with two registers, dst_reg which contains a pointer

 * to ctx (context) and src_reg which contains the value that should be

 * stored. However, we need an additional register since we cannot overwrite

 * dst_reg because it may be used later in the program.

 * Instead we "borrow" one of the other register. We first save its value

 * into a new (temp) field in bpf_sock_ops_kern, use it, and then restore

 * it at the end of the macro.

 */

#define SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)			      \

	do {								      \

		int reg = BPF_REG_9;					      \

		BUILD_BUG_ON(FIELD_SIZEOF(OBJ, OBJ_FIELD) >		      \

			     FIELD_SIZEOF(struct bpf_sock_ops, BPF_FIELD));   \

		if (si->dst_reg == reg || si->src_reg == reg)		      \

			reg--;						      \

		if (si->dst_reg == reg || si->src_reg == reg)		      \

			reg--;						      \

		*insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, reg,		      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       temp));			      \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern,     \

						is_fullsock),		      \

				      reg, si->dst_reg,			      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       is_fullsock));		      \

		*insn++ = BPF_JMP_IMM(BPF_JEQ, reg, 0, 2);		      \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern, sk),\

				      reg, si->dst_reg,			      \

				      offsetof(struct bpf_sock_ops_kern, sk));\

		*insn++ = BPF_STX_MEM(BPF_FIELD_SIZEOF(OBJ, OBJ_FIELD),	      \

				      reg, si->src_reg,			      \

				      offsetof(OBJ, OBJ_FIELD));	      \

		*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->dst_reg,		      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       temp));			      \

	} while (0)



#define SOCK_OPS_GET_OR_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ, TYPE)	      \

	do {								      \

		if (TYPE == BPF_WRITE)					      \

			SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \

		else							      \

			SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \

	} while (0)



	CONVERT_COMMON_TCP_SOCK_FIELDS(struct bpf_sock_ops,

				       SOCK_OPS_GET_TCP_SOCK_FIELD);



	if (insn > insn_buf)

		return insn - insn_buf;



	switch (si->off) {

	case offsetof(struct bpf_sock_ops, op) ...

	     offsetof(struct bpf_sock_ops, replylong[3]):
@@ -7353,175 +7480,15 @@ static u32 sock_ops_convert_ctx_access(enum bpf_access_type type, 
				      FIELD_SIZEOF(struct minmax_sample, t));

		break;



/* Helper macro for adding read access to tcp_sock or sock fields. */

#define SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)			      \

	do {								      \

		BUILD_BUG_ON(FIELD_SIZEOF(OBJ, OBJ_FIELD) >		      \

			     FIELD_SIZEOF(struct bpf_sock_ops, BPF_FIELD));   \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern,     \

						is_fullsock),		      \

				      si->dst_reg, si->src_reg,		      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       is_fullsock));		      \

		*insn++ = BPF_JMP_IMM(BPF_JEQ, si->dst_reg, 0, 2);	      \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern, sk),\

				      si->dst_reg, si->src_reg,		      \

				      offsetof(struct bpf_sock_ops_kern, sk));\

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(OBJ,		      \

						       OBJ_FIELD),	      \

				      si->dst_reg, si->dst_reg,		      \

				      offsetof(OBJ, OBJ_FIELD));	      \

	} while (0)



/* Helper macro for adding write access to tcp_sock or sock fields.

 * The macro is called with two registers, dst_reg which contains a pointer

 * to ctx (context) and src_reg which contains the value that should be

 * stored. However, we need an additional register since we cannot overwrite

 * dst_reg because it may be used later in the program.

 * Instead we "borrow" one of the other register. We first save its value

 * into a new (temp) field in bpf_sock_ops_kern, use it, and then restore

 * it at the end of the macro.

 */

#define SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ)			      \

	do {								      \

		int reg = BPF_REG_9;					      \

		BUILD_BUG_ON(FIELD_SIZEOF(OBJ, OBJ_FIELD) >		      \

			     FIELD_SIZEOF(struct bpf_sock_ops, BPF_FIELD));   \

		if (si->dst_reg == reg || si->src_reg == reg)		      \

			reg--;						      \

		if (si->dst_reg == reg || si->src_reg == reg)		      \

			reg--;						      \

		*insn++ = BPF_STX_MEM(BPF_DW, si->dst_reg, reg,		      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       temp));			      \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern,     \

						is_fullsock),		      \

				      reg, si->dst_reg,			      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       is_fullsock));		      \

		*insn++ = BPF_JMP_IMM(BPF_JEQ, reg, 0, 2);		      \

		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(			      \

						struct bpf_sock_ops_kern, sk),\

				      reg, si->dst_reg,			      \

				      offsetof(struct bpf_sock_ops_kern, sk));\

		*insn++ = BPF_STX_MEM(BPF_FIELD_SIZEOF(OBJ, OBJ_FIELD),	      \

				      reg, si->src_reg,			      \

				      offsetof(OBJ, OBJ_FIELD));	      \

		*insn++ = BPF_LDX_MEM(BPF_DW, reg, si->dst_reg,		      \

				      offsetof(struct bpf_sock_ops_kern,      \

					       temp));			      \

	} while (0)



#define SOCK_OPS_GET_OR_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ, TYPE)	      \

	do {								      \

		if (TYPE == BPF_WRITE)					      \

			SOCK_OPS_SET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \

		else							      \

			SOCK_OPS_GET_FIELD(BPF_FIELD, OBJ_FIELD, OBJ);	      \

	} while (0)



	case offsetof(struct bpf_sock_ops, snd_cwnd):

		SOCK_OPS_GET_FIELD(snd_cwnd, snd_cwnd, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, srtt_us):

		SOCK_OPS_GET_FIELD(srtt_us, srtt_us, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, bpf_sock_ops_cb_flags):

		SOCK_OPS_GET_FIELD(bpf_sock_ops_cb_flags, bpf_sock_ops_cb_flags,

				   struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, snd_ssthresh):

		SOCK_OPS_GET_FIELD(snd_ssthresh, snd_ssthresh, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, rcv_nxt):

		SOCK_OPS_GET_FIELD(rcv_nxt, rcv_nxt, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, snd_nxt):

		SOCK_OPS_GET_FIELD(snd_nxt, snd_nxt, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, snd_una):

		SOCK_OPS_GET_FIELD(snd_una, snd_una, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, mss_cache):

		SOCK_OPS_GET_FIELD(mss_cache, mss_cache, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, ecn_flags):

		SOCK_OPS_GET_FIELD(ecn_flags, ecn_flags, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, rate_delivered):

		SOCK_OPS_GET_FIELD(rate_delivered, rate_delivered,

				   struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, rate_interval_us):

		SOCK_OPS_GET_FIELD(rate_interval_us, rate_interval_us,

				   struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, packets_out):

		SOCK_OPS_GET_FIELD(packets_out, packets_out, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, retrans_out):

		SOCK_OPS_GET_FIELD(retrans_out, retrans_out, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, total_retrans):

		SOCK_OPS_GET_FIELD(total_retrans, total_retrans,

				   struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, segs_in):

		SOCK_OPS_GET_FIELD(segs_in, segs_in, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, data_segs_in):

		SOCK_OPS_GET_FIELD(data_segs_in, data_segs_in, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, segs_out):

		SOCK_OPS_GET_FIELD(segs_out, segs_out, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, data_segs_out):

		SOCK_OPS_GET_FIELD(data_segs_out, data_segs_out,

				   struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, lost_out):

		SOCK_OPS_GET_FIELD(lost_out, lost_out, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, sacked_out):

		SOCK_OPS_GET_FIELD(sacked_out, sacked_out, struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, sk_txhash):

		SOCK_OPS_GET_OR_SET_FIELD(sk_txhash, sk_txhash,

					  struct sock, type);

		break;



	case offsetof(struct bpf_sock_ops, bytes_received):

		SOCK_OPS_GET_FIELD(bytes_received, bytes_received,

				   struct tcp_sock);

		break;



	case offsetof(struct bpf_sock_ops, bytes_acked):

		SOCK_OPS_GET_FIELD(bytes_acked, bytes_acked, struct tcp_sock);

		break;



	}

	return insn - insn_buf;

}