// SPDX-License-Identifier: GPL-2.0 /* * Ceph msgr2 protocol implementation * * Copyright (C) 2020 Ilya Dryomov <[email protected]> */ #include <linux/ceph/ceph_debug.h> #include <crypto/aead.h> #include <crypto/hash.h> #include <crypto/sha2.h> #include <crypto/utils.h> #include <linux/bvec.h> #include <linux/crc32c.h> #include <linux/net.h> #include <linux/scatterlist.h> #include <linux/socket.h> #include <linux/sched/mm.h> #include <net/sock.h> #include <net/tcp.h> #include <linux/ceph/ceph_features.h> #include <linux/ceph/decode.h> #include <linux/ceph/libceph.h> #include <linux/ceph/messenger.h> #include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */ #define FRAME_TAG_HELLO … #define FRAME_TAG_AUTH_REQUEST … #define FRAME_TAG_AUTH_BAD_METHOD … #define FRAME_TAG_AUTH_REPLY_MORE … #define FRAME_TAG_AUTH_REQUEST_MORE … #define FRAME_TAG_AUTH_DONE … #define FRAME_TAG_AUTH_SIGNATURE … #define FRAME_TAG_CLIENT_IDENT … #define FRAME_TAG_SERVER_IDENT … #define FRAME_TAG_IDENT_MISSING_FEATURES … #define FRAME_TAG_SESSION_RECONNECT … #define FRAME_TAG_SESSION_RESET … #define FRAME_TAG_SESSION_RETRY … #define FRAME_TAG_SESSION_RETRY_GLOBAL … #define FRAME_TAG_SESSION_RECONNECT_OK … #define FRAME_TAG_WAIT … #define FRAME_TAG_MESSAGE … #define FRAME_TAG_KEEPALIVE2 … #define FRAME_TAG_KEEPALIVE2_ACK … #define FRAME_TAG_ACK … #define FRAME_LATE_STATUS_ABORTED … #define FRAME_LATE_STATUS_COMPLETE … #define FRAME_LATE_STATUS_ABORTED_MASK … #define IN_S_HANDLE_PREAMBLE … #define IN_S_HANDLE_CONTROL … #define IN_S_HANDLE_CONTROL_REMAINDER … #define IN_S_PREPARE_READ_DATA … #define IN_S_PREPARE_READ_DATA_CONT … #define IN_S_PREPARE_READ_ENC_PAGE … #define IN_S_PREPARE_SPARSE_DATA … #define IN_S_PREPARE_SPARSE_DATA_CONT … #define IN_S_HANDLE_EPILOGUE … #define IN_S_FINISH_SKIP … #define OUT_S_QUEUE_DATA … #define OUT_S_QUEUE_DATA_CONT … #define OUT_S_QUEUE_ENC_PAGE … #define OUT_S_QUEUE_ZEROS … #define OUT_S_FINISH_MESSAGE … #define OUT_S_GET_NEXT … #define CTRL_BODY(p) … #define FRONT_PAD(p) … #define MIDDLE_PAD(p) … #define DATA_PAD(p) … #define CEPH_MSG_FLAGS … static int do_recvmsg(struct socket *sock, struct iov_iter *it) { … } /* * Read as much as possible. * * Return: * 1 - done, nothing (else) to read * 0 - socket is empty, need to wait * <0 - error */ static int ceph_tcp_recv(struct ceph_connection *con) { … } static int do_sendmsg(struct socket *sock, struct iov_iter *it) { … } static int do_try_sendpage(struct socket *sock, struct iov_iter *it) { … } /* * Write as much as possible. The socket is expected to be corked, * so we don't bother with MSG_MORE here. * * Return: * 1 - done, nothing (else) to write * 0 - socket is full, need to wait * <0 - error */ static int ceph_tcp_send(struct ceph_connection *con) { … } static void add_in_kvec(struct ceph_connection *con, void *buf, int len) { … } static void reset_in_kvecs(struct ceph_connection *con) { … } static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv) { … } static void set_in_skip(struct ceph_connection *con, int len) { … } static void add_out_kvec(struct ceph_connection *con, void *buf, int len) { … } static void reset_out_kvecs(struct ceph_connection *con) { … } static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv, bool zerocopy) { … } static void set_out_bvec_zero(struct ceph_connection *con) { … } static void out_zero_add(struct ceph_connection *con, int len) { … } static void *alloc_conn_buf(struct ceph_connection *con, int len) { … } static void free_conn_bufs(struct ceph_connection *con) { … } static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len) { … } static void clear_in_sign_kvecs(struct ceph_connection *con) { … } static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len) { … } static void clear_out_sign_kvecs(struct ceph_connection *con) { … } static bool con_secure(struct ceph_connection *con) { … } static int front_len(const struct ceph_msg *msg) { … } static int middle_len(const struct ceph_msg *msg) { … } static int data_len(const struct ceph_msg *msg) { … } static bool need_padding(int len) { … } static int padded_len(int len) { … } static int padding_len(int len) { … } /* preamble + control segment */ static int head_onwire_len(int ctrl_len, bool secure) { … } /* front, middle and data segments + epilogue */ static int __tail_onwire_len(int front_len, int middle_len, int data_len, bool secure) { … } static int tail_onwire_len(const struct ceph_msg *msg, bool secure) { … } /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */ #define MESSAGE_HEAD_PLAIN_LEN … static const int frame_aligns[] = …; /* * Discards trailing empty segments, unless there is just one segment. * A frame always has at least one (possibly empty) segment. */ static int calc_segment_count(const int *lens, int len_cnt) { … } static void init_frame_desc(struct ceph_frame_desc *desc, int tag, const int *lens, int len_cnt) { … } /* * Preamble crc covers everything up to itself (28 bytes) and * is calculated and verified irrespective of the connection mode * (i.e. even if the frame is encrypted). */ static void encode_preamble(const struct ceph_frame_desc *desc, void *p) { … } static int decode_preamble(void *p, struct ceph_frame_desc *desc) { … } static void encode_epilogue_plain(struct ceph_connection *con, bool aborted) { … } static void encode_epilogue_secure(struct ceph_connection *con, bool aborted) { … } static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc, u32 *data_crc) { … } static void fill_header(struct ceph_msg_header *hdr, const struct ceph_msg_header2 *hdr2, int front_len, int middle_len, int data_len, const struct ceph_entity_name *peer_name) { … } static void fill_header2(struct ceph_msg_header2 *hdr2, const struct ceph_msg_header *hdr, u64 ack_seq) { … } static int verify_control_crc(struct ceph_connection *con) { … } static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc, u32 middle_crc, u32 data_crc) { … } static int setup_crypto(struct ceph_connection *con, const u8 *session_key, int session_key_len, const u8 *con_secret, int con_secret_len) { … } static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs, int kvec_cnt, u8 *hmac) { … } static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce) { … } static int gcm_crypt(struct ceph_connection *con, bool encrypt, struct scatterlist *src, struct scatterlist *dst, int src_len) { … } static void get_bvec_at(struct ceph_msg_data_cursor *cursor, struct bio_vec *bv) { … } static int calc_sg_cnt(void *buf, int buf_len) { … } static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor) { … } static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad) { … } static void init_sgs_cursor(struct scatterlist **sg, struct ceph_msg_data_cursor *cursor, u8 *pad) { … } /** * init_sgs_pages: set up scatterlist on an array of page pointers * @sg: scatterlist to populate * @pages: pointer to page array * @dpos: position in the array to start (bytes) * @dlen: len to add to sg (bytes) * @pad: pointer to pad destination (if any) * * Populate the scatterlist from the page array, starting at an arbitrary * byte in the array and running for a specified length. */ static void init_sgs_pages(struct scatterlist **sg, struct page **pages, int dpos, int dlen, u8 *pad) { … } static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg, u8 *front_pad, u8 *middle_pad, u8 *data_pad, void *epilogue, struct page **pages, int dpos, bool add_tag) { … } static int decrypt_preamble(struct ceph_connection *con) { … } static int decrypt_control_remainder(struct ceph_connection *con) { … } /* Process sparse read data that lives in a buffer */ static int process_v2_sparse_read(struct ceph_connection *con, struct page **pages, int spos) { … } static int decrypt_tail(struct ceph_connection *con) { … } static int prepare_banner(struct ceph_connection *con) { … } /* * base: * preamble * control body (ctrl_len bytes) * space for control crc * * extdata (optional): * control body (extdata_len bytes) * * Compute control crc and gather base and extdata into: * * preamble * control body (ctrl_len + extdata_len bytes) * control crc * * Preamble should already be encoded at the start of base. */ static void prepare_head_plain(struct ceph_connection *con, void *base, int ctrl_len, void *extdata, int extdata_len, bool to_be_signed) { … } static int prepare_head_secure_small(struct ceph_connection *con, void *base, int ctrl_len) { … } /* * base: * preamble * control body (ctrl_len bytes) * space for padding, if needed * space for control remainder auth tag * space for preamble auth tag * * Encrypt preamble and the inline portion, then encrypt the remainder * and gather into: * * preamble * control body (48 bytes) * preamble auth tag * control body (ctrl_len - 48 bytes) * zero padding, if needed * control remainder auth tag * * Preamble should already be encoded at the start of base. */ static int prepare_head_secure_big(struct ceph_connection *con, void *base, int ctrl_len) { … } static int __prepare_control(struct ceph_connection *con, int tag, void *base, int ctrl_len, void *extdata, int extdata_len, bool to_be_signed) { … } static int prepare_control(struct ceph_connection *con, int tag, void *base, int ctrl_len) { … } static int prepare_hello(struct ceph_connection *con) { … } /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */ #define AUTH_BUF_LEN … static int prepare_auth_request(struct ceph_connection *con) { … } static int prepare_auth_request_more(struct ceph_connection *con, void *reply, int reply_len) { … } static int prepare_auth_signature(struct ceph_connection *con) { … } static int prepare_client_ident(struct ceph_connection *con) { … } static int prepare_session_reconnect(struct ceph_connection *con) { … } static int prepare_keepalive2(struct ceph_connection *con) { … } static int prepare_ack(struct ceph_connection *con) { … } static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted) { … } /* * For "used" empty segments, crc is -1. For unused (trailing) * segments, crc is 0. */ static void prepare_message_plain(struct ceph_connection *con) { … } /* * Unfortunately the kernel crypto API doesn't support streaming * (piecewise) operation for AEAD algorithms, so we can't get away * with a fixed size buffer and a couple sgs. Instead, we have to * allocate pages for the entire tail of the message (currently up * to ~32M) and two sgs arrays (up to ~256K each)... */ static int prepare_message_secure(struct ceph_connection *con) { … } static int prepare_message(struct ceph_connection *con) { … } static int prepare_read_banner_prefix(struct ceph_connection *con) { … } static int prepare_read_banner_payload(struct ceph_connection *con, int payload_len) { … } static void prepare_read_preamble(struct ceph_connection *con) { … } static int prepare_read_control(struct ceph_connection *con) { … } static int prepare_read_control_remainder(struct ceph_connection *con) { … } static int prepare_read_data(struct ceph_connection *con) { … } static void prepare_read_data_cont(struct ceph_connection *con) { … } static int prepare_sparse_read_cont(struct ceph_connection *con) { … } static int prepare_sparse_read_data(struct ceph_connection *con) { … } static int prepare_read_tail_plain(struct ceph_connection *con) { … } static void prepare_read_enc_page(struct ceph_connection *con) { … } static int prepare_read_tail_secure(struct ceph_connection *con) { … } static void __finish_skip(struct ceph_connection *con) { … } static void prepare_skip_message(struct ceph_connection *con) { … } static int process_banner_prefix(struct ceph_connection *con) { … } static int process_banner_payload(struct ceph_connection *con) { … } static int process_hello(struct ceph_connection *con, void *p, void *end) { … } static int process_auth_bad_method(struct ceph_connection *con, void *p, void *end) { … } static int process_auth_reply_more(struct ceph_connection *con, void *p, void *end) { … } /* * Align session_key and con_secret to avoid GFP_ATOMIC allocation * inside crypto_shash_setkey() and crypto_aead_setkey() called from * setup_crypto(). __aligned(16) isn't guaranteed to work for stack * objects, so do it by hand. */ static int process_auth_done(struct ceph_connection *con, void *p, void *end) { … } static int process_auth_signature(struct ceph_connection *con, void *p, void *end) { … } static int process_server_ident(struct ceph_connection *con, void *p, void *end) { … } static int process_ident_missing_features(struct ceph_connection *con, void *p, void *end) { … } static int process_session_reconnect_ok(struct ceph_connection *con, void *p, void *end) { … } static int process_session_retry(struct ceph_connection *con, void *p, void *end) { … } static int process_session_retry_global(struct ceph_connection *con, void *p, void *end) { … } static int process_session_reset(struct ceph_connection *con, void *p, void *end) { … } static int process_keepalive2_ack(struct ceph_connection *con, void *p, void *end) { … } static int process_ack(struct ceph_connection *con, void *p, void *end) { … } static int process_control(struct ceph_connection *con, void *p, void *end) { … } /* * Return: * 1 - con->in_msg set, read message * 0 - skip message * <0 - error */ static int process_message_header(struct ceph_connection *con, void *p, void *end) { … } static int process_message(struct ceph_connection *con) { … } static int __handle_control(struct ceph_connection *con, void *p) { … } static int handle_preamble(struct ceph_connection *con) { … } static int handle_control(struct ceph_connection *con) { … } static int handle_control_remainder(struct ceph_connection *con) { … } static int handle_epilogue(struct ceph_connection *con) { … } static void finish_skip(struct ceph_connection *con) { … } static int populate_in_iter(struct ceph_connection *con) { … } int ceph_con_v2_try_read(struct ceph_connection *con) { … } static void queue_data(struct ceph_connection *con) { … } static void queue_data_cont(struct ceph_connection *con) { … } static void queue_enc_page(struct ceph_connection *con) { … } static void queue_zeros(struct ceph_connection *con) { … } static void finish_message(struct ceph_connection *con) { … } static int populate_out_iter(struct ceph_connection *con) { … } int ceph_con_v2_try_write(struct ceph_connection *con) { … } static u32 crc32c_zeros(u32 crc, int zero_len) { … } static void prepare_zero_front(struct ceph_connection *con, int resid) { … } static void prepare_zero_middle(struct ceph_connection *con, int resid) { … } static void prepare_zero_data(struct ceph_connection *con) { … } static void revoke_at_queue_data(struct ceph_connection *con) { … } static void revoke_at_queue_data_cont(struct ceph_connection *con) { … } static void revoke_at_finish_message(struct ceph_connection *con) { … } void ceph_con_v2_revoke(struct ceph_connection *con) { … } static void revoke_at_prepare_read_data(struct ceph_connection *con) { … } static void revoke_at_prepare_read_data_cont(struct ceph_connection *con) { … } static void revoke_at_prepare_read_enc_page(struct ceph_connection *con) { … } static void revoke_at_prepare_sparse_data(struct ceph_connection *con) { … } static void revoke_at_handle_epilogue(struct ceph_connection *con) { … } void ceph_con_v2_revoke_incoming(struct ceph_connection *con) { … } bool ceph_con_v2_opened(struct ceph_connection *con) { … } void ceph_con_v2_reset_session(struct ceph_connection *con) { … } void ceph_con_v2_reset_protocol(struct ceph_connection *con) { … }
Codebase Browser
linux