// // // Copyright 2017 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // // #include <grpc/support/port_platform.h> #include <grpc/support/log.h> #include "src/core/lib/gprpp/crash.h" #include "src/core/lib/iomgr/port.h" // This polling engine is only relevant on linux kernels supporting epoll // epoll_create() or epoll_create1() #ifdef GRPC_LINUX_EPOLL #include <assert.h> #include <errno.h> #include <fcntl.h> #include <limits.h> #include <poll.h> #include <pthread.h> #include <string.h> #include <sys/epoll.h> #include <sys/socket.h> #include <unistd.h> #include <string> #include <vector> #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include <grpc/support/alloc.h> #include <grpc/support/cpu.h> #include "src/core/lib/debug/stats.h" #include "src/core/lib/debug/stats_data.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/strerror.h" #include "src/core/lib/iomgr/block_annotate.h" #include "src/core/lib/iomgr/ev_epoll1_linux.h" #include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/lockfree_event.h" #include "src/core/lib/iomgr/wakeup_fd_posix.h" static grpc_wakeup_fd global_wakeup_fd; //****************************************************************************** // Singleton epoll set related fields // #define MAX_EPOLL_EVENTS … #define MAX_EPOLL_EVENTS_HANDLED_PER_ITERATION … // NOTE ON SYNCHRONIZATION: // - Fields in this struct are only modified by the designated poller. Hence // there is no need for any locks to protect the struct. // - num_events and cursor fields have to be of atomic type to provide memory // visibility guarantees only. i.e In case of multiple pollers, the designated // polling thread keeps changing; the thread that wrote these values may be // different from the thread reading the values // epoll_set; // The global singleton epoll set static epoll_set g_epoll_set; static int epoll_create_and_cloexec() { … } // Must be called *only* once static bool epoll_set_init() { … } // epoll_set_init() MUST be called before calling this. static void epoll_set_shutdown() { … } //****************************************************************************** // Fd Declarations // // Only used when GRPC_ENABLE_FORK_SUPPORT=1 struct grpc_fork_fd_list { … }; struct grpc_fd { … }; static void fd_global_init(void); static void fd_global_shutdown(void); //****************************************************************************** // Pollset Declarations // kick_state; static const char* kick_state_string(kick_state st) { … } struct grpc_pollset_worker { … }; #define SET_KICK_STATE(worker, kick_state) … #define MAX_NEIGHBORHOODS … pollset_neighborhood; struct grpc_pollset { … }; //****************************************************************************** // Pollset-set Declarations // struct grpc_pollset_set { … }; //****************************************************************************** // Common helpers // static bool append_error(grpc_error_handle* composite, grpc_error_handle error, const char* desc) { … } //****************************************************************************** // Fd Definitions // // We need to keep a freelist not because of any concerns of malloc performance // but instead so that implementations with multiple threads in (for example) // epoll_wait deal with the race between pollset removal and incoming poll // notifications. // // The problem is that the poller ultimately holds a reference to this // object, so it is very difficult to know when is safe to free it, at least // without some expensive synchronization. // // If we keep the object freelisted, in the worst case losing this race just // becomes a spurious read notification on a reused fd. // // The alarm system needs to be able to wakeup 'some poller' sometimes // (specifically when a new alarm needs to be triggered earlier than the next // alarm 'epoch'). This wakeup_fd gives us something to alert on when such a // case occurs. static grpc_fd* fd_freelist = …; static gpr_mu fd_freelist_mu; // Only used when GRPC_ENABLE_FORK_SUPPORT=1 static grpc_fd* fork_fd_list_head = …; static gpr_mu fork_fd_list_mu; static void fd_global_init(void) { … } static void fd_global_shutdown(void) { … } static void fork_fd_list_add_grpc_fd(grpc_fd* fd) { … } static void fork_fd_list_remove_grpc_fd(grpc_fd* fd) { … } static grpc_fd* fd_create(int fd, const char* name, bool track_err) { … } static int fd_wrapped_fd(grpc_fd* fd) { … } // if 'releasing_fd' is true, it means that we are going to detach the internal // fd from grpc_fd structure (i.e which means we should not be calling // shutdown() syscall on that fd) static void fd_shutdown_internal(grpc_fd* fd, grpc_error_handle why, bool releasing_fd) { … } // Might be called multiple times static void fd_shutdown(grpc_fd* fd, grpc_error_handle why) { … } static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd, const char* reason) { … } static bool fd_is_shutdown(grpc_fd* fd) { … } static void fd_notify_on_read(grpc_fd* fd, grpc_closure* closure) { … } static void fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) { … } static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) { … } static void fd_become_readable(grpc_fd* fd) { … } static void fd_become_writable(grpc_fd* fd) { … } static void fd_has_errors(grpc_fd* fd) { … } static void fd_set_pre_allocated(grpc_fd* fd) { … } //****************************************************************************** // Pollset Definitions // static thread_local grpc_pollset* g_current_thread_pollset; static thread_local grpc_pollset_worker* g_current_thread_worker; // The designated poller static gpr_atm g_active_poller; static pollset_neighborhood* g_neighborhoods; static size_t g_num_neighborhoods; // Return true if first in list static bool worker_insert(grpc_pollset* pollset, grpc_pollset_worker* worker) { … } // Return true if last in list worker_remove_result; static worker_remove_result worker_remove(grpc_pollset* pollset, grpc_pollset_worker* worker) { … } static size_t choose_neighborhood(void) { … } static grpc_error_handle pollset_global_init(void) { … } static void pollset_global_shutdown(void) { … } static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) { … } static void pollset_destroy(grpc_pollset* pollset) { … } static grpc_error_handle pollset_kick_all(grpc_pollset* pollset) { … } static void pollset_maybe_finish_shutdown(grpc_pollset* pollset) { … } static void pollset_shutdown(grpc_pollset* pollset, grpc_closure* closure) { … } static int poll_deadline_to_millis_timeout(grpc_core::Timestamp millis) { … } // Process the epoll events found by do_epoll_wait() function. // - g_epoll_set.cursor points to the index of the first event to be processed // - This function then processes up-to MAX_EPOLL_EVENTS_PER_ITERATION and // updates the g_epoll_set.cursor // NOTE ON SYNCRHONIZATION: Similar to do_epoll_wait(), this function is only // called by g_active_poller thread. So there is no need for synchronization // when accessing fields in g_epoll_set static grpc_error_handle process_epoll_events(grpc_pollset* /*pollset*/) { … } // Do epoll_wait and store the events in g_epoll_set.events field. This does not // "process" any of the events yet; that is done in process_epoll_events(). // *See process_epoll_events() function for more details. // NOTE ON SYNCHRONIZATION: At any point of time, only the g_active_poller // (i.e the designated poller thread) will be calling this function. So there is // no need for any synchronization when accesing fields in g_epoll_set static grpc_error_handle do_epoll_wait(grpc_pollset* ps, grpc_core::Timestamp deadline) { … } static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, grpc_pollset_worker** worker_hdl, grpc_core::Timestamp deadline) { … } static bool check_neighborhood_for_available_poller( pollset_neighborhood* neighborhood) { … } static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, grpc_pollset_worker** worker_hdl) { … } // pollset->po.mu lock must be held by the caller before calling this. // The function pollset_work() may temporarily release the lock (pollset->po.mu) // during the course of its execution but it will always re-acquire the lock and // ensure that it is held by the time the function returns static grpc_error_handle pollset_work(grpc_pollset* ps, grpc_pollset_worker** worker_hdl, grpc_core::Timestamp deadline) { … } static grpc_error_handle pollset_kick(grpc_pollset* pollset, grpc_pollset_worker* specific_worker) { … } static void pollset_add_fd(grpc_pollset* /*pollset*/, grpc_fd* /*fd*/) { … } //****************************************************************************** // Pollset-set Definitions // static grpc_pollset_set* pollset_set_create(void) { … } static void pollset_set_destroy(grpc_pollset_set* /*pss*/) { … } static void pollset_set_add_fd(grpc_pollset_set* /*pss*/, grpc_fd* /*fd*/) { … } static void pollset_set_del_fd(grpc_pollset_set* /*pss*/, grpc_fd* /*fd*/) { … } static void pollset_set_add_pollset(grpc_pollset_set* /*pss*/, grpc_pollset* /*ps*/) { … } static void pollset_set_del_pollset(grpc_pollset_set* /*pss*/, grpc_pollset* /*ps*/) { … } static void pollset_set_add_pollset_set(grpc_pollset_set* /*bag*/, grpc_pollset_set* /*item*/) { … } static void pollset_set_del_pollset_set(grpc_pollset_set* /*bag*/, grpc_pollset_set* /*item*/) { … } //****************************************************************************** // Event engine binding // static bool is_any_background_poller_thread(void) { … } static void shutdown_background_closure(void) { … } static bool add_closure_to_background_poller(grpc_closure* /*closure*/, grpc_error_handle /*error*/) { … } static void shutdown_engine(void) { … } static bool init_epoll1_linux(); const grpc_event_engine_vtable grpc_ev_epoll1_posix = …; // Called by the child process's post-fork handler to close open fds, including // the global epoll fd. This allows gRPC to shutdown in the child process // without interfering with connections or RPCs ongoing in the parent. static void reset_event_manager_on_fork() { … } // It is possible that GLIBC has epoll but the underlying kernel doesn't. // Create epoll_fd (epoll_set_init() takes care of that) to make sure epoll // support is available static bool init_epoll1_linux() { … } #else // defined(GRPC_LINUX_EPOLL) #if defined(GRPC_POSIX_SOCKET_EV_EPOLL1) #include "src/core/lib/iomgr/ev_epoll1_linux.h" const grpc_event_engine_vtable grpc_ev_epoll1_posix = { 1, true, false, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, /* name = */ "epoll1", /* check_engine_available = */ [](bool) { return false; }, nullptr, nullptr, nullptr, nullptr, nullptr, }; #endif // defined(GRPC_POSIX_SOCKET_EV_EPOLL1) #endif // !defined(GRPC_LINUX_EPOLL)
Codebase Browser
chromium