Libevent 學習--------(eventop)事件多路分發器

一、事件多路分發器一般使用I/O複用接口。
Libvent中使用eventop 結構作爲後端I/O複用的統一接口, libevent裏面稱之爲後端， 也就是一種用於檢測哪種事件已經就緒的方法。

/** Structure to define the backend of a given event_base. */
struct eventop {                                                                                                                                                            
    const char *name;
    void *(*init)(struct event_base *);
    int (*add)(struct event_base *, evutil_socket_t fd, short old, short events, void *fdinfo);
    int (*del)(struct event_base *, evutil_socket_t fd, short old, short events, void *fdinfo);
    int (*dispatch)(struct event_base *, struct timeval *);
    void (*dealloc)(struct event_base *);
    int need_reinit;
    enum event_method_feature features;
    size_t fdinfo_len;
};

name: 表示後端的名字，如：evport, kqueue, epoll, devpoll, poll, select, win32等.
init: 回調函數，入參爲event_base, 該函數創建一個新結構體，這個結構體裏面包含了後端需要使用的數據， 返回一個void*指針，這個返回值指針最終賦值給event_base.ev_base
add: 回調函數，將fd或者信號加入到對應的I/O複用結構中
del: 回調函數，將fd或者信號移除對應的I/O服用結構
dispath: 回調函數, 實現事件循環處理的主要函數，等待事件發生，將對應事件加入激活事件隊列，調用事件處理函數
dealloc：回調函數，從event_base中清理和釋放數據
need_reinit: 標記位， 是否需要重新初始化event base
features: 位數組，表示後端能夠支持的event_method_features
fdinfo_len: 額外需要爲每一個fd記錄的信息長度

二、在Libvent中eventop 存放在event_base結構中的const struct eventop *evsel字段, 創建event_base時初始化，代碼如下:

struct event_base *
event_base_new_with_config(const struct event_config *cfg)
{
    .....
    for (i = 0; eventops[i] && !base->evbase; i++) {
        if (cfg != NULL) { 
            /* determine if this backend should be avoided */
            if (event_config_is_avoided_method(cfg, eventops[i]->name))
                continue;
            if ((eventops[i]->features & cfg->require_features)!= cfg->require_features)
                continue;
        }

        /* also obey the environment variables */
        if (should_check_environment &&
            event_is_method_disabled(eventops[i]->name))
            continue;

        base->evsel = eventops[i];

        base->evbase = base->evsel->init(base);
    }

    if (base->evbase == NULL) {
        event_warnx("%s: no event mechanism available", __func__);
        base->evsel = NULL;
        event_base_free(base);
        return NULL;
    } 
    .....
}

重點關注的結構是eventops數組，取一個有效的eventops數組元素， 賦值給base裏面的evsel， 並且調用init回調函數創建後端處理需要的數據結構複製給evbase，如果創建的evbase不爲空，就取滿足這個條件的第一個eventops數組元素作爲最終複製給base裏面的evsel。 在linux默認第一個有效的eventops是epollops，使用的I/O複用方法是epoll。

/* Array of backends in order of preference. */                                                                                                                                        
static const struct eventop *eventops[] = {
#ifdef _EVENT_HAVE_EVENT_PORTS
    &evportops,
#endif
#ifdef _EVENT_HAVE_WORKING_KQUEUE
    &kqops,
#endif
#ifdef _EVENT_HAVE_EPOLL
    &epollops,
#endif
#ifdef _EVENT_HAVE_DEVPOLL
    &devpollops,
#endif
#ifdef _EVENT_HAVE_POLL
    &pollops,
#endif
#ifdef _EVENT_HAVE_SELECT
    &selectops,
#endif
#ifdef WIN32
    &win32ops,
#endif
    NULL
};

epollops 結構

const struct eventop epollops = {
    "epoll",
    epoll_init,
    epoll_nochangelist_add,
    epoll_nochangelist_del,
    epoll_dispatch,
    epoll_dealloc,
    1, /* need reinit */
    EV_FEATURE_ET|EV_FEATURE_O1,
    0
};

調用base->evbase = base->evsel->init(base); 實際就是調用epoll_init, base->evbase最終指向了epollop結構

struct epollop {                                                                                                                                                                      
    struct epoll_event *events;
    int nevents;
    int epfd;
};  

#define INITIAL_NEVENT 32                                                                                                                                                               
#define MAX_NEVENT 4096

/* On Linux kernels at least up to 2.6.24.4, epoll can't handle timeout
 * values bigger than (LONG_MAX - 999ULL)/HZ.  HZ in the wild can be
 * as big as 1000, and LONG_MAX can be as small as (1<<31)-1, so the
 * largest number of msec we can support here is 2147482.  Let's
 * round that down by 47 seconds.
 */
#define MAX_EPOLL_TIMEOUT_MSEC (35*60*1000)

static void *
epoll_init(struct event_base *base)
{
    int epfd;
    struct epollop *epollop;

    /* Initialize the kernel queue.  (The size field is ignored since
     * 2.6.8.) */
    if ((epfd = epoll_create(32000)) == -1) {
        if (errno != ENOSYS)
            event_warn("epoll_create");
        return (NULL);
    }

    evutil_make_socket_closeonexec(epfd);

    if (!(epollop = mm_calloc(1, sizeof(struct epollop)))) {
        close(epfd);
        return (NULL);
    }

    epollop->epfd = epfd;

    /* Initialize fields */
    epollop->events = mm_calloc(INITIAL_NEVENT, sizeof(struct epoll_event));
    if (epollop->events == NULL) {
        mm_free(epollop);
        close(epfd);
        return (NULL);
    }
    epollop->nevents = INITIAL_NEVENT;


    if ((base->flags & EVENT_BASE_FLAG_EPOLL_USE_CHANGELIST) != 0 ||
        ((base->flags & EVENT_BASE_FLAG_IGNORE_ENV) == 0 &&
        evutil_getenv("EVENT_EPOLL_USE_CHANGELIST") != NULL))
        base->evsel = &epollops_changelist;

    evsig_init(base);

    return (epollop);
}

至此，事件多路分發器已經創建完畢。
接下來就應該將event事件註冊到事件多路分發器中
三、事件的添加
Libevent中通過 event_add來添加event, 實質上是調用event_add_internal

int
event_add(struct event *ev, const struct timeval *tv)
{
    int res;

    if (EVUTIL_FAILURE_CHECK(!ev->ev_base)) {
        event_warnx("%s: event has no event_base set.", __func__);
        return -1;
    }

    EVBASE_ACQUIRE_LOCK(ev->ev_base, th_base_lock);

    res = event_add_internal(ev, tv, 0);                                                                                                                                 

    EVBASE_RELEASE_LOCK(ev->ev_base, th_base_lock);

    return (res);
}

static inline int
event_add_internal(struct event *ev, const struct timeval *tv,
    int tv_is_absolute)
{
    ......
    /*處理沒有激活的READ WRITE SIGNAL 事件*/
    if ((ev->ev_events & (EV_READ|EV_WRITE|EV_SIGNAL)) &&
        !(ev->ev_flags & (EVLIST_INSERTED|EVLIST_ACTIVE))) {
        /*如果是I/O事件，添加I/O事件和I/O事件處理器的映射關係*/
        if (ev->ev_events & (EV_READ|EV_WRITE))
            res = evmap_io_add(base, ev->ev_fd, ev);
        else if (ev->ev_events & EV_SIGNAL)
            res = evmap_signal_add(base, (int)ev->ev_fd, ev);/*如果是信號事件，添加信號事件和信號事件處理器的映射關係*/
        /*將事件處理器插入註冊事件隊列中*/
        if (res != -1)
            event_queue_insert(base, ev, EVLIST_INSERTED);
        if (res == 1) {
            /* evmap says we need to notify the main thread. */
            /*事件多路分發器中添加了新的事件，所以要通知主線程*/
            notify = 1;
            res = 0;
        }
    }
    ......
}

int
evmap_io_add(struct event_base *base, evutil_socket_t fd, struct event *ev)
{
    ......
    /*一方面，將fd加入epoll中*/
    if (evsel->add(base, ev->ev_fd,
        old, (ev->ev_events & EV_ET) | res, extra) == -1)
        return (-1);
    ......

    /*另一方面，將event本身放入以fd爲key的hash數組隊列中
      在epoll_wait返回fd後，可以找到對應的event，然後加入
      active隊列*/
    ctx->nread = (ev_uint16_t) nread;
    ctx->nwrite = (ev_uint16_t) nwrite;
    TAILQ_INSERT_TAIL(&ctx->events, ev, ev_io_next);
   ......
}

int
evmap_signal_add(struct event_base *base, int sig, struct event *ev)
{
    const struct eventop *evsel = base->evsigsel;
    struct event_signal_map *map = &base->sigmap;
    struct evmap_signal *ctx = NULL;

    if (sig >= map->nentries) {
        if (evmap_make_space(
            map, sig, sizeof(struct evmap_signal *)) == -1)
            return (-1);
    }
    GET_SIGNAL_SLOT_AND_CTOR(ctx, map, sig, evmap_signal, evmap_signal_init,
        base->evsigsel->fdinfo_len);

    if (TAILQ_EMPTY(&ctx->events)) {
        if (evsel->add(base, ev->ev_fd, 0, EV_SIGNAL, NULL)
            == -1)
            return (-1);
    }

    TAILQ_INSERT_TAIL(&ctx->events, ev, ev_signal_next);

    return (1);
}

如果是fd事件，則通過evmap_io_add將fd加入到I/O複用函數中，
如果是信號事件，則通過evmap_signal_add將信號加入到I/O複用函數。
在evmap_io_add和evmap_signal_add分別調用evsel->add, 即epoll_nochangelist_add, epoll_nochangelist_add再調用epoll_apply_one_change，裏面調用epoll_ctl將對應的fd或者信號加入到epoll中

static int
epoll_nochangelist_add(struct event_base *base, evutil_socket_t fd,
    short old, short events, void *p)
{
    struct event_change ch;
    ch.fd = fd;
    ch.old_events = old;
    ch.read_change = ch.write_change = 0;
    if (events & EV_WRITE)
        ch.write_change = EV_CHANGE_ADD |
            (events & EV_ET);
    if (events & EV_READ)
        ch.read_change = EV_CHANGE_ADD |
            (events & EV_ET);
    /*注意這裏傳入的參數就是base->evbase, 這個就是後端需要使用的數據 ：）*/
    return epoll_apply_one_change(base, base->evbase, &ch);                                                                                                                           
}

static int
epoll_apply_one_change(struct event_base *base,
    struct epollop *epollop,
    const struct event_change *ch)
{
    struct epoll_event epev;
    int op, events = 0;

    if (1) {
        ......

        memset(&epev, 0, sizeof(epev));
        epev.data.fd = ch->fd;
        epev.events = events;
        if (epoll_ctl(epollop->epfd, op, ch->fd, &epev) == -1) {
            ......
        }
        ......
    }
    return 0;
}

注意這裏只是將事件號（fd或者信號值）和事件類型賦值給了epoll_event結構。意思就是說在epoll判斷有數據的時候，返回的就是fd或者信號值和事件類型，通過fd或者信號值找到對應的hash數組鏈表，然後在衝突鏈中根據事件類型找到對應的event

四、事件處理函數
調用event_base_dispatch，等待事件多路分發器返回激活事件，然後調用對應事件的事件處理函數(*ev->ev_callback).

event_base_dispath調用函數event_base_loop，event_base_loop調用evsel->dispatch函數, linux 下實際就是調用epoll_dispatch， epoll_dispatch調用epoll_wait。epoll_dispatch返回激活的事件隊列，然後調用event_process_active 函數依次處理就緒的信號事件和I/O事件

int
event_base_loop(struct event_base *base, int flags)
{
    const struct eventop *evsel = base->evsel;
    while (!done) {
        ......
        /*調用事件多路分發器的dispatch方法等待事件， 將就緒事件插入活動事件隊列*/
        res = evsel->dispatch(base, tv_p);
        ......
     if (N_ACTIVE_CALLBACKS(base)) {
      /*調用event_process_active 函數依次處理就緒的信號事件和I/O事件*/
      int n = event_process_active(base);
      if ((flags & EVLOOP_ONCE)
          && N_ACTIVE_CALLBACKS(base) == 0
          && n != 0)
          done = 1;
      } else if (flags & EVLOOP_NONBLOCK)
          done = 1;
    }
}

static int
epoll_dispatch(struct event_base *base, struct timeval *tv)
{
    struct epollop *epollop = base->evbase;
    struct epoll_event *events = epollop->events;
    int i, res;
    long timeout = -1;

     ......
    /*調用epoll_wait等待事件發生*/
    res = epoll_wait(epollop->epfd, events, epollop->nevents, timeout);

    ......
   /*事件發生後， 找出fd對應的event事件，並將該事件添加到active隊列*/
    for (i = 0; i < res; i++) {
        int what = events[i].events;
        short ev = 0;
         /*此處有點奇怪，爲啥EPOLLHUP和EPOLLERR，認爲是
           同時可讀可寫*/
        if (what & (EPOLLHUP|EPOLLERR)) {
            ev = EV_READ | EV_WRITE;
        } else {
            if (what & EPOLLIN)
                ev |= EV_READ;
            if (what & EPOLLOUT)
                ev |= EV_WRITE;
        }

        if (!ev)
            continue;

        evmap_io_active(base, events[i].data.fd, ev | EV_ET);
    }

    ......

    return (0);
}
/*在epoll判斷有數據的時候，返回的就是fd或者信號值和事件類型，通過fd或者信號值找到對應的hash數組鏈表，然後在衝突鏈中根據事件類型找到對應的event*/
evmap_io_active(struct event_base *base, evutil_socket_t fd, short events)
{
    struct event_io_map *io = &base->io;                                                                                                                                              
    struct evmap_io *ctx;
    struct event *ev;

#ifndef EVMAP_USE_HT
    EVUTIL_ASSERT(fd < io->nentries);
#endif
    GET_IO_SLOT(ctx, io, fd, evmap_io);

    EVUTIL_ASSERT(ctx);
    TAILQ_FOREACH(ev, &ctx->events, ev_io_next) {
        if (ev->ev_events & events)
            event_active_nolock(ev, ev->ev_events & events, 1);
    }
}
event_active_nolock將event插入到&base->activequeues隊列中

void
event_active_nolock(struct event *ev, int res, short ncalls)
{
    struct event_base *base;

    event_debug(("event_active: %p (fd "EV_SOCK_FMT"), res %d, callback %p",
        ev, EV_SOCK_ARG(ev->ev_fd), (int)res, ev->ev_callback));


    /* We get different kinds of events, add them together */
    if (ev->ev_flags & EVLIST_ACTIVE) {
        ev->ev_res |= res;
        return;
    }

    base = ev->ev_base;

    EVENT_BASE_ASSERT_LOCKED(base);

    ev->ev_res = res;

    if (ev->ev_pri < base->event_running_priority)
        base->event_continue = 1;

    if (ev->ev_events & EV_SIGNAL) {
#ifndef _EVENT_DISABLE_THREAD_SUPPORT
        if (base->current_event == ev && !EVBASE_IN_THREAD(base)) {
            ++base->current_event_waiters;
            EVTHREAD_COND_WAIT(base->current_event_cond, base->th_base_lock);
        }
#endif
        ev->ev_ncalls = ncalls;
        ev->ev_pncalls = NULL;
    }
    /*根據優先級插入激活隊列*/
    event_queue_insert(base, ev, EVLIST_ACTIVE);

    if (EVBASE_NEED_NOTIFY(base))
        evthread_notify_base(base);
}

static int
event_process_active(struct event_base *base)
{
    /* Caller must hold th_base_lock */
    struct event_list *activeq = NULL;
    int i, c = 0;
   /*循環處理base->activequeues隊列*/
    for (i = 0; i < base->nactivequeues; ++i) {
        if (TAILQ_FIRST(&base->activequeues[i]) != NULL) {
            base->event_running_priority = i;
            activeq = &base->activequeues[i];
            /*調用event_process_active_single_queue處理激活事件*/
            c = event_process_active_single_queue(base, activeq);                                                                                                        
            if (c < 0) {
                base->event_running_priority = -1;
                return -1;
            } else if (c > 0)
                break; /* Processed a real event; do not
                    * consider lower-priority events */
            /* If we get here, all of the events we processed
             * were internal.  Continue. */
        }
    }

    event_process_deferred_callbacks(&base->defer_queue,&base->event_break);
    base->event_running_priority = -1;
    return c;
}
static int
event_process_active_single_queue(struct event_base *base,
    struct event_list *activeq)
{
    struct event *ev;
    int count = 0;

    EVUTIL_ASSERT(activeq != NULL);

    for (ev = TAILQ_FIRST(activeq); ev; ev = TAILQ_FIRST(activeq)) {
        if (ev->ev_events & EV_PERSIST)
            event_queue_remove(base, ev, EVLIST_ACTIVE);
        else
            event_del_internal(ev);
        if (!(ev->ev_flags & EVLIST_INTERNAL))
            ++count;

        event_debug((
             "event_process_active: event: %p, %s%scall %p",
            ev,
            ev->ev_res & EV_READ ? "EV_READ " : " ",
            ev->ev_res & EV_WRITE ? "EV_WRITE " : " ",
            ev->ev_callback));

#ifndef _EVENT_DISABLE_THREAD_SUPPORT
        base->current_event = ev;
        base->current_event_waiters = 0;
#endif

        switch (ev->ev_closure) {
        case EV_CLOSURE_SIGNAL:
            event_signal_closure(base, ev);
            break;
        case EV_CLOSURE_PERSIST:
            event_persist_closure(base, ev);
            break;
        default:
        case EV_CLOSURE_NONE:
            EVBASE_RELEASE_LOCK(base, th_base_lock);
            /*調用事件處理函數*/
            (*ev->ev_callback)(
                ev->ev_fd, ev->ev_res, ev->ev_arg);
            break;
        }

        EVBASE_ACQUIRE_LOCK(base, th_base_lock);
#ifndef _EVENT_DISABLE_THREAD_SUPPORT
        base->current_event = NULL;
        if (base->current_event_waiters) {
            base->current_event_waiters = 0;
            EVTHREAD_COND_BROADCAST(base->current_event_cond);
        }
#endif

        if (base->event_break)
            return -1;
        if (base->event_continue)
            break;
    }
    return count;
}

[email protected]:~/test/libevent/my_libevent_test>more libevent_test_io.c 
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <sys/types.h>
#include <unistd.h>
#include <event.h>
#include <string.h>

#define SERVER_PORT 8888
#define BACKLOG 1024
#define MAX_BUFFER_LEN  1024
void do_process(evutil_socket_t fd, short event, void *arg)
{
    char buff[MAX_BUFFER_LEN];
    ssize_t len;

    memset(buff, 0, MAX_BUFFER_LEN);
    len = recv(fd, buff, MAX_BUFFER_LEN, 0);

    if (len > 0) {
        printf("%s\n", buff);
        send(fd, buff, len, 0);
    }

    return;
}

void do_accept(evutil_socket_t fd, short event, void *arg)
{
    evutil_socket_t connfd;
    struct event_base *base;
    struct event *ev;
    struct sockaddr_in client;
    socklen_t client_addr_len;

    base = (struct event_base*)arg;

    printf("do accept for fd %d\n", fd);

    client_addr_len = sizeof(struct sockaddr_in);
    connfd = accept(fd, (struct sockaddr*)&client, &client_addr_len);

    if (connfd < 0) {
        printf("Error: Accept socket error!\n");
        return;
    }
    ev = event_new(base, connfd, EV_READ | EV_PERSIST, do_process, (void*)base);

    event_add(ev, NULL);

    return;
}

int create_listen_fd_event(struct event_base *base)
{
    evutil_socket_t fd;
    struct sockaddr_in server;
    struct event *event;


    fd = socket(AF_INET, SOCK_STREAM, 0);
    if (fd < 0) {
        printf("Error: Create socket error!\n");
        return -1;
    }
    bzero(&server, sizeof(struct sockaddr_in));
    server.sin_family = AF_INET;
    server.sin_port = htons(SERVER_PORT);
    inet_pton(AF_INET, "127.0.0.1", &server.sin_addr.s_addr);

    if (bind(fd, (struct sockaddr*)&server, sizeof(server)) < 0) {
        printf("Error: Bind socket error!\n");
        return -1;
    }

    if (listen(fd, BACKLOG) < 0) {
        printf("Error: listen socket error!\n");
        return -1;
    }
    event = event_new(base, fd, EV_READ | EV_PERSIST, do_accept, (void*)base);
    event_add(event, NULL);
    return 0;
}

int main(int argc, char *argv[])
{
    struct event_base *base;

    base = event_base_new();
    if (!base) {
        printf("Error: Create an event base error!\n");
        return -1;
    }
    create_listen_fd_event(base);
    event_base_dispatch(base);


    return 0;
}