Songjinshan's Blog

初始化

shmtx应该是shared memory和mutex的缩写，如果开多个worker process，各worker process之间同步要用到这种mutex。

首先在master process中开一段共享内存：

#0  ngx_shm_alloc (shm=0xbfd01a64) at src/os/unix/ngx_shmem.c:16
#1  0x08064619 in ngx_event_module_init (cycle=0x950cbf8) at src/event/ngx_event.c:513
#2  0x0805af4b in ngx_init_cycle (old_cycle=0xbfd01c9c) at src/core/ngx_cycle.c:592
#3  0x0804a492 in main (argc=1, argv=0xbfd01e24) at src/core/nginx.c:317
 

在ngx_shm_alloc中：

    shm->addr = (u_char *) mmap(NULL, shm->size,
                                PROT_READ|PROT_WRITE,
                                MAP_ANON|MAP_SHARED, -1, 0);

在ngx_event_module_init中：

    if (ngx_shm_alloc(&shm) != NGX_OK) {
        return NGX_ERROR;
    }

    shared = shm.addr;

    ngx_accept_mutex_ptr = (ngx_atomic_t *) shared;

    if (ngx_shmtx_create(&ngx_accept_mutex, shared, cycle->lock_file.data)
        != NGX_OK)
    {
        return NGX_ERROR;
    }
 

在ngx_shmtx_create中，将shmtx的lock指向共享内存的首地址（共享内存的开头是ngx_accept_mutex的lock，后半部分还有一些统计信息）：

    mtx->lock = addr;

处理accept事件

ngx_posted_accept_events是一个event队列，由ngx_accept_mutex保护，在访问前需要调用ngx_trylock_accept_mutex加锁：

#0  ngx_shmtx_trylock (mtx=0x80caf24) at src/core/ngx_shmtx.h:34
#1  0x08066622 in ngx_trylock_accept_mutex (cycle=0x9d62bf8) at src/event/ngx_event_accept.c:261
#2  0x08063ec2 in ngx_process_events_and_timers (cycle=0x9d62bf8) at src/event/ngx_event.c:226
#3  0x0806cf75 in ngx_worker_process_cycle (cycle=0x9d62bf8, data=0x0) at src/os/unix/ngx_process_cycle.c:775
#4  0x0806a8c9 in ngx_spawn_process (cycle=0x9d62bf8, proc=0x806ce6a <ngx_worker_process_cycle>, data=0x0, name=0x80ba1b9 "worker process", respawn=-2)
    at src/os/unix/ngx_process.c:194
#5  0x0806c509 in ngx_start_worker_processes (cycle=0x9d62bf8, n=2, type=-2) at src/os/unix/ngx_process_cycle.c:331
#6  0x0806be26 in ngx_master_process_cycle (cycle=0x9d62bf8) at src/os/unix/ngx_process_cycle.c:123
#7  0x0804a69c in main (argc=1, argv=0xbffbb8d4) at src/core/nginx.c:382
 

然后ngx_process_events_and_timers调用ngx_process_events处理事件，处理之后调用ngx_shmtx_unlock解锁。

ngx_trylock_accept_mutex中调用ngx_shmtx_trylock：

    return (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid));

ngx_atomic_cmp_set在ngx_gcc_atomic_x86.h中定义：

/*
 * "cmpxchgl  r, [m]":
 *
 *     if (eax == [m]) {
 *         zf = 1;
 *         [m] = r;
 *     } else {
 *         zf = 0;
 *         eax = [m];
 *     }
 *
 *
 * The "r" means the general register.
 * The "=a" and "a" are the %eax register.
 * Although we can return result in any register, we use "a" because it is
 * used in cmpxchgl anyway.  The result is actually in %al but not in %eax,
 * however, as the code is inlined gcc can test %al as well as %eax,
 * and icc adds "movzbl %al, %eax" by itself.
 *
 * The "cc" means that flags were changed.
 */

static ngx_inline ngx_atomic_uint_t
ngx_atomic_cmp_set(ngx_atomic_t *lock, ngx_atomic_uint_t old,
    ngx_atomic_uint_t set)
{
    u_char  res;

    __asm__ volatile (

         NGX_SMP_LOCK
    "    cmpxchgl  %3, %1;   "
    "    sete      %0;       "

    : "=a" (res) : "m" (*lock), "a" (old), "r" (set) : "cc", "memory");

    return res;
}

解锁：

#define ngx_shmtx_unlock(mtx) (void) ngx_atomic_cmp_set((mtx)->lock, ngx_pid, 0)

处理其它事件

ngx_posted_events是另外一个event队列，存放除accept事件之外的其它事件，这个队列用锁ngx_posted_events_mutex保护，这并不是一个shmtx，而是pthread的mutex，用于线程间的同步。如果没有配置NGX_THREADS，则ngx_mutex_trylock、ngx_mutex_lock、ngx_mutex_unlock都被定义为空操作，详见src/os/unix/ngx_thread.h。如果配置了NGX_THREADS，各线程都往ngx_posted_events队列中添加待处理的事件，需要用这个mutex互斥。

问题

1. 为什么各进程的accept操作需要用mutex来serialize？

2. 为什么从ngx_posted_events队列中取走事件不需要加锁解锁？详见src/event/ngx_event_posted.h，ngx_post_event需要加锁解锁，而ngx_delete_posted_event不需要。猜想可能只有一个线程调用ngx_delete_posted_event。