FreeBSD and Linux AIO support
diff --git a/src/os/unix/ngx_file_aio_read.c b/src/os/unix/ngx_file_aio_read.c
new file mode 100644
index 0000000..bf089ec
--- /dev/null
+++ b/src/os/unix/ngx_file_aio_read.c
@@ -0,0 +1,210 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+/*
+ * FreeBSD file AIO features and quirks:
+ *
+ * if an asked data are already in VM cache, then aio_error() returns 0,
+ * and the data are already copied in buffer;
+ *
+ * aio_read() preread in VM cache as minimum 32K;
+ *
+ * aio_read/aio_error() may return EINPROGRESS for just written data;
+ *
+ * kqueue EVFILT_AIO filter is level triggered only: an event repeats
+ * until aio_return() will be called;
+ *
+ * aio_cancel() can not cancel file AIO: it returns AIO_NOTCANCELED always.
+ */
+
+
+extern int ngx_kqueue;
+
+
+static ssize_t ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio,
+ ngx_event_t *ev);
+static void ngx_file_aio_event_handler(ngx_event_t *ev);
+
+
+ssize_t
+ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
+ ngx_pool_t *pool)
+{
+ int n;
+ ngx_event_t *ev;
+ ngx_event_aio_t *aio;
+ static ngx_uint_t enosys = 0;
+
+ if (enosys) {
+ return ngx_read_file(file, buf, size, offset);
+ }
+
+ aio = file->aio;
+
+ if (aio == NULL) {
+ aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+ if (aio == NULL) {
+ return NGX_ERROR;
+ }
+
+ aio->file = file;
+ aio->fd = file->fd;
+ aio->event.data = aio;
+ aio->event.ready = 1;
+ aio->event.log = file->log;
+ file->aio = aio;
+ }
+
+ ev = &aio->event;
+
+ if (!ev->ready) {
+ ngx_log_error(NGX_LOG_ALERT, file->log, 0,
+ "second aio post for \"%V\"", &file->name);
+ return NGX_AGAIN;
+ }
+
+ ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0,
+ "aio complete:%d @%O:%z %V",
+ ev->complete, offset, size, &file->name);
+
+ if (ev->complete) {
+ ev->complete = 0;
+ ngx_set_errno(aio->err);
+
+ if (aio->err == 0) {
+ return aio->nbytes;
+ }
+
+ return NGX_ERROR;
+ }
+
+ ngx_memzero(&aio->aiocb, sizeof(struct aiocb));
+
+ aio->aiocb.aio_fildes = file->fd;
+ aio->aiocb.aio_offset = offset;
+ aio->aiocb.aio_buf = buf;
+ aio->aiocb.aio_nbytes = size;
+#if (NGX_HAVE_KQUEUE)
+ aio->aiocb.aio_sigevent.sigev_notify_kqueue = ngx_kqueue;
+ aio->aiocb.aio_sigevent.sigev_notify = SIGEV_KEVENT;
+ aio->aiocb.aio_sigevent.sigev_value.sigval_ptr = ev;
+#endif
+ ev->handler = ngx_file_aio_event_handler;
+
+ n = aio_read(&aio->aiocb);
+
+ if (n == -1) {
+ n = ngx_errno;
+
+ if (n == NGX_EAGAIN) {
+ return ngx_read_file(file, buf, size, offset);
+ }
+
+ ngx_log_error(NGX_LOG_CRIT, file->log, n,
+ "aio_read(\"%V\") failed", &file->name);
+
+ if (n == NGX_ENOSYS) {
+ enosys = 1;
+ return ngx_read_file(file, buf, size, offset);
+ }
+
+ return NGX_ERROR;
+ }
+
+ ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+ "aio_read: fd:%d %d", file->fd, n);
+
+ ev->active = 1;
+ ev->ready = 0;
+ ev->complete = 0;
+
+ return ngx_file_aio_result(aio->file, aio, ev);
+}
+
+
+static ssize_t
+ngx_file_aio_result(ngx_file_t *file, ngx_event_aio_t *aio, ngx_event_t *ev)
+{
+ int n;
+ ngx_err_t err;
+
+ n = aio_error(&aio->aiocb);
+
+ ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+ "aio_error: fd:%d %d", file->fd, n);
+
+ if (n == -1) {
+ err = ngx_errno;
+ aio->err = err;
+
+ ngx_log_error(NGX_LOG_ALERT, file->log, err,
+ "aio_error(\"%V\") failed", &file->name);
+ return NGX_ERROR;
+ }
+
+ if (n != 0) {
+ if (n == NGX_EINPROGRESS) {
+ if (ev->ready) {
+ ev->ready = 0;
+ ngx_log_error(NGX_LOG_ALERT, file->log, n,
+ "aio_read(\"%V\") still in progress",
+ &file->name);
+ }
+
+ return NGX_AGAIN;
+ }
+
+ aio->err = n;
+ ev->ready = 0;
+
+ ngx_log_error(NGX_LOG_CRIT, file->log, n,
+ "aio_read(\"%V\") failed", &file->name);
+ return NGX_ERROR;
+ }
+
+ n = aio_return(&aio->aiocb);
+
+ if (n == -1) {
+ err = ngx_errno;
+ aio->err = err;
+ ev->ready = 0;
+
+ ngx_log_error(NGX_LOG_ALERT, file->log, err,
+ "aio_return(\"%V\") failed", &file->name);
+ return NGX_ERROR;
+ }
+
+ aio->err = 0;
+ aio->nbytes = n;
+ ev->ready = 1;
+ ev->active = 0;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_CORE, file->log, 0,
+ "aio_return: fd:%d %d", file->fd, n);
+
+ return n;
+}
+
+
+static void
+ngx_file_aio_event_handler(ngx_event_t *ev)
+{
+ ngx_event_aio_t *aio;
+
+ aio = ev->data;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0,
+ "aio event handler fd:%d %V", aio->fd, &aio->file->name);
+
+ if (ngx_file_aio_result(aio->file, aio, ev) != NGX_AGAIN) {
+ aio->handler(ev);
+ }
+}
diff --git a/src/os/unix/ngx_files.h b/src/os/unix/ngx_files.h
index 5395acd..993ce8f 100644
--- a/src/os/unix/ngx_files.h
+++ b/src/os/unix/ngx_files.h
@@ -287,4 +287,12 @@
#define ngx_set_stderr_n "dup2(STDERR_FILENO)"
+#if (NGX_HAVE_FILE_AIO)
+
+ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size,
+ off_t offset, ngx_pool_t *pool);
+
+#endif
+
+
#endif /* _NGX_FILES_H_INCLUDED_ */
diff --git a/src/os/unix/ngx_freebsd_config.h b/src/os/unix/ngx_freebsd_config.h
index 997581a..20cb666 100644
--- a/src/os/unix/ngx_freebsd_config.h
+++ b/src/os/unix/ngx_freebsd_config.h
@@ -73,13 +73,14 @@
#endif
-#if (NGX_HAVE_AIO)
-#include <aio.h>
+#if (NGX_HAVE_KQUEUE)
+#include <sys/event.h>
#endif
-#if (NGX_HAVE_KQUEUE)
-#include <sys/event.h>
+#if (NGX_HAVE_FILE_AIO || NGX_HAVE_AIO)
+#include <aio.h>
+typedef struct aiocb ngx_aiocb_t;
#endif
diff --git a/src/os/unix/ngx_linux_aio_read.c b/src/os/unix/ngx_linux_aio_read.c
new file mode 100644
index 0000000..3b4185c
--- /dev/null
+++ b/src/os/unix/ngx_linux_aio_read.c
@@ -0,0 +1,131 @@
+
+/*
+ * Copyright (C) Igor Sysoev
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_event.h>
+
+
+extern int ngx_eventfd;
+extern aio_context_t ngx_aio_ctx;
+
+
+static void ngx_file_aio_event_handler(ngx_event_t *ev);
+
+
+static long
+io_submit(aio_context_t ctx, long n, struct iocb **paiocb)
+{
+ return syscall(SYS_io_submit, ctx, n, paiocb);
+}
+
+
+ssize_t
+ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset,
+ ngx_pool_t *pool)
+{
+ long n;
+ struct iocb *piocb[1];
+ ngx_event_t *ev;
+ ngx_event_aio_t *aio;
+ static ngx_uint_t enosys = 0;
+
+ if (enosys) {
+ return ngx_read_file(file, buf, size, offset);
+ }
+
+ aio = file->aio;
+
+ if (aio == NULL) {
+ aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t));
+ if (aio == NULL) {
+ return NGX_ERROR;
+ }
+
+ aio->file = file;
+ aio->fd = file->fd;
+ aio->event.data = aio;
+ aio->event.ready = 1;
+ aio->event.log = file->log;
+ file->aio = aio;
+ }
+
+ ev = &aio->event;
+
+ if (!ev->ready) {
+ ngx_log_error(NGX_LOG_ALERT, file->log, 0,
+ "second aio post for \"%V\"", &file->name);
+ return NGX_AGAIN;
+ }
+
+ ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0,
+ "aio complete:%d @%O:%z %V",
+ ev->complete, offset, size, &file->name);
+
+ if (ev->complete) {
+ ev->active = 0;
+ ev->complete = 0;
+
+ if (aio->res >= 0) {
+ ngx_set_errno(0);
+ return aio->res;
+ }
+
+ ngx_set_errno(-aio->res);
+ return NGX_ERROR;
+ }
+
+ ngx_memzero(&aio->aiocb, sizeof(struct iocb));
+
+ aio->aiocb.aio_data = (uint64_t) (uintptr_t) ev;
+ aio->aiocb.aio_lio_opcode = IOCB_CMD_PREAD;
+ aio->aiocb.aio_fildes = file->fd;
+ aio->aiocb.aio_buf = (uint64_t) (uintptr_t) buf;
+ aio->aiocb.aio_nbytes = size;
+ aio->aiocb.aio_offset = offset;
+ aio->aiocb.aio_flags = IOCB_FLAG_RESFD;
+ aio->aiocb.aio_resfd = ngx_eventfd;
+
+ ev->handler = ngx_file_aio_event_handler;
+
+ piocb[0] = &aio->aiocb;
+
+ n = io_submit(ngx_aio_ctx, 1, piocb);
+
+ if (n == 1) {
+ return NGX_AGAIN;
+ }
+
+ n = -n;
+
+ if (n == NGX_EAGAIN) {
+ return ngx_read_file(file, buf, size, offset);
+ }
+
+ ngx_log_error(NGX_LOG_CRIT, file->log, n,
+ "io_submit(\"%V\") failed", &file->name);
+
+ if (n == NGX_ENOSYS) {
+ enosys = 1;
+ return ngx_read_file(file, buf, size, offset);
+ }
+
+ return NGX_ERROR;
+}
+
+
+static void
+ngx_file_aio_event_handler(ngx_event_t *ev)
+{
+ ngx_event_aio_t *aio;
+
+ aio = ev->data;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_CORE, ev->log, 0,
+ "aio event handler fd:%d %V", aio->fd, &aio->file->name);
+
+ aio->handler(ev);
+}
diff --git a/src/os/unix/ngx_linux_config.h b/src/os/unix/ngx_linux_config.h
index fb978c4..cf45e54 100644
--- a/src/os/unix/ngx_linux_config.h
+++ b/src/os/unix/ngx_linux_config.h
@@ -81,6 +81,13 @@
#endif
+#if (NGX_HAVE_FILE_AIO)
+#include <sys/syscall.h>
+#include <linux/aio_abi.h>
+typedef struct iocb ngx_aiocb_t;
+#endif
+
+
#define NGX_LISTEN_BACKLOG 511
diff --git a/src/os/unix/ngx_posix_config.h b/src/os/unix/ngx_posix_config.h
index 3644151..86a80d7 100644
--- a/src/os/unix/ngx_posix_config.h
+++ b/src/os/unix/ngx_posix_config.h
@@ -112,6 +112,12 @@
#endif
+#if (NGX_HAVE_FILE_AIO)
+#include <aio.h>
+typedef struct aiocb ngx_aiocb_t;
+#endif
+
+
#define NGX_LISTEN_BACKLOG 511
diff --git a/src/os/unix/ngx_solaris_config.h b/src/os/unix/ngx_solaris_config.h
index 663f265..ae8212d 100644
--- a/src/os/unix/ngx_solaris_config.h
+++ b/src/os/unix/ngx_solaris_config.h
@@ -62,16 +62,6 @@
#endif
-#if (NGX_HAVE_SENDFILE)
-#include <sys/sendfile.h>
-#endif
-
-
-#if (NGX_HAVE_AIO)
-#include <aio.h>
-#endif
-
-
#if (NGX_HAVE_DEVPOLL)
#include <sys/ioctl.h>
#include <sys/devpoll.h>
@@ -83,6 +73,11 @@
#endif
+#if (NGX_HAVE_SENDFILE)
+#include <sys/sendfile.h>
+#endif
+
+
#define NGX_LISTEN_BACKLOG 511
