*) handle unaligned file part for directio
*) disable sendfile in directio mode
diff --git a/src/core/ngx_buf.h b/src/core/ngx_buf.h
index ec55ef3..d16656d 100644
--- a/src/core/ngx_buf.h
+++ b/src/core/ngx_buf.h
@@ -78,6 +78,7 @@
ngx_chain_t *busy;
unsigned sendfile;
+ unsigned directio;
unsigned need_in_memory;
unsigned need_in_temp;
diff --git a/src/core/ngx_file.h b/src/core/ngx_file.h
index a254ca7..8e08398 100644
--- a/src/core/ngx_file.h
+++ b/src/core/ngx_file.h
@@ -26,7 +26,8 @@
ngx_log_t *log;
- ngx_uint_t valid_info; /* unsigned valid_info:1; */
+ unsigned valid_info:1;
+ unsigned directio:1;
};
#define NGX_MAX_PATH_LEVEL 3
diff --git a/src/core/ngx_open_file_cache.c b/src/core/ngx_open_file_cache.c
index f2e8afc..614484d 100644
--- a/src/core/ngx_open_file_cache.c
+++ b/src/core/ngx_open_file_cache.c
@@ -502,6 +502,9 @@
if (ngx_directio(fd) == -1) {
ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
ngx_directio_n " \"%s\" failed", name);
+
+ } else {
+ of->is_directio = 1;
}
}
}
diff --git a/src/core/ngx_open_file_cache.h b/src/core/ngx_open_file_cache.h
index 1e1a279..b15de1d 100644
--- a/src/core/ngx_open_file_cache.h
+++ b/src/core/ngx_open_file_cache.h
@@ -33,6 +33,7 @@
unsigned is_file:1;
unsigned is_link:1;
unsigned is_exec:1;
+ unsigned is_directio:1;
} ngx_open_file_info_t;
diff --git a/src/core/ngx_output_chain.c b/src/core/ngx_output_chain.c
index b3bb850..e0f615e 100644
--- a/src/core/ngx_output_chain.c
+++ b/src/core/ngx_output_chain.c
@@ -18,13 +18,14 @@
static ngx_inline ngx_int_t
- ngx_output_chain_need_to_copy(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
+ ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf);
static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool,
ngx_chain_t **chain, ngx_chain_t *in);
+static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx,
+ off_t bsize);
static ngx_int_t ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx,
off_t bsize);
-static ngx_int_t ngx_output_chain_copy_buf(ngx_buf_t *dst, ngx_buf_t *src,
- ngx_uint_t sendfile);
+static ngx_int_t ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx);
ngx_int_t
@@ -50,7 +51,7 @@
#if (NGX_SENDFILE_LIMIT)
&& !(in->buf->in_file && in->buf->file_last > NGX_SENDFILE_LIMIT)
#endif
- && !ngx_output_chain_need_to_copy(ctx, in->buf))
+ && ngx_output_chain_as_is(ctx, in->buf))
{
return ctx->output_filter(ctx->filter_ctx, in);
}
@@ -74,7 +75,7 @@
/*
* cycle while there are the ctx->in bufs
- * or there are the free output bufs to copy in
+ * and there are the free output bufs to copy in
*/
bsize = ngx_buf_size(ctx->in->buf);
@@ -101,7 +102,7 @@
continue;
}
- if (!ngx_output_chain_need_to_copy(ctx, ctx->in->buf)) {
+ if (ngx_output_chain_as_is(ctx, ctx->in->buf)) {
/* move the chain link to the output chain */
@@ -117,25 +118,35 @@
if (ctx->buf == NULL) {
- /* get the free buf */
+ rc = ngx_output_chain_align_file_buf(ctx, bsize);
- if (ctx->free) {
- cl = ctx->free;
- ctx->buf = cl->buf;
- ctx->free = cl->next;
- ngx_free_chain(ctx->pool, cl);
-
- } else if (out || ctx->allocated == ctx->bufs.num) {
-
- break;
-
- } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
+ if (rc == NGX_ERROR) {
return NGX_ERROR;
}
+
+ if (rc != NGX_OK) {
+
+ if (ctx->free) {
+
+ /* get the free buf */
+
+ cl = ctx->free;
+ ctx->buf = cl->buf;
+ ctx->free = cl->next;
+
+ ngx_free_chain(ctx->pool, cl);
+
+ } else if (out || ctx->allocated == ctx->bufs.num) {
+
+ break;
+
+ } else if (ngx_output_chain_get_buf(ctx, bsize) != NGX_OK) {
+ return NGX_ERROR;
+ }
+ }
}
- rc = ngx_output_chain_copy_buf(ctx->buf, ctx->in->buf,
- ctx->sendfile);
+ rc = ngx_output_chain_copy_buf(ctx);
if (rc == NGX_ERROR) {
return rc;
@@ -189,11 +200,15 @@
static ngx_inline ngx_int_t
-ngx_output_chain_need_to_copy(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
+ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf)
{
ngx_uint_t sendfile;
if (ngx_buf_special(buf)) {
+ return 1;
+ }
+
+ if (buf->in_file && buf->file->directio) {
return 0;
}
@@ -210,21 +225,21 @@
if (!sendfile) {
if (!ngx_buf_in_memory(buf)) {
- return 1;
+ return 0;
}
buf->in_file = 0;
}
if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) {
- return 1;
+ return 0;
}
if (ctx->need_in_temp && (buf->memory || buf->mmap)) {
- return 1;
+ return 0;
}
- return 0;
+ return 1;
}
@@ -258,6 +273,8 @@
&& buf->file_pos < NGX_SENDFILE_LIMIT
&& buf->file_last > NGX_SENDFILE_LIMIT)
{
+ /* split a file buf on two bufs by the sendfile limit */
+
b = ngx_calloc_buf(pool);
if (b == NULL) {
return NGX_ERROR;
@@ -297,32 +314,82 @@
static ngx_int_t
+ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
+{
+ size_t size;
+ ngx_buf_t *in;
+
+ in = ctx->in->buf;
+
+ if (in->file == NULL || !in->file->directio) {
+ return NGX_DECLINED;
+ }
+
+ ctx->directio = 1;
+
+ size = (size_t) (in->file_pos - (in->file_pos & ~511));
+
+ if (size == 0) {
+
+ if (bsize >= ctx->bufs.size) {
+ return NGX_DECLINED;
+ }
+
+ size = (size_t) bsize;
+
+ } else {
+ size = 512 - size;
+
+ if (size > bsize) {
+ size = (size_t) bsize;
+ }
+ }
+
+ ctx->buf = ngx_create_temp_buf(ctx->pool, size);
+ if (ctx->buf == NULL) {
+ return NGX_ERROR;
+ }
+
+ /*
+ * we do not set ctx->buf->tag, because we do not want
+ * to reuse the buf via ctx->free list
+ */
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
ngx_output_chain_get_buf(ngx_output_chain_ctx_t *ctx, off_t bsize)
{
size_t size;
- ngx_buf_t *b;
+ ngx_buf_t *b, *in;
ngx_uint_t recycled;
+ in = ctx->in->buf;
size = ctx->bufs.size;
recycled = 1;
- if (ctx->in->buf->last_in_chain) {
+ if (in->last_in_chain) {
if (bsize < (off_t) size) {
- /*
- * allocate a small temp buf for a small last buf
- * or its small last part
- */
+ /*
+ * allocate a small temp buf for a small last buf
+ * or its small last part
+ */
size = (size_t) bsize;
recycled = 0;
- } else if (ctx->bufs.num == 1 && (bsize < (off_t) (size + size / 4))) {
-
+ } else if (!ctx->directio
+ && ctx->bufs.num == 1
+ && (bsize < (off_t) (size + size / 4)))
+ {
/*
- * allocate a temp buf that equals to a last buf, if the last buf
- * size is lesser than 1.25 of bufs.size and the temp buf is single
+ * allocate a temp buf that equals to a last buf,
+ * if there is no directio, the last buf size is lesser
+ * than 1.25 of bufs.size and the temp buf is single
*/
size = (size_t) bsize;
@@ -335,11 +402,23 @@
return NGX_ERROR;
}
- /* allocate block aligned to a disk sector size to enable O_DIRECT */
+ if (ctx->directio) {
- b->start = ngx_pmemalign(ctx->pool, size, 512);
- if (b->start == NULL) {
- return NGX_ERROR;
+ /*
+ * allocate block aligned to a disk sector size to enable
+ * userland buffer direct usage conjunctly with directio
+ */
+
+ b->start = ngx_pmemalign(ctx->pool, size, 512);
+ if (b->start == NULL) {
+ return NGX_ERROR;
+ }
+
+ } else {
+ b->start = ngx_palloc(ctx->pool, size);
+ if (b->start == NULL) {
+ return NGX_ERROR;
+ }
}
b->pos = b->start;
@@ -357,10 +436,15 @@
static ngx_int_t
-ngx_output_chain_copy_buf(ngx_buf_t *dst, ngx_buf_t *src, ngx_uint_t sendfile)
+ngx_output_chain_copy_buf(ngx_output_chain_ctx_t *ctx)
{
- off_t size;
- ssize_t n;
+ off_t size;
+ ssize_t n;
+ ngx_buf_t *src, *dst;
+ ngx_uint_t sendfile;
+
+ src = ctx->in->buf;
+ dst = ctx->buf;
size = ngx_buf_size(src);
@@ -368,6 +452,8 @@
size = dst->end - dst->pos;
}
+ sendfile = ctx->sendfile & !ctx->directio;
+
#if (NGX_SENDFILE_LIMIT)
if (src->in_file && src->file_pos >= NGX_SENDFILE_LIMIT) {
@@ -418,7 +504,7 @@
#endif
if (n != size) {
- ngx_log_error(NGX_LOG_ALERT, src->file->log, 0,
+ ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0,
ngx_read_file_n " read only %z of %O from file",
n, size);
if (n == 0) {
diff --git a/src/http/modules/ngx_http_flv_module.c b/src/http/modules/ngx_http_flv_module.c
index dceb68a..3cc7d81 100644
--- a/src/http/modules/ngx_http_flv_module.c
+++ b/src/http/modules/ngx_http_flv_module.c
@@ -244,6 +244,7 @@
b->file->fd = of.fd;
b->file->name = path;
b->file->log = log;
+ b->file->directio = of.is_directio;
out[1].buf = b;
out[1].next = NULL;
diff --git a/src/http/modules/ngx_http_gzip_static_module.c b/src/http/modules/ngx_http_gzip_static_module.c
index bc3a65f..affb766 100644
--- a/src/http/modules/ngx_http_gzip_static_module.c
+++ b/src/http/modules/ngx_http_gzip_static_module.c
@@ -235,6 +235,7 @@
b->file->fd = of.fd;
b->file->name = path;
b->file->log = log;
+ b->file->directio = of.is_directio;
out.buf = b;
out.next = NULL;
diff --git a/src/http/modules/ngx_http_static_module.c b/src/http/modules/ngx_http_static_module.c
index 7c2768d..9ff1f81 100644
--- a/src/http/modules/ngx_http_static_module.c
+++ b/src/http/modules/ngx_http_static_module.c
@@ -251,6 +251,7 @@
b->file->fd = of.fd;
b->file->name = path;
b->file->log = log;
+ b->file->directio = of.is_directio;
out.buf = b;
out.next = NULL;
diff --git a/src/http/modules/perl/nginx.xs b/src/http/modules/perl/nginx.xs
index fbc7f59..57aabec 100644
--- a/src/http/modules/perl/nginx.xs
+++ b/src/http/modules/perl/nginx.xs
@@ -681,6 +681,7 @@
b->file->fd = of.fd;
b->file->log = r->connection->log;
+ b->file->directio = of.is_directio;
(void) ngx_http_perl_output(r, b);
