Introduced the ngx_output_chain_to_iovec() function.
It deduplicates code of the send chain functions and uses only preallocated
memory, which completely solves the problem mentioned in d1bde5c3c5d2.
diff --git a/src/os/unix/ngx_darwin_sendfile_chain.c b/src/os/unix/ngx_darwin_sendfile_chain.c
index d28f6e1..d98fd74 100644
--- a/src/os/unix/ngx_darwin_sendfile_chain.c
+++ b/src/os/unix/ngx_darwin_sendfile_chain.c
@@ -31,17 +31,15 @@
ngx_darwin_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
{
int rc;
- u_char *prev;
off_t size, send, prev_send, aligned, sent, fprev;
- off_t header_size, file_size;
+ off_t file_size;
ngx_uint_t eintr;
ngx_err_t err;
ngx_buf_t *file;
- ngx_array_t header, trailer;
ngx_event_t *wev;
ngx_chain_t *cl;
+ ngx_iovec_t header, trailer;
struct sf_hdtr hdtr;
- struct iovec *iov;
struct iovec headers[NGX_IOVS_PREALLOCATE];
struct iovec trailers[NGX_IOVS_PREALLOCATE];
@@ -70,69 +68,27 @@
send = 0;
- header.elts = headers;
- header.size = sizeof(struct iovec);
+ header.iovs = headers;
header.nalloc = NGX_IOVS_PREALLOCATE;
- header.pool = c->pool;
- trailer.elts = trailers;
- trailer.size = sizeof(struct iovec);
+ trailer.iovs = trailers;
trailer.nalloc = NGX_IOVS_PREALLOCATE;
- trailer.pool = c->pool;
for ( ;; ) {
file = NULL;
file_size = 0;
- header_size = 0;
eintr = 0;
prev_send = send;
- header.nelts = 0;
- trailer.nelts = 0;
-
/* create the header iovec and coalesce the neighbouring bufs */
- prev = NULL;
- iov = NULL;
+ cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log);
- for (cl = in; cl && send < limit; cl = cl->next) {
-
- if (ngx_buf_special(cl->buf)) {
- continue;
- }
-
- if (!ngx_buf_in_memory_only(cl->buf)) {
- break;
- }
-
- size = cl->buf->last - cl->buf->pos;
-
- if (send + size > limit) {
- size = limit - send;
- }
-
- if (prev == cl->buf->pos) {
- iov->iov_len += (size_t) size;
-
- } else {
- if (header.nelts >= IOV_MAX) {
- break;
- }
-
- iov = ngx_array_push(&header);
- if (iov == NULL) {
- return NGX_CHAIN_ERROR;
- }
-
- iov->iov_base = (void *) cl->buf->pos;
- iov->iov_len = (size_t) size;
- }
-
- prev = cl->buf->pos + (size_t) size;
- header_size += size;
- send += size;
+ if (cl == NGX_CHAIN_ERROR) {
+ return NGX_CHAIN_ERROR;
}
+ send += header.size;
if (cl && cl->buf->in_file && send < limit) {
file = cl->buf;
@@ -165,51 +121,17 @@
&& fprev == cl->buf->file_pos);
}
- if (file && header.nelts == 0) {
+ if (file && header.count == 0) {
/* create the trailer iovec and coalesce the neighbouring bufs */
- prev = NULL;
- iov = NULL;
+ cl = ngx_output_chain_to_iovec(&trailer, cl, limit - send, c->log);
- while (cl && send < limit) {
-
- if (ngx_buf_special(cl->buf)) {
- cl = cl->next;
- continue;
- }
-
- if (!ngx_buf_in_memory_only(cl->buf)) {
- break;
- }
-
- size = cl->buf->last - cl->buf->pos;
-
- if (send + size > limit) {
- size = limit - send;
- }
-
- if (prev == cl->buf->pos) {
- iov->iov_len += (size_t) size;
-
- } else {
- if (trailer.nelts >= IOV_MAX) {
- break;
- }
-
- iov = ngx_array_push(&trailer);
- if (iov == NULL) {
- return NGX_CHAIN_ERROR;
- }
-
- iov->iov_base = (void *) cl->buf->pos;
- iov->iov_len = (size_t) size;
- }
-
- prev = cl->buf->pos + (size_t) size;
- send += size;
- cl = cl->next;
+ if (cl == NGX_CHAIN_ERROR) {
+ return NGX_CHAIN_ERROR;
}
+
+ send += trailer.size;
}
if (file) {
@@ -219,16 +141,16 @@
* but corresponding pointer is not NULL
*/
- hdtr.headers = header.nelts ? (struct iovec *) header.elts: NULL;
- hdtr.hdr_cnt = header.nelts;
- hdtr.trailers = trailer.nelts ? (struct iovec *) trailer.elts: NULL;
- hdtr.trl_cnt = trailer.nelts;
+ hdtr.headers = header.count ? header.iovs : NULL;
+ hdtr.hdr_cnt = header.count;
+ hdtr.trailers = trailer.count ? trailer.iovs : NULL;
+ hdtr.trl_cnt = trailer.count;
- sent = header_size + file_size;
+ sent = header.size + file_size;
ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0,
- "sendfile: @%O %O h:%O",
- file->file_pos, sent, header_size);
+ "sendfile: @%O %O h:%uz",
+ file->file_pos, sent, header.size);
rc = sendfile(file->file->fd, c->fd, file->file_pos,
&sent, &hdtr, 0);
@@ -271,13 +193,13 @@
ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
"sendfile: %d, @%O %O:%O",
- rc, file->file_pos, sent, file_size + header_size);
+ rc, file->file_pos, sent, file_size + header.size);
} else {
- rc = writev(c->fd, header.elts, header.nelts);
+ rc = writev(c->fd, header.iovs, header.count);
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
- "writev: %d of %O", rc, header_size);
+ "writev: %d of %uz", rc, header.size);
if (rc == -1) {
err = ngx_errno;
diff --git a/src/os/unix/ngx_freebsd_sendfile_chain.c b/src/os/unix/ngx_freebsd_sendfile_chain.c
index 666e0c5..83e0f68 100644
--- a/src/os/unix/ngx_freebsd_sendfile_chain.c
+++ b/src/os/unix/ngx_freebsd_sendfile_chain.c
@@ -33,17 +33,15 @@
ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
{
int rc, flags;
- u_char *prev;
off_t size, send, prev_send, aligned, sent, fprev;
- size_t header_size, file_size;
+ size_t file_size;
ngx_uint_t eintr, eagain;
ngx_err_t err;
ngx_buf_t *file;
- ngx_array_t header, trailer;
ngx_event_t *wev;
ngx_chain_t *cl;
+ ngx_iovec_t header, trailer;
struct sf_hdtr hdtr;
- struct iovec *iov;
struct iovec headers[NGX_IOVS_PREALLOCATE];
struct iovec trailers[NGX_IOVS_PREALLOCATE];
@@ -74,69 +72,27 @@
eagain = 0;
flags = 0;
- header.elts = headers;
- header.size = sizeof(struct iovec);
+ header.iovs = headers;
header.nalloc = NGX_IOVS_PREALLOCATE;
- header.pool = c->pool;
- trailer.elts = trailers;
- trailer.size = sizeof(struct iovec);
+ trailer.iovs = trailers;
trailer.nalloc = NGX_IOVS_PREALLOCATE;
- trailer.pool = c->pool;
for ( ;; ) {
file = NULL;
file_size = 0;
- header_size = 0;
eintr = 0;
prev_send = send;
- header.nelts = 0;
- trailer.nelts = 0;
-
/* create the header iovec and coalesce the neighbouring bufs */
- prev = NULL;
- iov = NULL;
+ cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log);
- for (cl = in; cl && send < limit; cl = cl->next) {
-
- if (ngx_buf_special(cl->buf)) {
- continue;
- }
-
- if (!ngx_buf_in_memory_only(cl->buf)) {
- break;
- }
-
- size = cl->buf->last - cl->buf->pos;
-
- if (send + size > limit) {
- size = limit - send;
- }
-
- if (prev == cl->buf->pos) {
- iov->iov_len += (size_t) size;
-
- } else {
- if (header.nelts >= IOV_MAX){
- break;
- }
-
- iov = ngx_array_push(&header);
- if (iov == NULL) {
- return NGX_CHAIN_ERROR;
- }
-
- iov->iov_base = (void *) cl->buf->pos;
- iov->iov_len = (size_t) size;
- }
-
- prev = cl->buf->pos + (size_t) size;
- header_size += (size_t) size;
- send += size;
+ if (cl == NGX_CHAIN_ERROR) {
+ return NGX_CHAIN_ERROR;
}
+ send += header.size;
if (cl && cl->buf->in_file && send < limit) {
file = cl->buf;
@@ -174,47 +130,13 @@
/* create the trailer iovec and coalesce the neighbouring bufs */
- prev = NULL;
- iov = NULL;
+ cl = ngx_output_chain_to_iovec(&trailer, cl, limit - send, c->log);
- while (cl && send < limit) {
-
- if (ngx_buf_special(cl->buf)) {
- cl = cl->next;
- continue;
- }
-
- if (!ngx_buf_in_memory_only(cl->buf)) {
- break;
- }
-
- size = cl->buf->last - cl->buf->pos;
-
- if (send + size > limit) {
- size = limit - send;
- }
-
- if (prev == cl->buf->pos) {
- iov->iov_len += (size_t) size;
-
- } else {
- if (trailer.nelts >= IOV_MAX){
- break;
- }
-
- iov = ngx_array_push(&trailer);
- if (iov == NULL) {
- return NGX_CHAIN_ERROR;
- }
-
- iov->iov_base = (void *) cl->buf->pos;
- iov->iov_len = (size_t) size;
- }
-
- prev = cl->buf->pos + (size_t) size;
- send += size;
- cl = cl->next;
+ if (cl == NGX_CHAIN_ERROR) {
+ return NGX_CHAIN_ERROR;
}
+
+ send += trailer.size;
}
if (file) {
@@ -250,10 +172,10 @@
* but corresponding pointer is not NULL
*/
- hdtr.headers = header.nelts ? (struct iovec *) header.elts: NULL;
- hdtr.hdr_cnt = header.nelts;
- hdtr.trailers = trailer.nelts ? (struct iovec *) trailer.elts: NULL;
- hdtr.trl_cnt = trailer.nelts;
+ hdtr.headers = header.count ? header.iovs : NULL;
+ hdtr.hdr_cnt = header.count;
+ hdtr.trailers = trailer.count ? trailer.iovs : NULL;
+ hdtr.trl_cnt = trailer.count;
/*
* the "nbytes bug" of the old sendfile() syscall:
@@ -261,7 +183,7 @@
*/
if (!ngx_freebsd_sendfile_nbytes_bug) {
- header_size = 0;
+ header.size = 0;
}
sent = 0;
@@ -271,7 +193,7 @@
#endif
rc = sendfile(file->file->fd, c->fd, file->file_pos,
- file_size + header_size, &hdtr, &sent, flags);
+ file_size + header.size, &hdtr, &sent, flags);
if (rc == -1) {
err = ngx_errno;
@@ -322,13 +244,13 @@
ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0,
"sendfile: %d, @%O %O:%uz",
- rc, file->file_pos, sent, file_size + header_size);
+ rc, file->file_pos, sent, file_size + header.size);
} else {
- rc = writev(c->fd, header.elts, header.nelts);
+ rc = writev(c->fd, header.iovs, header.count);
ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0,
- "writev: %d of %uz", rc, header_size);
+ "writev: %d of %uz", rc, header.size);
if (rc == -1) {
err = ngx_errno;
diff --git a/src/os/unix/ngx_linux_sendfile_chain.c b/src/os/unix/ngx_linux_sendfile_chain.c
index 99d00bf..a84a134 100644
--- a/src/os/unix/ngx_linux_sendfile_chain.c
+++ b/src/os/unix/ngx_linux_sendfile_chain.c
@@ -32,15 +32,14 @@
{
int rc, tcp_nodelay;
off_t size, send, prev_send, aligned, sent, fprev;
- u_char *prev;
size_t file_size;
ngx_err_t err;
ngx_buf_t *file;
ngx_uint_t eintr;
- ngx_array_t header;
ngx_event_t *wev;
ngx_chain_t *cl;
- struct iovec *iov, headers[NGX_IOVS_PREALLOCATE];
+ ngx_iovec_t header;
+ struct iovec headers[NGX_IOVS_PREALLOCATE];
#if (NGX_HAVE_SENDFILE64)
off_t offset;
#else
@@ -63,10 +62,8 @@
send = 0;
- header.elts = headers;
- header.size = sizeof(struct iovec);
+ header.iovs = headers;
header.nalloc = NGX_IOVS_PREALLOCATE;
- header.pool = c->pool;
for ( ;; ) {
file = NULL;
@@ -74,75 +71,20 @@
eintr = 0;
prev_send = send;
- header.nelts = 0;
-
- prev = NULL;
- iov = NULL;
-
/* create the iovec and coalesce the neighbouring bufs */
- for (cl = in; cl && send < limit; cl = cl->next) {
+ cl = ngx_output_chain_to_iovec(&header, in, limit - send, c->log);
- if (ngx_buf_special(cl->buf)) {
- continue;
- }
-
-#if 1
- if (!ngx_buf_in_memory(cl->buf) && !cl->buf->in_file) {
- ngx_log_error(NGX_LOG_ALERT, c->log, 0,
- "zero size buf in sendfile "
- "t:%d r:%d f:%d %p %p-%p %p %O-%O",
- cl->buf->temporary,
- cl->buf->recycled,
- cl->buf->in_file,
- cl->buf->start,
- cl->buf->pos,
- cl->buf->last,
- cl->buf->file,
- cl->buf->file_pos,
- cl->buf->file_last);
-
- ngx_debug_point();
-
- return NGX_CHAIN_ERROR;
- }
-#endif
-
- if (!ngx_buf_in_memory_only(cl->buf)) {
- break;
- }
-
- size = cl->buf->last - cl->buf->pos;
-
- if (send + size > limit) {
- size = limit - send;
- }
-
- if (prev == cl->buf->pos) {
- iov->iov_len += (size_t) size;
-
- } else {
- if (header.nelts >= IOV_MAX) {
- break;
- }
-
- iov = ngx_array_push(&header);
- if (iov == NULL) {
- return NGX_CHAIN_ERROR;
- }
-
- iov->iov_base = (void *) cl->buf->pos;
- iov->iov_len = (size_t) size;
- }
-
- prev = cl->buf->pos + (size_t) size;
- send += size;
+ if (cl == NGX_CHAIN_ERROR) {
+ return NGX_CHAIN_ERROR;
}
+ send += header.size;
+
/* set TCP_CORK if there is a header before a file */
if (c->tcp_nopush == NGX_TCP_NOPUSH_UNSET
- && header.nelts != 0
+ && header.count != 0
&& cl
&& cl->buf->in_file)
{
@@ -206,7 +148,7 @@
/* get the file buf */
- if (header.nelts == 0 && cl && cl->buf->in_file && send < limit) {
+ if (header.count == 0 && cl && cl->buf->in_file && send < limit) {
file = cl->buf;
/* coalesce the neighbouring file bufs */
@@ -283,7 +225,7 @@
rc, file->file_pos, sent, file_size);
} else {
- rc = writev(c->fd, header.elts, header.nelts);
+ rc = writev(c->fd, header.iovs, header.count);
if (rc == -1) {
err = ngx_errno;
diff --git a/src/os/unix/ngx_os.h b/src/os/unix/ngx_os.h
index a158642..fa6a6a6 100644
--- a/src/os/unix/ngx_os.h
+++ b/src/os/unix/ngx_os.h
@@ -64,6 +64,17 @@
#endif
+typedef struct {
+ struct iovec *iovs;
+ ngx_uint_t count;
+ size_t size;
+ ngx_uint_t nalloc;
+} ngx_iovec_t;
+
+ngx_chain_t *ngx_output_chain_to_iovec(ngx_iovec_t *vec, ngx_chain_t *in,
+ size_t limit, ngx_log_t *log);
+
+
extern ngx_os_io_t ngx_os_io;
extern ngx_int_t ngx_ncpu;
extern ngx_int_t ngx_max_sockets;
diff --git a/src/os/unix/ngx_writev_chain.c b/src/os/unix/ngx_writev_chain.c
index 2f5b175..d9c94dd 100644
--- a/src/os/unix/ngx_writev_chain.c
+++ b/src/os/unix/ngx_writev_chain.c
@@ -13,15 +13,14 @@
ngx_chain_t *
ngx_writev_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit)
{
- u_char *prev;
- ssize_t n, size, sent;
+ ssize_t n, sent;
off_t send, prev_send;
ngx_uint_t eintr;
ngx_err_t err;
- ngx_array_t vec;
ngx_chain_t *cl;
ngx_event_t *wev;
- struct iovec *iov, iovs[NGX_IOVS_PREALLOCATE];
+ ngx_iovec_t vec;
+ struct iovec iovs[NGX_IOVS_PREALLOCATE];
wev = c->write;
@@ -48,61 +47,43 @@
send = 0;
- vec.elts = iovs;
- vec.size = sizeof(struct iovec);
+ vec.iovs = iovs;
vec.nalloc = NGX_IOVS_PREALLOCATE;
- vec.pool = c->pool;
for ( ;; ) {
- prev = NULL;
- iov = NULL;
eintr = 0;
prev_send = send;
- vec.nelts = 0;
-
/* create the iovec and coalesce the neighbouring bufs */
- for (cl = in; cl && send < limit; cl = cl->next) {
+ cl = ngx_output_chain_to_iovec(&vec, in, limit - send, c->log);
- if (ngx_buf_special(cl->buf)) {
- continue;
- }
-
-#if 1
- if (!ngx_buf_in_memory(cl->buf)) {
- ngx_debug_point();
- }
-#endif
-
- size = cl->buf->last - cl->buf->pos;
-
- if (send + size > limit) {
- size = (ssize_t) (limit - send);
- }
-
- if (prev == cl->buf->pos) {
- iov->iov_len += size;
-
- } else {
- if (vec.nelts >= IOV_MAX) {
- break;
- }
-
- iov = ngx_array_push(&vec);
- if (iov == NULL) {
- return NGX_CHAIN_ERROR;
- }
-
- iov->iov_base = (void *) cl->buf->pos;
- iov->iov_len = size;
- }
-
- prev = cl->buf->pos + size;
- send += size;
+ if (cl == NGX_CHAIN_ERROR) {
+ return NGX_CHAIN_ERROR;
}
- n = writev(c->fd, vec.elts, vec.nelts);
+ if (cl && cl->buf->in_file) {
+ ngx_log_error(NGX_LOG_ALERT, c->log, 0,
+ "file buf in writev "
+ "t:%d r:%d f:%d %p %p-%p %p %O-%O",
+ cl->buf->temporary,
+ cl->buf->recycled,
+ cl->buf->in_file,
+ cl->buf->start,
+ cl->buf->pos,
+ cl->buf->last,
+ cl->buf->file,
+ cl->buf->file_pos,
+ cl->buf->file_last);
+
+ ngx_debug_point();
+
+ return NGX_CHAIN_ERROR;
+ }
+
+ send += vec.size;
+
+ n = writev(c->fd, vec.iovs, vec.count);
if (n == -1) {
err = ngx_errno;
@@ -148,3 +129,77 @@
}
}
}
+
+
+ngx_chain_t *
+ngx_output_chain_to_iovec(ngx_iovec_t *vec, ngx_chain_t *in, size_t limit,
+ ngx_log_t *log)
+{
+ size_t total, size;
+ u_char *prev;
+ ngx_uint_t n;
+ struct iovec *iov;
+
+ iov = NULL;
+ prev = NULL;
+ total = 0;
+ n = 0;
+
+ for ( /* void */ ; in && total < limit; in = in->next) {
+
+ if (ngx_buf_special(in->buf)) {
+ continue;
+ }
+
+ if (in->buf->in_file) {
+ break;
+ }
+
+ if (!ngx_buf_in_memory(in->buf)) {
+ ngx_log_error(NGX_LOG_ALERT, log, 0,
+ "bad buf in output chain "
+ "t:%d r:%d f:%d %p %p-%p %p %O-%O",
+ in->buf->temporary,
+ in->buf->recycled,
+ in->buf->in_file,
+ in->buf->start,
+ in->buf->pos,
+ in->buf->last,
+ in->buf->file,
+ in->buf->file_pos,
+ in->buf->file_last);
+
+ ngx_debug_point();
+
+ return NGX_CHAIN_ERROR;
+ }
+
+ size = in->buf->last - in->buf->pos;
+
+ if (size > limit - total) {
+ size = limit - total;
+ }
+
+ if (prev == in->buf->pos) {
+ iov->iov_len += size;
+
+ } else {
+ if (n == vec->nalloc) {
+ break;
+ }
+
+ iov = &vec->iovs[n++];
+
+ iov->iov_base = (void *) in->buf->pos;
+ iov->iov_len = size;
+ }
+
+ prev = in->buf->pos + size;
+ total += size;
+ }
+
+ vec->count = n;
+ vec->size = total;
+
+ return in;
+}
