FreeBSD and Linux AIO support
diff --git a/src/http/ngx_http_cache.h b/src/http/ngx_http_cache.h index 6ac6fd8..e04eba3 100644 --- a/src/http/ngx_http_cache.h +++ b/src/http/ngx_http_cache.h
@@ -18,6 +18,7 @@ #define NGX_HTTP_CACHE_STALE 3 #define NGX_HTTP_CACHE_UPDATING 4 #define NGX_HTTP_CACHE_HIT 5 +#define NGX_HTTP_CACHE_SCARCE 6 #define NGX_HTTP_CACHE_KEY_LEN 16
diff --git a/src/http/ngx_http_copy_filter_module.c b/src/http/ngx_http_copy_filter_module.c index 688677f..411e731 100644 --- a/src/http/ngx_http_copy_filter_module.c +++ b/src/http/ngx_http_copy_filter_module.c
@@ -14,6 +14,12 @@ } ngx_http_copy_filter_conf_t; +#if (NGX_HAVE_FILE_AIO) +static void ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx, + ngx_file_t *file); +static void ngx_http_copy_aio_event_handler(ngx_event_t *ev); +#endif + static void *ngx_http_copy_filter_create_conf(ngx_conf_t *cf); static char *ngx_http_copy_filter_merge_conf(ngx_conf_t *cf, void *parent, void *child); @@ -73,10 +79,15 @@ ngx_int_t rc; ngx_connection_t *c; ngx_output_chain_ctx_t *ctx; + ngx_http_core_loc_conf_t *clcf; ngx_http_copy_filter_conf_t *conf; c = r->connection; + if (r->aio) { + return NGX_AGAIN; + } + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, "copy filter: \"%V?%V\"", &r->uri, &r->args); @@ -104,6 +115,13 @@ ctx->output_filter = (ngx_output_chain_filter_pt) ngx_http_next_filter; ctx->filter_ctx = r; +#if (NGX_HAVE_FILE_AIO) + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + if (clcf->aio) { + ctx->aio = ngx_http_copy_aio_handler; + } +#endif + r->request_output = 1; } @@ -123,6 +141,41 @@ } +#if (NGX_HAVE_FILE_AIO) + +static void +ngx_http_copy_aio_handler(ngx_output_chain_ctx_t *ctx, ngx_file_t *file) +{ + ngx_http_request_t *r; + + r = ctx->filter_ctx; + + file->aio->data = r; + file->aio->handler = ngx_http_copy_aio_event_handler; + + r->main->blocked++; + r->aio = 1; +} + + +static void +ngx_http_copy_aio_event_handler(ngx_event_t *ev) +{ + ngx_event_aio_t *aio; + ngx_http_request_t *r; + + aio = ev->data; + r = aio->data; + + r->main->blocked--; + r->aio = 0; + + r->connection->write->handler(r->connection->write); +} + +#endif + + static void * ngx_http_copy_filter_create_conf(ngx_conf_t *cf) {
diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index 7b47fbb..da91ee6 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c
@@ -383,6 +383,17 @@ offsetof(ngx_http_core_loc_conf_t, sendfile_max_chunk), NULL }, +#if (NGX_HAVE_FILE_AIO) + + { ngx_string("aio"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_core_loc_conf_t, aio), + NULL }, + +#endif + { ngx_string("directio"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, ngx_http_core_directio, @@ -2916,6 +2927,9 @@ lcf->internal = NGX_CONF_UNSET; lcf->sendfile = NGX_CONF_UNSET; lcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE; +#if (NGX_HAVE_FILE_AIO) + lcf->aio = NGX_CONF_UNSET; +#endif lcf->directio = NGX_CONF_UNSET; lcf->tcp_nopush = NGX_CONF_UNSET; lcf->tcp_nodelay = NGX_CONF_UNSET; @@ -3113,6 +3127,9 @@ ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0); ngx_conf_merge_size_value(conf->sendfile_max_chunk, prev->sendfile_max_chunk, 0); +#if (NGX_HAVE_FILE_AIO) + ngx_conf_merge_value(conf->aio, prev->aio, 0); +#endif ngx_conf_merge_off_value(conf->directio, prev->directio, NGX_MAX_OFF_T_VALUE); ngx_conf_merge_value(conf->tcp_nopush, prev->tcp_nopush, 0);
diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index 71d5b95..522509b 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h
@@ -347,6 +347,9 @@ /* client_body_in_singe_buffer */ ngx_flag_t internal; /* internal */ ngx_flag_t sendfile; /* sendfile */ +#if (NGX_HAVE_FILE_AIO) + ngx_flag_t aio; /* aio */ +#endif ngx_flag_t tcp_nopush; /* tcp_nopush */ ngx_flag_t tcp_nodelay; /* tcp_nodelay */ ngx_flag_t reset_timedout_connection; /* reset_timedout_connection */
diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c index 8972f47..a6271a8 100644 --- a/src/http/ngx_http_file_cache.c +++ b/src/http/ngx_http_file_cache.c
@@ -10,6 +10,11 @@ #include <ngx_md5.h> +static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r, + ngx_http_cache_t *c); +#if (NGX_HAVE_FILE_AIO) +static void ngx_http_cache_aio_event_handler(ngx_event_t *ev); +#endif static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c); static ngx_http_file_cache_node_t * @@ -173,20 +178,22 @@ ngx_int_t ngx_http_file_cache_open(ngx_http_request_t *r) { - u_char *p; - time_t now; - ssize_t n; - ngx_int_t rc, rv; - ngx_uint_t cold, test; - ngx_path_t *path; - ngx_http_cache_t *c; - ngx_pool_cleanup_t *cln; - ngx_open_file_info_t of; - ngx_http_file_cache_t *cache; - ngx_http_core_loc_conf_t *clcf; - ngx_http_file_cache_header_t *h; + u_char *p; + ngx_int_t rc, rv; + ngx_uint_t cold, test; + ngx_path_t *path; + ngx_http_cache_t *c; + ngx_pool_cleanup_t *cln; + ngx_open_file_info_t of; + ngx_http_file_cache_t *cache; + ngx_http_core_loc_conf_t *clcf; c = r->cache; + + if (c->buf) { + return ngx_http_file_cache_read(r, c); + } + cache = c->file_cache; cln = ngx_pool_cleanup_add(r->pool, 0); @@ -207,7 +214,7 @@ cln->data = c; if (rc == NGX_AGAIN) { - return rc; + return NGX_HTTP_CACHE_SCARCE; } cold = cache->sh->cold; @@ -227,11 +234,11 @@ if (c->min_uses > 1) { if (!cold) { - return NGX_AGAIN; + return NGX_HTTP_CACHE_SCARCE; } test = 1; - rv = NGX_AGAIN; + rv = NGX_HTTP_CACHE_SCARCE; } else { c->temp_file = 1; @@ -299,14 +306,58 @@ c->file.fd = of.fd; c->file.log = r->connection->log; + c->uniq = of.uniq; + c->length = of.size; c->buf = ngx_create_temp_buf(r->pool, c->body_start); if (c->buf == NULL) { return NGX_ERROR; } + return ngx_http_file_cache_read(r, c); +} + + +static ngx_int_t +ngx_http_file_cache_read(ngx_http_request_t *r, ngx_http_cache_t *c) +{ + time_t now; + ssize_t n; + ngx_int_t rc; + ngx_http_file_cache_t *cache; + ngx_http_file_cache_header_t *h; + + c = r->cache; + +#if (NGX_HAVE_FILE_AIO) + { + ngx_http_core_loc_conf_t *clcf; + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + if (clcf->aio) { + n = ngx_file_aio_read(&c->file, c->buf->pos, c->body_start, 0, r->pool); + + if (n == NGX_AGAIN) { + c->file.aio->data = r; + c->file.aio->handler = ngx_http_cache_aio_event_handler; + + r->main->blocked++; + r->aio = 1; + + return NGX_AGAIN; + } + + } else { + n = ngx_read_file(&c->file, c->buf->pos, c->body_start, 0); + } + } +#else + n = ngx_read_file(&c->file, c->buf->pos, c->body_start, 0); +#endif + if (n == NGX_ERROR) { return n; } @@ -331,12 +382,13 @@ c->last_modified = h->last_modified; c->date = h->date; c->valid_msec = h->valid_msec; - c->length = of.size; c->body_start = h->body_start; r->cached = 1; - if (cold) { + cache = c->file_cache; + + if (cache->sh->cold) { ngx_shmtx_lock(&cache->shpool->mutex); @@ -344,7 +396,7 @@ c->node->uses = 1; c->node->body_start = c->body_start; c->node->exists = 1; - c->node->uniq = of.uniq; + c->node->uniq = c->uniq; cache->sh->size += (c->length + cache->bsize - 1) / cache->bsize; } @@ -379,6 +431,27 @@ } +#if (NGX_HAVE_FILE_AIO) + + +static void +ngx_http_cache_aio_event_handler(ngx_event_t *ev) +{ + ngx_event_aio_t *aio; + ngx_http_request_t *r; + + aio = ev->data; + r = aio->data; + + r->main->blocked--; + r->aio = 0; + + r->connection->write->handler(r->connection->write); +} + +#endif + + static ngx_int_t ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c) {
diff --git a/src/http/ngx_http_request.c b/src/http/ngx_http_request.c index ea8843b..5f46e61 100644 --- a/src/http/ngx_http_request.c +++ b/src/http/ngx_http_request.c
@@ -1868,6 +1868,10 @@ return; } + if (r->main->blocked) { + r->write_event_handler = ngx_http_request_finalizer; + } + ngx_http_terminate_request(r, rc); return; } @@ -1969,7 +1973,7 @@ return; } - if (r->buffered || c->buffered || r->postponed) { + if (r->buffered || c->buffered || r->postponed || r->blocked) { if (ngx_http_set_write_handler(r) != NGX_OK) { ngx_http_terminate_request(r, 0); @@ -2022,7 +2026,7 @@ mr = r->main; ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http terminate request count: %d", mr->count); + "http terminate request count:%d", mr->count); cln = mr->cleanup; mr->cleanup = NULL; @@ -2035,10 +2039,16 @@ cln = cln->next; } - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http terminate cleanup count: %d", mr->count); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http terminate cleanup count:%d blk:%d", + mr->count, mr->blocked); if (mr->write_event_handler) { + + if (mr->blocked) { + return; + } + mr->posted_requests = NULL; mr->write_event_handler = ngx_http_terminate_handler; (void) ngx_http_post_request(mr); @@ -2053,7 +2063,7 @@ ngx_http_terminate_handler(ngx_http_request_t *r) { ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http terminate handler count: %d", r->count); + "http terminate handler count:%d", r->count); r->count = 1; @@ -2161,7 +2171,7 @@ } } else { - if (wev->delayed) { + if (wev->delayed || r->aio) { ngx_log_debug0(NGX_LOG_DEBUG_HTTP, wev->log, 0, "http writer delayed"); @@ -2830,8 +2840,8 @@ r = r->main; c = r->connection; - ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0, - "http request count: %d", r->count); + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http request count:%d blk:%d", r->count, r->blocked); if (r->count == 0) { ngx_log_error(NGX_LOG_ALERT, c->log, 0, "http request count is zero"); @@ -2839,7 +2849,7 @@ r->count--; - if (r->count) { + if (r->count || r->blocked) { return; }
diff --git a/src/http/ngx_http_request.h b/src/http/ngx_http_request.h index 7b984fb..52ac90c 100644 --- a/src/http/ngx_http_request.h +++ b/src/http/ngx_http_request.h
@@ -416,6 +416,12 @@ ngx_http_cleanup_t *cleanup; + unsigned subrequests:8; + unsigned count:8; + unsigned blocked:8; + + unsigned aio:1; + unsigned http_state:4; /* URI with "/." and on Win32 with "//" */ @@ -501,9 +507,6 @@ unsigned stat_writing:1; #endif - unsigned subrequests:8; - unsigned count:8; - /* used to parse HTTP headers */ ngx_uint_t state;
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c index f552d2e..a864fb1 100644 --- a/src/http/ngx_http_upstream.c +++ b/src/http/ngx_http_upstream.c
@@ -18,6 +18,7 @@ ngx_http_variable_value_t *v, uintptr_t data); #endif +static void ngx_http_upstream_init_request(ngx_http_request_t *r); static void ngx_http_upstream_resolve_handler(ngx_resolver_ctx_t *ctx); static void ngx_http_upstream_rd_check_broken_connection(ngx_http_request_t *r); static void ngx_http_upstream_wr_check_broken_connection(ngx_http_request_t *r); @@ -386,15 +387,7 @@ void ngx_http_upstream_init(ngx_http_request_t *r) { - ngx_str_t *host; - ngx_uint_t i; - ngx_connection_t *c; - ngx_resolver_ctx_t *ctx, temp; - ngx_http_cleanup_t *cln; - ngx_http_upstream_t *u; - ngx_http_core_loc_conf_t *clcf; - ngx_http_upstream_srv_conf_t *uscf, **uscfp; - ngx_http_upstream_main_conf_t *umcf; + ngx_connection_t *c; c = r->connection; @@ -405,15 +398,6 @@ ngx_del_timer(c->read); } - u = r->upstream; - - u->store = (u->conf->store || u->conf->store_lengths); - - if (!u->store && !r->post_action && !u->conf->ignore_client_abort) { - r->read_event_handler = ngx_http_upstream_rd_check_broken_connection; - r->write_event_handler = ngx_http_upstream_wr_check_broken_connection; - } - if (ngx_event_flags & NGX_USE_CLEAR_EVENT) { if (!c->write->active) { @@ -426,10 +410,28 @@ } } - if (r->request_body) { - u->request_bufs = r->request_body->bufs; + ngx_http_upstream_init_request(r); +} + + +static void +ngx_http_upstream_init_request(ngx_http_request_t *r) +{ + ngx_str_t *host; + ngx_uint_t i; + ngx_resolver_ctx_t *ctx, temp; + ngx_http_cleanup_t *cln; + ngx_http_upstream_t *u; + ngx_http_core_loc_conf_t *clcf; + ngx_http_upstream_srv_conf_t *uscf, **uscfp; + ngx_http_upstream_main_conf_t *umcf; + + if (r->aio) { + return; } + u = r->upstream; + #if (NGX_HTTP_CACHE) if (u->conf->cache) { @@ -437,6 +439,11 @@ rc = ngx_http_upstream_cache(r, u); + if (rc == NGX_AGAIN) { + r->write_event_handler = ngx_http_upstream_init_request; + return; + } + if (rc == NGX_DONE) { return; } @@ -449,6 +456,17 @@ #endif + u->store = (u->conf->store || u->conf->store_lengths); + + if (!u->store && !r->post_action && !u->conf->ignore_client_abort) { + r->read_event_handler = ngx_http_upstream_rd_check_broken_connection; + r->write_event_handler = ngx_http_upstream_wr_check_broken_connection; + } + + if (r->request_body) { + u->request_bufs = r->request_body->bufs; + } + if (u->create_request(r) != NGX_OK) { ngx_http_finalize_request(r, NGX_HTTP_INTERNAL_SERVER_ERROR); return; @@ -543,7 +561,7 @@ } if (ctx == NGX_NO_RESOLVER) { - ngx_log_error(NGX_LOG_ERR, c->log, 0, + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "no resolver defined to resolve %V", host); ngx_http_finalize_request(r, NGX_HTTP_BAD_GATEWAY); @@ -657,10 +675,6 @@ break; - case NGX_ERROR: - - return NGX_ERROR; - case NGX_HTTP_CACHE_STALE: c->valid_sec = 0; @@ -681,12 +695,20 @@ break; - case NGX_AGAIN: + case NGX_HTTP_CACHE_SCARCE: u->cacheable = 0; break; + case NGX_AGAIN: + + return NGX_AGAIN; + + case NGX_ERROR: + + return NGX_ERROR; + default: /* cached NGX_HTTP_BAD_GATEWAY, NGX_HTTP_GATEWAY_TIME_OUT, etc. */