Upstream: cache revalidation with conditional requests.
The following new directives are introduced: proxy_cache_revalidate,
fastcgi_cache_revalidate, scgi_cache_revalidate, uwsgi_cache_revalidate.
Default is off. When set to on, they enable cache revalidation using
conditional requests with If-Modified-Since for expired cache items.
As of now, no attempts are made to merge headers given in a 304 response
during cache revalidation with headers previously stored in a cache item.
Headers in a 304 response are only used to calculate new validity time
of a cache item.
diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c
index dd79418..51672e2 100644
--- a/src/http/modules/ngx_http_fastcgi_module.c
+++ b/src/http/modules/ngx_http_fastcgi_module.c
@@ -405,6 +405,13 @@
offsetof(ngx_http_fastcgi_loc_conf_t, upstream.cache_lock_timeout),
NULL },
+ { ngx_string("fastcgi_cache_revalidate"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_fastcgi_loc_conf_t, upstream.cache_revalidate),
+ NULL },
+
#endif
{ ngx_string("fastcgi_temp_path"),
@@ -563,7 +570,8 @@
#if (NGX_HTTP_CACHE)
static ngx_keyval_t ngx_http_fastcgi_cache_headers[] = {
- { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") },
+ { ngx_string("HTTP_IF_MODIFIED_SINCE"),
+ ngx_string("$upstream_cache_last_modified") },
{ ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") },
{ ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") },
{ ngx_string("HTTP_IF_MATCH"), ngx_string("") },
@@ -2336,6 +2344,7 @@
conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
conf->upstream.cache_lock = NGX_CONF_UNSET;
conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+ conf->upstream.cache_revalidate = NGX_CONF_UNSET;
#endif
conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -2582,6 +2591,9 @@
ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
prev->upstream.cache_lock_timeout, 5000);
+ ngx_conf_merge_value(conf->upstream.cache_revalidate,
+ prev->upstream.cache_revalidate, 0);
+
#endif
ngx_conf_merge_value(conf->upstream.pass_request_headers,
diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c
index cf68711..4046075 100644
--- a/src/http/modules/ngx_http_proxy_module.c
+++ b/src/http/modules/ngx_http_proxy_module.c
@@ -465,6 +465,13 @@
offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock_timeout),
NULL },
+ { ngx_string("proxy_cache_revalidate"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_revalidate),
+ NULL },
+
#endif
{ ngx_string("proxy_temp_path"),
@@ -622,7 +629,8 @@
{ ngx_string("Keep-Alive"), ngx_string("") },
{ ngx_string("Expect"), ngx_string("") },
{ ngx_string("Upgrade"), ngx_string("") },
- { ngx_string("If-Modified-Since"), ngx_string("") },
+ { ngx_string("If-Modified-Since"),
+ ngx_string("$upstream_cache_last_modified") },
{ ngx_string("If-Unmodified-Since"), ngx_string("") },
{ ngx_string("If-None-Match"), ngx_string("") },
{ ngx_string("If-Match"), ngx_string("") },
@@ -2454,6 +2462,7 @@
conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
conf->upstream.cache_lock = NGX_CONF_UNSET;
conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+ conf->upstream.cache_revalidate = NGX_CONF_UNSET;
#endif
conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -2710,6 +2719,9 @@
ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
prev->upstream.cache_lock_timeout, 5000);
+ ngx_conf_merge_value(conf->upstream.cache_revalidate,
+ prev->upstream.cache_revalidate, 0);
+
#endif
ngx_conf_merge_str_value(conf->method, prev->method, "");
diff --git a/src/http/modules/ngx_http_scgi_module.c b/src/http/modules/ngx_http_scgi_module.c
index f9c0d14..9163c6b 100644
--- a/src/http/modules/ngx_http_scgi_module.c
+++ b/src/http/modules/ngx_http_scgi_module.c
@@ -262,6 +262,13 @@
offsetof(ngx_http_scgi_loc_conf_t, upstream.cache_lock_timeout),
NULL },
+ { ngx_string("scgi_cache_revalidate"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_scgi_loc_conf_t, upstream.cache_revalidate),
+ NULL },
+
#endif
{ ngx_string("scgi_temp_path"),
@@ -369,7 +376,8 @@
#if (NGX_HTTP_CACHE)
static ngx_keyval_t ngx_http_scgi_cache_headers[] = {
- { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") },
+ { ngx_string("HTTP_IF_MODIFIED_SINCE"),
+ ngx_string("$upstream_cache_last_modified") },
{ ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") },
{ ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") },
{ ngx_string("HTTP_IF_MATCH"), ngx_string("") },
@@ -1093,6 +1101,7 @@
conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
conf->upstream.cache_lock = NGX_CONF_UNSET;
conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+ conf->upstream.cache_revalidate = NGX_CONF_UNSET;
#endif
conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -1333,6 +1342,9 @@
ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
prev->upstream.cache_lock_timeout, 5000);
+ ngx_conf_merge_value(conf->upstream.cache_revalidate,
+ prev->upstream.cache_revalidate, 0);
+
#endif
ngx_conf_merge_value(conf->upstream.pass_request_headers,
diff --git a/src/http/modules/ngx_http_uwsgi_module.c b/src/http/modules/ngx_http_uwsgi_module.c
index 0635299..9c5d762 100644
--- a/src/http/modules/ngx_http_uwsgi_module.c
+++ b/src/http/modules/ngx_http_uwsgi_module.c
@@ -289,6 +289,13 @@
offsetof(ngx_http_uwsgi_loc_conf_t, upstream.cache_lock_timeout),
NULL },
+ { ngx_string("uwsgi_cache_revalidate"),
+ NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+ ngx_conf_set_flag_slot,
+ NGX_HTTP_LOC_CONF_OFFSET,
+ offsetof(ngx_http_uwsgi_loc_conf_t, upstream.cache_revalidate),
+ NULL },
+
#endif
{ ngx_string("uwsgi_temp_path"),
@@ -402,7 +409,8 @@
#if (NGX_HTTP_CACHE)
static ngx_keyval_t ngx_http_uwsgi_cache_headers[] = {
- { ngx_string("HTTP_IF_MODIFIED_SINCE"), ngx_string("") },
+ { ngx_string("HTTP_IF_MODIFIED_SINCE"),
+ ngx_string("$upstream_cache_last_modified") },
{ ngx_string("HTTP_IF_UNMODIFIED_SINCE"), ngx_string("") },
{ ngx_string("HTTP_IF_NONE_MATCH"), ngx_string("") },
{ ngx_string("HTTP_IF_MATCH"), ngx_string("") },
@@ -1130,6 +1138,7 @@
conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
conf->upstream.cache_lock = NGX_CONF_UNSET;
conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
+ conf->upstream.cache_revalidate = NGX_CONF_UNSET;
#endif
conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -1370,6 +1379,9 @@
ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
prev->upstream.cache_lock_timeout, 5000);
+ ngx_conf_merge_value(conf->upstream.cache_revalidate,
+ prev->upstream.cache_revalidate, 0);
+
#endif
ngx_conf_merge_value(conf->upstream.pass_request_headers,
diff --git a/src/http/ngx_http_cache.h b/src/http/ngx_http_cache.h
index 2a2d722..193a353 100644
--- a/src/http/ngx_http_cache.h
+++ b/src/http/ngx_http_cache.h
@@ -19,8 +19,9 @@
#define NGX_HTTP_CACHE_EXPIRED 3
#define NGX_HTTP_CACHE_STALE 4
#define NGX_HTTP_CACHE_UPDATING 5
-#define NGX_HTTP_CACHE_HIT 6
-#define NGX_HTTP_CACHE_SCARCE 7
+#define NGX_HTTP_CACHE_REVALIDATED 6
+#define NGX_HTTP_CACHE_HIT 7
+#define NGX_HTTP_CACHE_SCARCE 8
#define NGX_HTTP_CACHE_KEY_LEN 16
@@ -143,6 +144,7 @@
ngx_int_t ngx_http_file_cache_open(ngx_http_request_t *r);
void ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf);
void ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf);
+void ngx_http_file_cache_update_header(ngx_http_request_t *r);
ngx_int_t ngx_http_cache_send(ngx_http_request_t *);
void ngx_http_file_cache_free(ngx_http_cache_t *c, ngx_temp_file_t *tf);
time_t ngx_http_file_cache_valid(ngx_array_t *cache_valid, ngx_uint_t status);
diff --git a/src/http/ngx_http_file_cache.c b/src/http/ngx_http_file_cache.c
index 7e46d12..9e6be15 100644
--- a/src/http/ngx_http_file_cache.c
+++ b/src/http/ngx_http_file_cache.c
@@ -53,6 +53,7 @@
ngx_string("EXPIRED"),
ngx_string("STALE"),
ngx_string("UPDATING"),
+ ngx_string("REVALIDATED"),
ngx_string("HIT")
};
@@ -971,6 +972,116 @@
}
+void
+ngx_http_file_cache_update_header(ngx_http_request_t *r)
+{
+ ssize_t n;
+ ngx_err_t err;
+ ngx_file_t file;
+ ngx_file_info_t fi;
+ ngx_http_cache_t *c;
+ ngx_http_file_cache_header_t h;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http file cache update header");
+
+ c = r->cache;
+
+ ngx_memzero(&file, sizeof(ngx_file_t));
+
+ file.name = c->file.name;
+ file.log = r->connection->log;
+ file.fd = ngx_open_file(file.name.data, NGX_FILE_RDWR, NGX_FILE_OPEN, 0);
+
+ if (file.fd == NGX_INVALID_FILE) {
+ err = ngx_errno;
+
+ /* cache file may have been deleted */
+
+ if (err == NGX_ENOENT) {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http file cache \"%s\" not found",
+ file.name.data);
+ return;
+ }
+
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, err,
+ ngx_open_file_n " \"%s\" failed", file.name.data);
+ return;
+ }
+
+ /*
+ * make sure cache file wasn't replaced;
+ * if it was, do nothing
+ */
+
+ if (ngx_fd_info(file.fd, &fi) == NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, ngx_errno,
+ ngx_fd_info_n " \"%s\" failed", file.name.data);
+ goto done;
+ }
+
+ if (c->uniq != ngx_file_uniq(&fi)
+ || c->length != ngx_file_size(&fi))
+ {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http file cache \"%s\" changed",
+ file.name.data);
+ goto done;
+ }
+
+ n = ngx_read_file(&file, (u_char *) &h,
+ sizeof(ngx_http_file_cache_header_t), 0);
+
+ if (n == NGX_ERROR) {
+ goto done;
+ }
+
+ if ((size_t) n != sizeof(ngx_http_file_cache_header_t)) {
+ ngx_log_error(NGX_LOG_CRIT, r->connection->log, 0,
+ ngx_read_file_n " read only %z of %z from \"%s\"",
+ n, sizeof(ngx_http_file_cache_header_t), file.name.data);
+ goto done;
+ }
+
+ if (h.last_modified != c->last_modified
+ || h.crc32 != c->crc32
+ || h.header_start != c->header_start
+ || h.body_start != c->body_start)
+ {
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http file cache \"%s\" content changed",
+ file.name.data);
+ goto done;
+ }
+
+ /*
+ * update cache file header with new data,
+ * notably h.valid_sec and h.date
+ */
+
+ ngx_memzero(&h, sizeof(ngx_http_file_cache_header_t));
+
+ h.valid_sec = c->valid_sec;
+ h.last_modified = c->last_modified;
+ h.date = c->date;
+ h.crc32 = c->crc32;
+ h.valid_msec = (u_short) c->valid_msec;
+ h.header_start = (u_short) c->header_start;
+ h.body_start = (u_short) c->body_start;
+
+ (void) ngx_write_file(&file, (u_char *) &h,
+ sizeof(ngx_http_file_cache_header_t), 0);
+
+done:
+
+ if (ngx_close_file(file.fd) == NGX_FILE_ERROR) {
+ ngx_log_error(NGX_LOG_ALERT, r->connection->log, ngx_errno,
+ ngx_close_file_n " \"%s\" failed", file.name.data);
+ }
+}
+
+
ngx_int_t
ngx_http_cache_send(ngx_http_request_t *r)
{
diff --git a/src/http/ngx_http_upstream.c b/src/http/ngx_http_upstream.c
index 88d84c9..9570ccc 100644
--- a/src/http/ngx_http_upstream.c
+++ b/src/http/ngx_http_upstream.c
@@ -17,6 +17,8 @@
ngx_http_upstream_t *u);
static ngx_int_t ngx_http_upstream_cache_status(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_upstream_cache_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
#endif
static void ngx_http_upstream_init_request(ngx_http_request_t *r);
@@ -358,6 +360,10 @@
ngx_http_upstream_cache_status, 0,
NGX_HTTP_VAR_NOCACHEABLE, 0 },
+ { ngx_string("upstream_cache_last_modified"), NULL,
+ ngx_http_upstream_cache_last_modified, 0,
+ NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 },
+
#endif
{ ngx_null_string, NULL, NULL, 0, 0, 0 }
@@ -1806,6 +1812,56 @@
#endif
}
+#if (NGX_HTTP_CACHE)
+
+ if (status == NGX_HTTP_NOT_MODIFIED
+ && u->cache_status == NGX_HTTP_CACHE_EXPIRED
+ && u->conf->cache_revalidate)
+ {
+ time_t now, valid;
+ ngx_int_t rc;
+
+ ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+ "http upstream not modified");
+
+ now = ngx_time();
+ valid = r->cache->valid_sec;
+
+ rc = u->reinit_request(r);
+
+ if (rc != NGX_OK) {
+ ngx_http_upstream_finalize_request(r, u, rc);
+ return NGX_OK;
+ }
+
+ u->cache_status = NGX_HTTP_CACHE_REVALIDATED;
+ rc = ngx_http_upstream_cache_send(r, u);
+
+ if (valid == 0) {
+ valid = r->cache->valid_sec;
+ }
+
+ if (valid == 0) {
+ valid = ngx_http_file_cache_valid(u->conf->cache_valid,
+ u->headers_in.status_n);
+ if (valid) {
+ valid = now + valid;
+ }
+ }
+
+ if (valid) {
+ r->cache->valid_sec = valid;
+ r->cache->date = now;
+
+ ngx_http_file_cache_update_header(r);
+ }
+
+ ngx_http_upstream_finalize_request(r, u, rc);
+ return NGX_OK;
+ }
+
+#endif
+
return NGX_DECLINED;
}
@@ -4492,6 +4548,35 @@
return NGX_OK;
}
+
+static ngx_int_t
+ngx_http_upstream_cache_last_modified(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ u_char *p;
+
+ if (!r->upstream->conf->cache_revalidate
+ || r->upstream->cache_status != NGX_HTTP_CACHE_EXPIRED
+ || r->cache->last_modified == -1)
+ {
+ v->not_found = 1;
+ return NGX_OK;
+ }
+
+ p = ngx_pnalloc(r->pool, sizeof("Mon, 28 Sep 1970 06:00:00 GMT") - 1);
+ if (p == NULL) {
+ return NGX_ERROR;
+ }
+
+ v->len = ngx_http_time(p, r->cache->last_modified) - p;
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ v->data = p;
+
+ return NGX_OK;
+}
+
#endif
diff --git a/src/http/ngx_http_upstream.h b/src/http/ngx_http_upstream.h
index fd4e36b..469988e 100644
--- a/src/http/ngx_http_upstream.h
+++ b/src/http/ngx_http_upstream.h
@@ -178,6 +178,8 @@
ngx_flag_t cache_lock;
ngx_msec_t cache_lock_timeout;
+ ngx_flag_t cache_revalidate;
+
ngx_array_t *cache_valid;
ngx_array_t *cache_bypass;
ngx_array_t *no_cache;
