HTTP/2: graceful shutdown of active connections (closes #1106).

Previously, while shutting down gracefully, the HTTP/2 connections were
closed in transition to idle state after all active streams have been
processed.  That might never happen if the client continued opening new
streams.

Now, nginx sends GOAWAY to all HTTP/2 connections and ignores further
attempts to open new streams.  A worker process will quit as soon as
processing of already opened streams is finished.

src/http/v2/ngx_http_v2.c

diff --git a/src/http/v2/ngx_http_v2.c b/src/http/v2/ngx_http_v2.c
index d0cd2ab..18f891b 100644
--- a/src/http/v2/ngx_http_v2.c
+++ b/src/http/v2/ngx_http_v2.c
@@ -136,6 +136,8 @@
     ngx_uint_t sid, size_t window);
 static ngx_int_t ngx_http_v2_send_rst_stream(ngx_http_v2_connection_t *h2c,
     ngx_uint_t sid, ngx_uint_t status);
+static ngx_int_t ngx_http_v2_send_goaway(ngx_http_v2_connection_t *h2c,
+    ngx_uint_t status);
 
 static ngx_http_v2_out_frame_t *ngx_http_v2_get_frame(
     ngx_http_v2_connection_t *h2c, size_t length, ngx_uint_t type,
@@ -293,6 +295,8 @@
     rev->handler = ngx_http_v2_read_handler;
     c->write->handler = ngx_http_v2_write_handler;
 
+    c->idle = 1;
+
     ngx_http_v2_read_handler(rev);
 }
 
@@ -320,6 +324,25 @@
 
     h2c->blocked = 1;
 
+    if (c->close) {
+        c->close = 0;
+        h2c->goaway = 1;
+
+        if (ngx_http_v2_send_goaway(h2c, NGX_HTTP_V2_NO_ERROR) == NGX_ERROR) {
+            ngx_http_v2_finalize_connection(h2c, 0);
+            return;
+        }
+
+        if (ngx_http_v2_send_output_queue(h2c) == NGX_ERROR) {
+            ngx_http_v2_finalize_connection(h2c, 0);
+            return;
+        }
+
+        h2c->blocked = 0;
+
+        return;
+    }
+
     h2mcf = ngx_http_get_module_main_conf(h2c->http_connection->conf_ctx,
                                           ngx_http_v2_module);
 
@@ -633,6 +656,11 @@
         /* rc == NGX_OK */
     }
 
+    if (h2c->goaway) {
+        ngx_http_close_connection(c);
+        return;
+    }
+
     h2scf = ngx_http_get_module_srv_conf(h2c->http_connection->conf_ctx,
                                          ngx_http_v2_module);
     if (h2c->state.incomplete) {
@@ -640,11 +668,6 @@
         return;
     }
 
-    if (ngx_terminate || ngx_exiting) {
-        ngx_http_v2_finalize_connection(h2c, NGX_HTTP_V2_NO_ERROR);
-        return;
-    }
-
     ngx_destroy_pool(h2c->pool);
 
     h2c->pool = NULL;
@@ -658,7 +681,6 @@
 #endif
 
     c->destroyed = 1;
-    c->idle = 1;
     ngx_reusable_connection(c, 1);
 
     c->write->handler = ngx_http_empty_handler;
@@ -1027,6 +1049,12 @@
         return ngx_http_v2_connection_error(h2c, NGX_HTTP_V2_SIZE_ERROR);
     }
 
+    if (h2c->goaway) {
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, h2c->connection->log, 0,
+                       "skipping http2 HEADERS frame");
+        return ngx_http_v2_state_skip(h2c, pos, end);
+    }
+
     if ((size_t) (end - pos) < size) {
         return ngx_http_v2_state_save(h2c, pos, end,
                                       ngx_http_v2_state_headers);
@@ -4162,7 +4190,6 @@
 #endif
 
     c->destroyed = 0;
-    c->idle = 0;
     ngx_reusable_connection(c, 0);
 
     h2scf = ngx_http_get_module_srv_conf(h2c->http_connection->conf_ctx,
@@ -4197,8 +4224,10 @@
 
     h2c->blocked = 1;
 
-    if (!c->error && ngx_http_v2_send_goaway(h2c, status) != NGX_ERROR) {
-        (void) ngx_http_v2_send_output_queue(h2c);
+    if (!c->error && !h2c->goaway) {
+        if (ngx_http_v2_send_goaway(h2c, status) != NGX_ERROR) {
+            (void) ngx_http_v2_send_output_queue(h2c);
+        }
     }
 
     c->error = 1;

src/http/v2/ngx_http_v2.h

diff --git a/src/http/v2/ngx_http_v2.h b/src/http/v2/ngx_http_v2.h
index 9e738aa..2c901d7 100644
--- a/src/http/v2/ngx_http_v2.h
+++ b/src/http/v2/ngx_http_v2.h
@@ -146,6 +146,7 @@
     unsigned                         closed_nodes:8;
     unsigned                         settings_ack:1;
     unsigned                         blocked:1;
+    unsigned                         goaway:1;
 };