From df7edb5ddae609ea1fd4285f7439f0d590d97b37 Mon Sep 17 00:00:00 2001
From: Yann Ylavic <ylavic@apache.org>
Date: Wed, 13 Mar 2019 08:59:54 +0000
Subject: [PATCH] Merge r1855306 from trunk:

MPMs unix: bind the bucket number of each child to its slot number

We need not remember each child's bucket number in SHM for restarts, for the
lifetime of the httpd main process the bucket number can be bound to the slot
number such that: bucket = slot % num_buckets.

This both simplifies the logic and helps children maintenance per bucket in
threaded MPMs, where previously perform_idle_server_maintenance() could create
or kill children processes for the buckets it was not in charge of.

Submitted by: ylavic
Reviewed by: ylavic, rpluem, jorton


git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1855378 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES                      |  3 +++
 include/scoreboard.h         |  4 +++-
 server/mpm/event/event.c     | 13 ++++++++-----
 server/mpm/prefork/prefork.c | 19 +++++++------------
 server/mpm/worker/worker.c   | 10 ++++++----
 5 files changed, 27 insertions(+), 22 deletions(-)

Index: httpd-2.4.23/include/scoreboard.h
===================================================================
--- httpd-2.4.23.orig/include/scoreboard.h	2016-02-25 11:27:27.000000000 +0100
+++ httpd-2.4.23/include/scoreboard.h	2019-04-03 14:25:01.501137706 +0200
@@ -143,7 +143,9 @@ struct process_score {
     apr_uint32_t lingering_close;   /* async connections in lingering close */
     apr_uint32_t keep_alive;        /* async connections in keep alive */
     apr_uint32_t suspended;         /* connections suspended by some module */
-    int bucket;             /* Listener bucket used by this child */
+    int bucket;  /* Listener bucket used by this child; this field is DEPRECATED
+                  * and no longer updated by the MPMs (i.e. always zero).
+                  */
 };
 
 /* Scoreboard is now in 'local' memory, since it isn't updated once created,
Index: httpd-2.4.23/server/mpm/event/event.c
===================================================================
--- httpd-2.4.23.orig/server/mpm/event/event.c	2016-06-28 13:46:34.000000000 +0200
+++ httpd-2.4.23/server/mpm/event/event.c	2019-04-03 14:27:04.377758149 +0200
@@ -354,7 +354,7 @@ typedef struct event_retained_data {
     int num_buckets, max_buckets;
 } event_retained_data;
 static event_retained_data *retained;
- 
+
 typedef struct event_child_bucket {
     ap_pod_t *pod;
     ap_listen_rec *listeners;
@@ -845,7 +845,7 @@ static int start_lingering_close_common(
         cs->pub.state = CONN_STATE_LINGER_NORMAL;
     }
     apr_atomic_inc32(&lingering_count);
-    if (in_worker) { 
+    if (in_worker) {
         notify_suspend(cs);
     }
     else {
@@ -2307,7 +2307,7 @@ static void child_main(int child_num_arg
         if (rv != APR_SUCCESS && rv != APR_ENOTIMPL) {
             ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(02436)
                          "WARNING: ThreadStackSize of %" APR_SIZE_T_FMT " is "
-                         "inappropriate, using default", 
+                         "inappropriate, using default",
                          ap_thread_stacksize);
         }
     }
@@ -2473,7 +2473,6 @@ static int make_child(server_rec * s, in
     }
     ap_scoreboard_image->parent[slot].quiescing = 0;
     ap_scoreboard_image->parent[slot].not_accepting = 0;
-    ap_scoreboard_image->parent[slot].bucket = bucket;
     event_note_child_started(slot, pid);
     return 0;
 }
@@ -2522,6 +2521,7 @@ static void perform_idle_server_maintena
         int any_dead_threads = 0;
         int all_dead_threads = 1;
         int child_threads_active = 0;
+        int bucket = i % num_buckets;
 
         if (i >= retained->max_daemons_limit &&
             totally_free_length == retained->idle_spawn_rate[child_bucket]) {
@@ -2553,7 +2553,7 @@ static void perform_idle_server_maintena
                                    for loop if no pid?  not much else matters */
                 if (status <= SERVER_READY && !ps->quiescing && !ps->not_accepting
                     && ps->generation == retained->my_generation
-                    && ps->bucket == child_bucket)
+                    && bucket == child_bucket)
                 {
                     ++idle_thread_count;
                 }
@@ -2564,6 +2564,7 @@ static void perform_idle_server_maintena
         }
         active_thread_count += child_threads_active;
         if (any_dead_threads
+            && bucket == child_bucket
             && totally_free_length < retained->idle_spawn_rate[child_bucket]
             && free_length < MAX_SPAWN_RATE / num_buckets
             && (!ps->pid      /* no process in the slot */
@@ -2697,7 +2698,7 @@ static void server_main_loop(int remaini
             child_slot = ap_find_child_by_pid(&pid);
             if (processed_status == APEXIT_CHILDFATAL) {
                 /* fix race condition found in PR 39311
-                 * A child created at the same time as a graceful happens 
+                 * A child created at the same time as a graceful happens
                  * can find the lock missing and create a fatal error.
                  * It is not fatal for the last generation to be in this state.
                  */
@@ -2735,14 +2736,15 @@ static void server_main_loop(int remaini
                 ps->quiescing = 0;
                 if (processed_status == APEXIT_CHILDSICK) {
                     /* resource shortage, minimize the fork rate */
-                    retained->idle_spawn_rate[ps->bucket] = 1;
+                    retained->idle_spawn_rate[child_slot % num_buckets] = 1;
                 }
                 else if (remaining_children_to_start
                          && child_slot < ap_daemons_limit) {
                     /* we're still doing a 1-for-1 replacement of dead
                      * children with new children
                      */
-                    make_child(ap_server_conf, child_slot, ps->bucket);
+                     make_child(ap_server_conf, child_slot,
+                                                    child_slot % num_buckets);
                     --remaining_children_to_start;
                 }
             }
@@ -3005,13 +3007,13 @@ static int event_run(apr_pool_t * _pconf
     return OK;
 }
 
-static void setup_slave_conn(conn_rec *c, void *csd) 
+static void setup_slave_conn(conn_rec *c, void *csd)
 {
     event_conn_state_t *mcs;
     event_conn_state_t *cs;
-    
+
     mcs = ap_get_module_config(c->master->conn_config, &mpm_event_module);
-    
+
     cs = apr_pcalloc(c->pool, sizeof(*cs));
     cs->c = c;
     cs->r = NULL;
@@ -3023,7 +3025,7 @@ static void setup_slave_conn(conn_rec *c
     cs->pub = mcs->pub;
     cs->pub.state = CONN_STATE_READ_REQUEST_LINE;
     cs->pub.sense = CONN_SENSE_DEFAULT;
-    
+
     c->cs = &(cs->pub);
     ap_set_module_config(c->conn_config, &mpm_event_module, cs);
 }
@@ -3047,7 +3049,7 @@ static int event_protocol_switch(conn_re
          * other than http/1.1, this might never happen.
          */
         event_conn_state_t *cs;
-        
+
         cs = ap_get_module_config(c->conn_config, &mpm_event_module);
         cs->sc = ap_get_module_config(s->module_config, &mpm_event_module);
     }
Index: httpd-2.4.23/server/mpm/prefork/prefork.c
===================================================================
--- httpd-2.4.23.orig/server/mpm/prefork/prefork.c	2016-04-01 14:26:05.000000000 +0200
+++ httpd-2.4.23/server/mpm/prefork/prefork.c	2019-04-03 14:25:01.501137706 +0200
@@ -745,8 +745,9 @@ static void child_main(int child_num_arg
 }
 
 
-static int make_child(server_rec *s, int slot, int bucket)
+static int make_child(server_rec *s, int slot)
 {
+    int bucket = slot % num_buckets;
     int pid;
 
     if (slot + 1 > retained->max_daemons_limit) {
@@ -824,7 +825,6 @@ static int make_child(server_rec *s, int
         child_main(slot, bucket);
     }
 
-    ap_scoreboard_image->parent[slot].bucket = bucket;
     prefork_note_child_started(slot, pid);
 
     return 0;
@@ -840,7 +840,7 @@ static void startup_children(int number_
         if (ap_scoreboard_image->servers[i][0].status != SERVER_DEAD) {
             continue;
         }
-        if (make_child(ap_server_conf, i, i % num_buckets) < 0) {
+        if (make_child(ap_server_conf, i) < 0) {
             break;
         }
         --number_to_start;
@@ -849,8 +849,6 @@ static void startup_children(int number_
 
 static void perform_idle_server_maintenance(apr_pool_t *p)
 {
-    static int bucket_make_child_record = -1;
-    static int bucket_kill_child_record = -1;
     int i;
     int idle_count;
     worker_score *ws;
@@ -897,6 +895,7 @@ static void perform_idle_server_maintena
     }
     retained->max_daemons_limit = last_non_dead + 1;
     if (idle_count > ap_daemons_max_free) {
+        static int bucket_kill_child_record = -1;
         /* kill off one child... we use the pod because that'll cause it to
          * shut down gracefully, in case it happened to pick up a request
          * while we were counting
@@ -927,10 +926,7 @@ static void perform_idle_server_maintena
                     idle_count, total_non_dead);
             }
             for (i = 0; i < free_length; ++i) {
-                bucket_make_child_record++;
-                bucket_make_child_record %= num_buckets;
-                make_child(ap_server_conf, free_slots[i],
-                           bucket_make_child_record);
+                make_child(ap_server_conf, free_slots[i]);
             }
             /* the next time around we want to spawn twice as many if this
              * wasn't good enough, but not if we've just done a graceful
@@ -976,7 +972,7 @@ static int prefork_run(apr_pool_t *_pcon
 
     if (one_process) {
         AP_MONCONTROL(1);
-        make_child(ap_server_conf, 0, 0);
+        make_child(ap_server_conf, 0);
         /* NOTREACHED */
         ap_assert(0);
         return !OK;
@@ -1051,7 +1047,7 @@ static int prefork_run(apr_pool_t *_pcon
             child_slot = ap_find_child_by_pid(&pid);
             if (processed_status == APEXIT_CHILDFATAL) {
                 /* fix race condition found in PR 39311
-                 * A child created at the same time as a graceful happens 
+                 * A child created at the same time as a graceful happens
                  * can find the lock missing and create a fatal error.
                  * It is not fatal for the last generation to be in this state.
                  */
@@ -1085,8 +1081,7 @@ static int prefork_run(apr_pool_t *_pcon
                     /* we're still doing a 1-for-1 replacement of dead
                      * children with new children
                      */
-                    make_child(ap_server_conf, child_slot,
-                               ap_get_scoreboard_process(child_slot)->bucket);
+                    make_child(ap_server_conf, child_slot);
                     --remaining_children_to_start;
                 }
 #if APR_HAS_OTHER_CHILD
Index: httpd-2.4.23/server/mpm/worker/worker.c
===================================================================
--- httpd-2.4.23.orig/server/mpm/worker/worker.c	2016-06-28 13:46:34.000000000 +0200
+++ httpd-2.4.23/server/mpm/worker/worker.c	2019-04-03 14:25:01.505137727 +0200
@@ -1305,7 +1305,7 @@ static void child_main(int child_num_arg
         if (rv != APR_SUCCESS && rv != APR_ENOTIMPL) {
             ap_log_error(APLOG_MARK, APLOG_WARNING, rv, ap_server_conf, APLOGNO(02435)
                          "WARNING: ThreadStackSize of %" APR_SIZE_T_FMT " is "
-                         "inappropriate, using default", 
+                         "inappropriate, using default",
                          ap_thread_stacksize);
         }
     }
@@ -1467,7 +1467,6 @@ static int make_child(server_rec *s, int
         worker_note_child_lost_slot(slot, pid);
     }
     ap_scoreboard_image->parent[slot].quiescing = 0;
-    ap_scoreboard_image->parent[slot].bucket = bucket;
     worker_note_child_started(slot, pid);
     return 0;
 }
@@ -1516,6 +1515,7 @@ static void perform_idle_server_maintena
         int any_dead_threads = 0;
         int all_dead_threads = 1;
         int child_threads_active = 0;
+        int bucket = i % num_buckets;
 
         if (i >= retained->max_daemons_limit &&
             totally_free_length == retained->idle_spawn_rate[child_bucket]) {
@@ -1548,7 +1548,7 @@ static void perform_idle_server_maintena
                 if (status <= SERVER_READY &&
                         !ps->quiescing &&
                         ps->generation == retained->my_generation &&
-                        ps->bucket == child_bucket) {
+                        bucket == child_bucket) {
                     ++idle_thread_count;
                 }
                 if (status >= SERVER_READY && status < SERVER_GRACEFUL) {
@@ -1558,6 +1558,7 @@ static void perform_idle_server_maintena
         }
         active_thread_count += child_threads_active;
         if (any_dead_threads
+                && bucket == child_bucket
                 && totally_free_length < retained->idle_spawn_rate[child_bucket]
                 && free_length < MAX_SPAWN_RATE / num_buckets
                 && (!ps->pid               /* no process in the slot */
@@ -1705,7 +1706,7 @@ static void server_main_loop(int remaini
             child_slot = ap_find_child_by_pid(&pid);
             if (processed_status == APEXIT_CHILDFATAL) {
                 /* fix race condition found in PR 39311
-                 * A child created at the same time as a graceful happens 
+                 * A child created at the same time as a graceful happens
                  * can find the lock missing and create a fatal error.
                  * It is not fatal for the last generation to be in this state.
                  */
@@ -1743,14 +1744,15 @@ static void server_main_loop(int remaini
                 ps->quiescing = 0;
                 if (processed_status == APEXIT_CHILDSICK) {
                     /* resource shortage, minimize the fork rate */
-                    retained->idle_spawn_rate[ps->bucket] = 1;
+                    retained->idle_spawn_rate[child_slot % num_buckets] = 1;
                 }
                 else if (remaining_children_to_start
                     && child_slot < ap_daemons_limit) {
                     /* we're still doing a 1-for-1 replacement of dead
                      * children with new children
                      */
-                    make_child(ap_server_conf, child_slot, ps->bucket);
+                    make_child(ap_server_conf, child_slot,
+                               child_slot % num_buckets);
                     --remaining_children_to_start;
                 }
             }
@@ -2527,4 +2529,3 @@ AP_DECLARE_MODULE(mpm_worker) = {
     worker_cmds,                /* command apr_table_t */
     worker_hooks                /* register_hooks */
 };
-
