commit a52691efec2e784ec496f145acc4230ea2c8e14c
Author: Mark Thomas <markt@apache.org>
Date:   Fri Aug 24 12:12:27 2018 +0000

    Fix https://bz.apache.org/bugzilla/show_bug.cgi?id=62408
    Add the new load-balancer worker property lb_retries to improve the control over the number of retries.
    Based on a patch provided by Frederik Nosi.
    
    git-svn-id: https://svn.apache.org/repos/asf/tomcat/jk/trunk@1838830 13f79535-47bb-0310-9956-ffa450edef68

Index: tomcat-connectors-1.2.40-src/native/common/jk_ajp_common.c
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_ajp_common.c
+++ tomcat-connectors-1.2.40-src/native/common/jk_ajp_common.c
@@ -2909,6 +2909,10 @@ int ajp_init(jk_worker_t *pThis,
             jk_get_worker_retries(props, p->name,
                                   JK_RETRIES);
 
+        p->lb_retries =
+            jk_get_worker_lb_retries(props, p->name,
+                                  JK_LB_RETRIES);
+
         p->max_packet_size =
             jk_get_max_packet_size(props, p->name);
 
Index: tomcat-connectors-1.2.40-src/native/common/jk_ajp_common.h
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_ajp_common.h
+++ tomcat-connectors-1.2.40-src/native/common/jk_ajp_common.h
@@ -363,6 +363,13 @@ struct ajp_worker
      */
     int retries;
 
+    /*
+     * Public property used in load balancer workers, meaning
+     * the maximum number of failover attempts between ajp
+     * workers of cluster.
+     */
+    int lb_retries;
+
     unsigned int max_packet_size;  /*  Maximum AJP Packet size */
 
     int retry_interval;            /*  Number of milliseconds to sleep before doing a retry */
Index: tomcat-connectors-1.2.40-src/native/common/jk_lb_worker.c
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_lb_worker.c
+++ tomcat-connectors-1.2.40-src/native/common/jk_lb_worker.c
@@ -1164,6 +1164,7 @@ static int JK_METHOD service(jk_endpoint
     if (p->worker->sequence < p->worker->s->h.sequence)
         jk_lb_pull(p->worker, JK_FALSE, l);
     for (i = 0; i < num_of_workers; i++) {
+        jk_log(l, JK_LOG_DEBUG, "LB - num_of_workers: %d, retry: %d, lb_retries: %d", num_of_workers, i, p->worker->lb_retries);
         lb_sub_worker_t *rec = &(p->worker->lb_workers[i]);
         if (rec->s->state == JK_LB_STATE_BUSY) {
             if (ajp_has_endpoint(rec->worker, l)) {
@@ -1209,7 +1210,10 @@ static int JK_METHOD service(jk_endpoint
                p->worker->sticky_session, sessionid ? sessionid : "empty");
 
     while (recoverable == JK_TRUE) {
-        if (attempt >= num_of_workers) {
+        if (JK_IS_DEBUG_LEVEL(l))
+            jk_log(l, JK_LOG_DEBUG, "attempt %d, max attempts %d, worker count %d",
+                    attempt, p->worker->lb_retries, num_of_workers);
+        if (attempt >= num_of_workers || attempt >= p->worker->lb_retries) {
             retry++;
             if (retry >= p->worker->retries) {
                 /* Done with retrying */
@@ -1479,7 +1483,7 @@ static int JK_METHOD service(jk_endpoint
                     }
                     else {
                         /*
-                         * Reply timeout, bot not yet too many of them.
+                         * Reply timeout, but not yet too many of them.
                          * Keep previous global state.
                          * Do not try to reuse the same node for the same request.
                          * Failing over to another node could help.
@@ -1811,6 +1815,8 @@ static int JK_METHOD init(jk_worker_t *p
     p->worker.we = we;
     p->retries = jk_get_worker_retries(props, p->name,
                                        JK_RETRIES);
+    p->lb_retries = jk_get_worker_lb_retries(props, p->name,
+                                       JK_LB_RETRIES);
     p->retry_interval =
             jk_get_worker_retry_interval(props, p->name,
                                         JK_SLEEP_DEF);
Index: tomcat-connectors-1.2.40-src/native/common/jk_lb_worker.h
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_lb_worker.h
+++ tomcat-connectors-1.2.40-src/native/common/jk_lb_worker.h
@@ -190,6 +190,7 @@ struct lb_worker
     int          error_escalation_time;
     int          max_reply_timeouts;
     int          retries;
+    int          lb_retries;
     int          retry_interval;
     int          lbmethod;
     int          lblock;
Index: tomcat-connectors-1.2.40-src/native/common/jk_service.h
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_service.h
+++ tomcat-connectors-1.2.40-src/native/common/jk_service.h
@@ -36,6 +36,7 @@
 #include "jk_msg_buff.h"
 
 #define JK_RETRIES 2
+#define JK_LB_RETRIES 2
 
 #ifdef __cplusplus
 extern "C"
Index: tomcat-connectors-1.2.40-src/native/common/jk_util.c
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_util.c
+++ tomcat-connectors-1.2.40-src/native/common/jk_util.c
@@ -109,6 +109,7 @@
 #define DEFAULT_WORKER_TYPE         JK_AJP13_WORKER_NAME
 #define SECRET_KEY_OF_WORKER        "secretkey"
 #define RETRIES_OF_WORKER           "retries"
+#define LB_RETRIES_OF_WORKER        "lb_retries"
 #define STATUS_FAIL_OF_WORKER       "fail_on_status"
 
 #define DEFAULT_WORKER              JK_AJP13_WORKER_NAME
@@ -228,6 +229,7 @@ static const char *unique_properties[] =
     STYLE_SHEET_OF_WORKER,
     READ_ONLY_OF_WORKER,
     RETRIES_OF_WORKER,
+    LB_RETRIES_OF_WORKER,
     WORKER_MAINTAIN_PROPERTY_NAME,
     NAMESPACE_OF_WORKER,
     XML_NAMESPACE_OF_WORKER,
@@ -336,6 +338,7 @@ static const char *supported_properties[
     BAD_RATING_OF_WORKER,
     SECRET_KEY_OF_WORKER,
     RETRIES_OF_WORKER,
+    LB_RETRIES_OF_WORKER,
     STATUS_FAIL_OF_WORKER,
     LIST_PROPERTY_NAME,
     MAINTAIN_PROPERTY_NAME,
@@ -1219,6 +1222,24 @@ int jk_get_worker_retries(jk_map_t *m, c
     return rv;
 }
 
+int jk_get_worker_lb_retries(jk_map_t *m, const char *wname, int def)
+{
+    char buf[1024];
+    int rv;
+    if (!m || !wname) {
+        return -1;
+    }
+
+    MAKE_WORKER_PARAM(LB_RETRIES_OF_WORKER);
+
+    rv = jk_map_get_int(m, buf, def);
+    if (rv < 1)
+        rv = 1;
+
+    return rv;
+}
+
+
 int jk_get_worker_recovery_opts(jk_map_t *m, const char *wname, int def)
 {
     char buf[PARAM_BUFFER_SIZE];
Index: tomcat-connectors-1.2.40-src/xdocs/reference/workers.xml
===================================================================
--- tomcat-connectors-1.2.40-src.orig/xdocs/reference/workers.xml
+++ tomcat-connectors-1.2.40-src/xdocs/reference/workers.xml
@@ -601,15 +601,59 @@ This feature has been added in <b>jk 1.2
 
 <directive name="retries" default="2" required="false">
 <warn>This directive also exists for normal workers.
-For those it has a <a href="#Advanced Worker Directives">different meaning</a>.</warn>
-If the load balancer can not get a valid member worker or in case of failover,
-it will try again a number of times given by <b>retries</b>.
-Before each retry, it will make a pause define by <b>retry_interval</b> directive.
+For those it has a <a href="#Advanced Worker Directives">different
+meaning</a>.</warn>
+When making a request, the load-balancer worker will allocate the request to a
+member worker. If that member worker is either unable to service the request or
+fails to service the request, the request will be passed to another member
+worker until the request is processed, every member worker has attempted to
+process the request or <b>lb_retries</b> member workers have attempted to
+process the request.
+<p>
+If the request remains unprocessed, the load-balancer worker will repeat the
+above process a maximum of <b>retries</b> times (including the original
+attempt). Before each retry, the load-balancer worker will pause for a time
+defined by the <b>retry_interval</b> directive.
+</p>
+<p>
+ Note that this means that, if all workers fail, there will be a total of
+worker.retries * min(lb.lb_retries,member worker count) * lb.retries requests
+before a 504 response is returned to the client. So for an lb worker with four
+members and a default configuration, if all workers fail there will be a total
+of 8 requests before a 504 response is returned to the client.
+</p>
 <p>
 Until version <b>1.2.16</b> the default value was 3.
 </p>
 </directive>
 
+<directive name="lb_retries" default="2" required="false">
+When making a request, the load-balancer worker will allocate the request to a
+member worker. If that member worker is either unable to service the request or
+fails to service the request, the request will be passed to another member
+worker until the request is processed, every member worker has attempted to
+process the request or <b>lb_retries</b> member workers have attempted to
+ process the request.
+<p>
+If the request remains unprocessed, the load-balancer worker will repeat the
+above process a maximum of <b>retries</b> times (including the original
+attempt). Before each retry, the load-balancer worker will pause for a time
+defined by the <b>retry_interval</b> directive.
+</p>
+<p>
+Note that this means that, if all workers fail, there will be a total of
+worker.retries * min(lb.lb_retries,member worker count) * lb.retries requests
+before a 504 response is returned to the client. So for an lb worker with four
+members and a default configuration, if all workers fail there will be a total
+of 8 requests before a 504 response is returned to the client.
+</p>
+<p>
+This feature has been added in <b>jk 1.2.44</b>. Prior to this feature being
+added, the load-balancer worker behaved as if <b>lb_retries</b> was equal to the
+number of member workers.
+</p>
+</directive>
+
 </directives>
 
 </subsection>
Index: tomcat-connectors-1.2.40-src/native/common/jk_util.h
===================================================================
--- tomcat-connectors-1.2.40-src.orig/native/common/jk_util.h
+++ tomcat-connectors-1.2.40-src/native/common/jk_util.h
@@ -131,6 +131,8 @@ const char *jk_get_lb_session_cookie_pat
 
 int jk_get_worker_retries(jk_map_t *m, const char *wname, int def);
 
+int jk_get_worker_lb_retries(jk_map_t *m, const char *wname, int def);
+
 int jk_get_is_worker_disabled(jk_map_t *m, const char *wname);
 
 int jk_get_is_worker_stopped(jk_map_t *m, const char *wname);
