=== modified file 'doc/release-notes/release-3.2.sgml' --- doc/release-notes/release-3.2.sgml 2009-11-22 20:37:27 +0000 +++ doc/release-notes/release-3.2.sgml 2009-11-26 10:10:14 +0000 @@ -320,12 +320,22 @@ acl random

New type random. Pseudo-randomly match requests based on a configured probability. + auth_param +

New options for Basic, Digest, NTLM, Negotiate children settings. + startup=N determins minimum number of helper processes used. + idle=N determines how many helper to retain as buffer against sudden traffic loads. + deny_info

Support URL format tags. For dynamically generated URL in denial redirect. external_acl_type -

New format tag %SRCEUI48 EUI-48 / MAC address of client from ARP lookup. -

New format tag %SRCEUI64 EUI-64 of clients with SLAAC address. +

New format tags and option parameters: +

%SRCEUI48 EUI-48 / MAC address of client from ARP lookup. +

%SRCEUI64 EUI-64 of clients with SLAAC address. +

children-max=N determins maximum number of helper processes used. +

children-startup=N determins minimum number of helper processes used. +

children-idle=N determines how many helper to retain as buffer against sudden traffic loads. +

Deprecated children=N in favor of children-max=N. logformat

%sn Unique sequence number per log line. Ported from 2.7 @@ -335,6 +345,12 @@ windows_ipaddrchangemonitor

Now only available to be set in Windows builds. + url_rewrite_children + New options startup=N, idle=N, concurrency=N +

startup=N allow finer tuning of how many helpers are started initially. +

idle=N allow fine tuning of how many helper to retain as buffer against sudden traffic loads. +

concurrency=N was previously called url_rewrite_concurrency as a distinct directive. + @@ -344,6 +360,9 @@ ftp_list_width

Obsolete. + url_rewrite_concurrency +

Replaced by url_rewrite_children ... concurrency=N option. + === modified file 'helpers/log_daemon/file/log_file_daemon.cc' --- helpers/log_daemon/file/log_file_daemon.cc 2009-11-23 01:16:57 +0000 +++ helpers/log_daemon/file/log_file_daemon.cc 2009-11-26 11:10:25 +0000 @@ -114,6 +114,28 @@ case 'L': if (buf[1] != '\0') { fprintf(fp, "%s", buf + 1); + /* try to detect the 32-bit file too big write error and rotate */ + int err = ferror(fp); + clearerr(fp); + if (err < 0) { + /* file too big - recover by rotating the logs and starting a new one. + * out of device space - recover by rotating and hoping that rotation count drops a big one. + */ + if (err == EFBIG || err == ENOSPC) { + fprintf(stderr, "WARNING: %s writing %s. Attempting to recover via a log rotation.\n",strerror(err),argv[1]); + fclose(fp); + rotate(argv[1], rotate_count); + fp = fopen(argv[1], "a"); + if (fp == NULL) { + perror("fopen"); + exit(1); + } + fprintf(fp, "%s", buf + 1); + } else { + perror("fprintf"); + exit(1); + } + } } if (!do_buffer) fflush(fp); === added file 'src/HelperChildConfig.cc' --- src/HelperChildConfig.cc 1970-01-01 00:00:00 +0000 +++ src/HelperChildConfig.cc 2009-12-04 00:38:23 +0000 @@ -0,0 +1,89 @@ +#include "config.h" +#include "HelperChildConfig.h" +#include "globals.h" + +#include + +HelperChildConfig::HelperChildConfig() : + n_max(0), + n_startup(1), + n_idle(1), + concurrency(1), + n_running(0), + n_active(0) +{} + +HelperChildConfig::HelperChildConfig(const unsigned int m, const unsigned int s, const unsigned int i, const unsigned int cc) : + n_max(m), + n_startup(s), + n_idle(i), + concurrency(cc), + n_running(0), + n_active(0) +{} + +HelperChildConfig::~HelperChildConfig() +{} + +HelperChildConfig & +HelperChildConfig::operator =(const HelperChildConfig &rhs) +{ + memcpy(this, &rhs, sizeof(HelperChildConfig)); + return *this; +} + +const int +HelperChildConfig::needNew() const { + /* during the startup and reconfigure use our special amount... */ + if (starting_up || reconfiguring) return n_startup; + + /* keep a minimum of n_idle helpers free... */ + if ( (n_active + n_idle) < n_max) return n_idle; + + /* dont ever start more than n_max processes. */ + return (n_max - n_active); +} + +void +HelperChildConfig::parseConfig() +{ + char const *token = strtok(NULL, w_space); + + if (!token) + self_destruct(); + + /* starts with a bare number for the max... back-compatible */ + n_max = atoi(token); + + if (n_max < 1) + self_destruct(); + + /* Parse extension options */ + for (; (token = strtok(NULL, w_space)) ;) { + if (strncmp(token, "startup=", 8) == 0) { + n_startup = atoi(token + 8); + } else if (strncmp(token, "idle=", 5) == 0) { + n_idle = atoi(token + 5); + if (n_idle < 1) { + debugs(0,0,"WARNING OVERIDE: Using idle=0 for helpers causes request failures. Overiding to use idle=1 instead."); + n_idle = 1; + } + } else if (strncmp(token, "concurrency=", 12) == 0) { + concurrency = atoi(token + 12); + } else { + self_destruct(); + } + } + + /* simple sanity. */ + + if (n_startup > n_max) { + debugs(0,0,"WARNING OVERIDE: Capping startup=" << n_startup << " to the defined maximum (" << n_max <<")"); + n_startup = n_max; + } + + if (n_idle > n_max) { + debugs(0,0,"WARNING OVERIDE: Capping idle=" << n_idle << " to the defined maximum (" << n_max <<")"); + n_idle = n_max; + } +} === added file 'src/HelperChildConfig.h' --- src/HelperChildConfig.h 1970-01-01 00:00:00 +0000 +++ src/HelperChildConfig.h 2009-12-04 00:35:05 +0000 @@ -0,0 +1,78 @@ +#ifndef _SQUID_SRC_HELPERCHILDCONFIG_H +#define _SQUID_SRC_HELPERCHILDCONFIG_H + +/** + * Contains statistics of a particular type of child helper. + * + * Some derived from a helper children configuration option, + * some from runtime stats on the currently active children. + */ +class HelperChildConfig { +public: + HelperChildConfig(); + HelperChildConfig(const unsigned int m, const unsigned int s, const unsigned int i, const unsigned int cc); + ~HelperChildConfig(); + HelperChildConfig &operator =(const HelperChildConfig &rhs); + + /* + * When new helpers are needed call this to find out how many more + * we are allowed to start. + * \retval 0 No more helpers may be started right now. + * \retval N < 0 Error. No more helpers may be started. + * \retval N N more helpers may be started immediately. + */ + const int needNew() const; + void parseConfig(); + + /* values from squid.conf */ +public: + + /** maximum child process limits. How many of this helper the system can cope with */ + unsigned int n_max; + + /** + * Number of children to kick off at startup. + * set via the startup=N option. + * + * By default if undefined 1 will be started immediately for use. + * The minimum/idle amount will be scheduled for starting as soon as possible after startup is completed. + */ + unsigned int n_startup; + + /** + * Number of helper children to keep available as a buffer against sudden bursts of requests. + * set via the idle=N option. May be zero. + * + * The default value for backward compatibility the default for this is the same as maximum children. + * For now the actual number of idle children is only reduced by a reconfigure operation. This may change. + */ + unsigned int n_idle; + + /** + * How many concurrent requests each child helper may be capable of handling. + * Default: 1 - no concurrency possible. + */ + unsigned int concurrency; + + /* derived from active operations */ +public: + + /** + * Total helper children objects currently existing. + * Produced as a side effect of starting children or their stopping. + */ + unsigned int n_running; + + /** + * Count of helper children active (not shutting down). + * This includes both idle and in-use children. + */ + unsigned int n_active; +}; + +/* Legacy parser interface */ +#define parse_HelperChildConfig(c) (c)->parseConfig() +#define dump_HelperChildConfig(e,n,c) storeAppendPrintf((e), "\n%s %d startup=%d idle=%d\n", (n), (c).n_max, (c).n_startup, (c).n_idle) +#define free_HelperChildConfig(dummy) // NO. + +#endif /* _SQUID_SRC_HELPERCHILDCONFIG_H */ === modified file 'src/Makefile.am' --- src/Makefile.am 2009-12-03 10:58:30 +0000 +++ src/Makefile.am 2009-12-03 22:13:57 +0000 @@ -330,6 +330,8 @@ gopher.cc \ helper.cc \ helper.h \ + HelperChildConfig.h \ + HelperChildConfig.cc \ hier_code.h \ HierarchyLogEntry.h \ $(HTCPSOURCE) \ @@ -600,6 +602,7 @@ ## mem.cc wants ClientInfo.h ## libbase.la wants cbdata.* ## libbase.la wants MemBuf.* +## structs.h wants HelperChildConfig.* (stub it) ufsdump_SOURCES = \ ClientInfo.h \ cbdata.h \ @@ -624,6 +627,8 @@ ufsdump.cc \ dlink.h \ dlink.cc \ + HelperChildConfig.h \ + tests/stub_HelperChildConfig.cc \ HttpRequestMethod.cc \ RemovalPolicy.cc \ squid.h \ @@ -938,6 +943,7 @@ Packer.h \ Packer.cc \ tests/stub_cache_manager.cc \ + tests/stub_HelperChildConfig.cc \ tests/stub_StatHist.cc \ tests/stub_store.cc \ SquidString.h \ @@ -962,6 +968,8 @@ tests_testAuth_SOURCES = \ tests/testAuth.cc tests/testMain.cc tests/testAuth.h \ ConfigParser.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ tests/stub_acl.cc tests/stub_cache_cf.cc \ tests/stub_helper.cc cbdata.cc String.cc \ tests/stub_store.cc HttpHeaderTools.cc HttpHeader.cc mem.cc ClientInfo.h \ @@ -1023,6 +1031,8 @@ ClientInfo.h \ ConfigParser.cc \ ETag.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ HttpHeader.cc \ HttpHeaderTools.cc \ HttpHdrContRange.cc \ @@ -1127,6 +1137,8 @@ gopher.cc \ hier_code.h \ helper.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ $(HTCPSOURCE) \ http.cc \ HttpBody.cc \ @@ -1234,7 +1246,8 @@ tests/testDiskIO.cc \ tests/testDiskIO.h \ tests/testMain.cc \ - tests/stub_cache_manager.cc + tests/stub_cache_manager.cc \ + tests/stub_HelperChildConfig.cc nodist_tests_testDiskIO_SOURCES= \ $(SWAP_TEST_GEN_SOURCES) \ SquidMath.cc \ @@ -1304,6 +1317,8 @@ gopher.cc \ hier_code.h \ helper.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ $(HTCPSOURCE) \ http.cc \ HttpBody.cc \ @@ -1453,6 +1468,8 @@ ftp.cc \ gopher.cc \ helper.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ hier_code.h \ $(HTCPSOURCE) \ http.cc \ @@ -1592,6 +1609,8 @@ ftp.cc \ gopher.cc \ helper.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ hier_code.h \ $(HTCPSOURCE) \ http.cc \ @@ -1747,6 +1766,8 @@ ftp.cc \ gopher.cc \ helper.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ hier_code.h \ $(HTCPSOURCE) \ http.cc \ @@ -1908,6 +1929,7 @@ tests/TestSwapDir.cc \ tests/TestSwapDir.h \ tests/stub_fd.cc \ + tests/stub_HelperChildConfig.cc \ tests/stub_HttpReply.cc \ tests/stub_cache_manager.cc \ $(STORE_TEST_SOURCES) @@ -1940,6 +1962,7 @@ tests/testString.cc \ tests/testString.h \ tests/stub_cache_manager.cc \ + tests/stub_HelperChildConfig.cc \ time.cc nodist_tests_testString_SOURCES = \ $(TESTSOURCES) @@ -2012,6 +2035,7 @@ tests/testMain.cc \ tests/testUfs.h \ tests/stub_cache_manager.cc \ + tests/stub_HelperChildConfig.cc \ $(SWAP_TEST_SOURCES) nodist_tests_testUfs_SOURCES = \ $(SWAP_TEST_GEN_SOURCES) \ @@ -2032,6 +2056,7 @@ tests/testMain.cc \ tests/testCoss.h \ tests/stub_cache_manager.cc \ + tests/stub_HelperChildConfig.cc \ $(SWAP_TEST_SOURCES) nodist_tests_testCoss_SOURCES = \ swap_log_op.cc \ @@ -2111,6 +2136,8 @@ ftp.cc \ gopher.cc \ helper.cc \ + HelperChildConfig.h \ + HelperChildConfig.cc \ hier_code.h \ $(HTCPSOURCE) \ http.cc \ === modified file 'src/auth/basic/auth_basic.cc' --- src/auth/basic/auth_basic.cc 2009-11-17 15:44:34 +0000 +++ src/auth/basic/auth_basic.cc 2009-12-03 22:53:13 +0000 @@ -106,7 +106,7 @@ bool AuthBasicConfig::configured() const { - if ((authenticate != NULL) && (authenticateChildren != 0) && + if ((authenticate != NULL) && (authenticateChildren.n_max != 0) && (basicAuthRealm != NULL)) { debugs(29, 9, HERE << "returning configured"); return true; @@ -303,16 +303,15 @@ storeAppendPrintf(entry, "\n"); storeAppendPrintf(entry, "%s basic realm %s\n", name, basicAuthRealm); - storeAppendPrintf(entry, "%s basic children %d\n", name, authenticateChildren); + storeAppendPrintf(entry, "%s basic children %d startup=%d idle=%d\n", name, authenticateChildren.n_max, authenticateChildren.n_startup, authenticateChildren.n_idle); storeAppendPrintf(entry, "%s basic concurrency %d\n", name, authenticateConcurrency); storeAppendPrintf(entry, "%s basic credentialsttl %d seconds\n", name, (int) credentialsTTL); storeAppendPrintf(entry, "%s basic casesensitive %s\n", name, casesensitive ? "on" : "off"); } -AuthBasicConfig::AuthBasicConfig() +AuthBasicConfig::AuthBasicConfig() : authenticateChildren(20,0,1,1) { /* TODO: move into initialisation list */ - authenticateChildren = 5; credentialsTTL = 2 * 60 * 60; /* two hours */ basicAuthRealm = xstrdup("Squid proxy-caching web server"); } @@ -333,7 +332,7 @@ requirePathnameExists("auth_param basic program", authenticate->key); } else if (strcasecmp(param_str, "children") == 0) { - parse_int(&authenticateChildren); + authenticateChildren.parseConfig(); } else if (strcasecmp(param_str, "concurrency") == 0) { parse_int(&authenticateConcurrency); } else if (strcasecmp(param_str, "realm") == 0) { @@ -611,9 +610,9 @@ basicauthenticators->cmdline = authenticate; - basicauthenticators->n_to_start = authenticateChildren; + basicauthenticators->childs = authenticateChildren; - basicauthenticators->concurrency = authenticateConcurrency; + basicauthenticators->childs.concurrency = authenticateConcurrency; basicauthenticators->ipc_type = IPC_STREAM; === modified file 'src/auth/basic/auth_basic.h' --- src/auth/basic/auth_basic.h 2009-03-31 12:39:30 +0000 +++ src/auth/basic/auth_basic.h 2009-11-25 21:04:37 +0000 @@ -107,6 +107,8 @@ MEMPROXY_CLASS_INLINE(AuthBasicUserRequest); +#include "HelperChildConfig.h" + /* configuration runtime data */ class AuthBasicConfig : public AuthConfig @@ -125,7 +127,7 @@ virtual void parse(AuthConfig *, int, char *); virtual void registerWithCacheManager(void); virtual const char * type() const; - int authenticateChildren; + HelperChildConfig authenticateChildren; int authenticateConcurrency; char *basicAuthRealm; wordlist *authenticate; === modified file 'src/auth/digest/auth_digest.cc' --- src/auth/digest/auth_digest.cc 2009-11-17 16:56:02 +0000 +++ src/auth/digest/auth_digest.cc 2009-12-03 22:54:07 +0000 @@ -534,9 +534,9 @@ list = list->next; } - storeAppendPrintf(entry, "\n%s %s realm %s\n%s %s children %d\n%s %s nonce_max_count %d\n%s %s nonce_max_duration %d seconds\n%s %s nonce_garbage_interval %d seconds\n", + storeAppendPrintf(entry, "\n%s %s realm %s\n%s %s children %d startup=%d idle=%d\n%s %s nonce_max_count %d\n%s %s nonce_max_duration %d seconds\n%s %s nonce_garbage_interval %d seconds\n", name, "digest", digestAuthRealm, - name, "digest", authenticateChildren, + name, "digest", authenticateChildren.n_max, authenticateChildren.n_startup, authenticateChildren.n_idle, name, "digest", noncemaxuses, name, "digest", (int) noncemaxduration, name, "digest", (int) nonceGCInterval); @@ -552,7 +552,7 @@ AuthDigestConfig::configured() const { if ((authenticate != NULL) && - (authenticateChildren != 0) && + (authenticateChildren.n_max != 0) && (digestAuthRealm != NULL) && (noncemaxduration > -1)) return true; @@ -877,7 +877,7 @@ digestauthenticators->cmdline = authenticate; - digestauthenticators->n_to_start = authenticateChildren; + digestauthenticators->childs = authenticateChildren; digestauthenticators->ipc_type = IPC_STREAM; @@ -906,11 +906,9 @@ safe_free(digestAuthRealm); } - -AuthDigestConfig::AuthDigestConfig() +AuthDigestConfig::AuthDigestConfig() : authenticateChildren(20,0,1,1) { /* TODO: move into initialisation list */ - authenticateChildren = 5; /* 5 minutes */ nonceGCInterval = 5 * 60; /* 30 minutes */ @@ -934,7 +932,7 @@ requirePathnameExists("auth_param digest program", authenticate->key); } else if (strcasecmp(param_str, "children") == 0) { - parse_int(&authenticateChildren); + authenticateChildren.parseConfig(); } else if (strcasecmp(param_str, "realm") == 0) { parse_eol(&digestAuthRealm); } else if (strcasecmp(param_str, "nonce_garbage_interval") == 0) { === modified file 'src/auth/digest/auth_digest.h' --- src/auth/digest/auth_digest.h 2009-03-31 12:39:30 +0000 +++ src/auth/digest/auth_digest.h 2009-11-25 21:05:21 +0000 @@ -134,6 +134,8 @@ } flags; }; +#include "HelperChildConfig.h" + /* configuration runtime data */ class AuthDigestConfig : public AuthConfig @@ -151,7 +153,7 @@ virtual void parse(AuthConfig *, int, char *); virtual void registerWithCacheManager(void); virtual const char * type() const; - int authenticateChildren; + HelperChildConfig authenticateChildren; char *digestAuthRealm; wordlist *authenticate; time_t nonceGCInterval; === modified file 'src/auth/negotiate/auth_negotiate.cc' --- src/auth/negotiate/auth_negotiate.cc 2009-11-17 16:56:02 +0000 +++ src/auth/negotiate/auth_negotiate.cc 2009-12-03 22:54:20 +0000 @@ -138,13 +138,13 @@ list = list->next; } - storeAppendPrintf(entry, "\n%s negotiate children %d\n", - name, authenticateChildren); + storeAppendPrintf(entry, "\n%s negotiate children %d startup=%d idle=%d\n", + name, authenticateChildren.n_max, authenticateChildren.n_startup, authenticateChildren.n_idle); storeAppendPrintf(entry, "%s %s keep_alive %s\n", name, "negotiate", keep_alive ? "on" : "off"); } -AuthNegotiateConfig::AuthNegotiateConfig() : authenticateChildren(5), keep_alive(1) +AuthNegotiateConfig::AuthNegotiateConfig() : authenticateChildren(20,0,1,1), keep_alive(1) { } void @@ -158,7 +158,7 @@ requirePathnameExists("auth_param negotiate program", authenticate->key); } else if (strcasecmp(param_str, "children") == 0) { - parse_int(&authenticateChildren); + authenticateChildren.parseConfig(); } else if (strcasecmp(param_str, "keep_alive") == 0) { parse_onoff(&keep_alive); } else { @@ -204,7 +204,7 @@ negotiateauthenticators->cmdline = authenticate; - negotiateauthenticators->n_to_start = authenticateChildren; + negotiateauthenticators->childs = authenticateChildren; negotiateauthenticators->ipc_type = IPC_STREAM; @@ -232,7 +232,7 @@ bool AuthNegotiateConfig::configured() const { - if ((authenticate != NULL) && (authenticateChildren != 0)) { + if ((authenticate != NULL) && (authenticateChildren.n_max != 0)) { debugs(29, 9, "AuthNegotiateConfig::configured: returning configured"); return true; } === modified file 'src/auth/negotiate/auth_negotiate.h' --- src/auth/negotiate/auth_negotiate.h 2009-07-29 09:07:56 +0000 +++ src/auth/negotiate/auth_negotiate.h 2009-11-25 21:05:55 +0000 @@ -110,6 +110,8 @@ MEMPROXY_CLASS_INLINE(AuthNegotiateUserRequest); +#include "HelperChildConfig.h" + /* configuration runtime data */ /// \ingroup AuthNegotiateAPI @@ -128,7 +130,7 @@ virtual void parse(AuthConfig *, int, char *); virtual void registerWithCacheManager(void); virtual const char * type() const; - int authenticateChildren; + HelperChildConfig authenticateChildren; int keep_alive; wordlist *authenticate; }; === modified file 'src/auth/ntlm/auth_ntlm.cc' --- src/auth/ntlm/auth_ntlm.cc 2009-11-17 15:44:34 +0000 +++ src/auth/ntlm/auth_ntlm.cc 2009-12-03 22:54:30 +0000 @@ -120,13 +120,13 @@ list = list->next; } - storeAppendPrintf(entry, "\n%s ntlm children %d\n", - name, authenticateChildren); + storeAppendPrintf(entry, "\n%s ntlm children %d startup=%d idle=%d\n", + name, authenticateChildren.n_max, authenticateChildren.n_startup, authenticateChildren.n_idle); storeAppendPrintf(entry, "%s %s keep_alive %s\n", name, "ntlm", keep_alive ? "on" : "off"); } -AuthNTLMConfig::AuthNTLMConfig() : authenticateChildren(5), keep_alive(1) +AuthNTLMConfig::AuthNTLMConfig() : authenticateChildren(20,0,1,1), keep_alive(1) { } void @@ -140,7 +140,7 @@ requirePathnameExists("auth_param ntlm program", authenticate->key); } else if (strcasecmp(param_str, "children") == 0) { - parse_int(&authenticateChildren); + authenticateChildren.parseConfig(); } else if (strcasecmp(param_str, "keep_alive") == 0) { parse_onoff(&keep_alive); } else { @@ -184,7 +184,7 @@ ntlmauthenticators->cmdline = authenticate; - ntlmauthenticators->n_to_start = authenticateChildren; + ntlmauthenticators->childs = authenticateChildren; ntlmauthenticators->ipc_type = IPC_STREAM; @@ -212,7 +212,7 @@ bool AuthNTLMConfig::configured() const { - if ((authenticate != NULL) && (authenticateChildren != 0)) { + if ((authenticate != NULL) && (authenticateChildren.n_max != 0)) { debugs(29, 9, "AuthNTLMConfig::configured: returning configured"); return true; } === modified file 'src/auth/ntlm/auth_ntlm.h' --- src/auth/ntlm/auth_ntlm.h 2009-07-29 09:07:56 +0000 +++ src/auth/ntlm/auth_ntlm.h 2009-11-25 21:06:35 +0000 @@ -96,6 +96,8 @@ MEMPROXY_CLASS_INLINE(AuthNTLMUserRequest); +#include "HelperChildConfig.h" + /* configuration runtime data */ class AuthNTLMConfig : public AuthConfig @@ -113,7 +115,7 @@ virtual void parse(AuthConfig *, int, char *); virtual void registerWithCacheManager(void); virtual const char * type() const; - int authenticateChildren; + HelperChildConfig authenticateChildren; int keep_alive; wordlist *authenticate; }; === modified file 'src/cache_cf.cc' --- src/cache_cf.cc 2009-11-23 01:16:57 +0000 +++ src/cache_cf.cc 2009-11-29 04:58:53 +0000 @@ -472,14 +472,14 @@ #if USE_DNSSERVERS - if (Config.dnsChildren < 1) + if (Config.dnsChildren.n_max < 1) fatal("No dnsservers allocated"); #endif if (Config.Program.redirect) { - if (Config.redirectChildren < 1) { - Config.redirectChildren = 0; + if (Config.redirectChildren.n_max < 1) { + Config.redirectChildren.n_max = 0; wordlistDestroy(&Config.Program.redirect); } } === modified file 'src/cf.data.depend' --- src/cf.data.depend 2009-08-04 02:07:56 +0000 +++ src/cf.data.depend 2009-11-26 08:40:19 +0000 @@ -19,6 +19,7 @@ denyinfo acl eol externalAclHelper auth_param +HelperChildConfig hostdomain cache_peer hostdomaintype cache_peer http_header_access === modified file 'src/cf.data.pre' --- src/cf.data.pre 2009-11-29 05:33:51 +0000 +++ src/cf.data.pre 2009-12-03 22:50:24 +0000 @@ -130,13 +130,20 @@ translate the HTTP iso-latin-1 charset to UTF-8 before sending the username & password to the helper. - "children" numberofchildren - The number of authenticator processes to spawn. If you start too few + "children" numberofchildren [startup=N] [idle=N] + The maximum number of authenticator processes to spawn. If you start too few Squid will have to wait for them to process a backlog of credential verifications, slowing it down. When password verifications are done via a (slow) network you are likely to need lots of authenticator processes. - auth_param basic children 5 + + The startup= and idle= options permit some skew in the exact amount + run. A minimum of startup=N will begin during startup and reconfigure + and Squid will start more in groups of up to idle=N in an attempt to meet + traffic needs and to keep idle=N free above those traffic needs up to + the maximum. + + auth_param basic children 20 startup=0 idle=1 "concurrency" concurrency The number of concurrent requests the helper can process. @@ -198,13 +205,20 @@ translate the HTTP iso-latin-1 charset to UTF-8 before sending the username & password to the helper. - "children" numberofchildren - The number of authenticator processes to spawn (no default). + "children" numberofchildren [startup=N] [idle=N] + The maximum number of authenticator processes to spawn (default 5). If you start too few Squid will have to wait for them to process a backlog of H(A1) calculations, slowing it down. When the H(A1) calculations are done via a (slow) network you are likely to need lots of authenticator processes. - auth_param digest children 5 + + The startup= and idle= options permit some skew in the exact amount + run. A minimum of startup=N will begin during startup and reconfigure + and Squid will start more in groups of up to idle=N in an attempt to meet + traffic needs and to keep idle=N free above those traffic needs up to + the maximum. + + auth_param digest children 20 startup=0 idle=1 "realm" realmstring Specifies the realm name which is to be reported to the @@ -254,15 +268,21 @@ auth_param ntlm program @DEFAULT_PREFIX@/bin/ntlm_auth - "children" numberofchildren - The number of authenticator processes to spawn (no default). + "children" numberofchildren [startup=N] [idle=N] + The maximum number of authenticator processes to spawn (default 5). If you start too few Squid will have to wait for them to process a backlog of credential verifications, slowing it down. When credential verifications are done via a (slow) network you are likely to need lots of authenticator processes. - auth_param ntlm children 5 + The startup= and idle= options permit some skew in the exact amount + run. A minimum of startup=N will begin during startup and reconfigure + and Squid will start more in groups of up to idle=N in an attempt to meet + traffic needs and to keep idle=N free above those traffic needs up to + the maximum. + + auth_param ntlm children 20 startup=0 idle=1 "keep_alive" on|off If you experience problems with PUT/POST requests when using the @@ -289,14 +309,21 @@ auth_param negotiate program @DEFAULT_PREFIX@/bin/ntlm_auth --helper-protocol=gss-spnego - "children" numberofchildren - The number of authenticator processes to spawn (no default). + "children" numberofchildren [startup=N] [idle=N] + The maximum number of authenticator processes to spawn (default 5). If you start too few Squid will have to wait for them to process a backlog of credential verifications, slowing it down. When crendential verifications are done via a (slow) network you are likely to need lots of authenticator processes. - auth_param negotiate children 5 + + The startup= and idle= options permit some skew in the exact amount + run. A minimum of startup=N will begin during startup and reconfigure + and Squid will start more in groups of up to idle=N in an attempt to meet + traffic needs and to keep idle=N free above those traffic needs up to + the maximum. + + auth_param negotiate children 20 startup=0 idle=1 "keep_alive" on|off If you experience problems with PUT/POST requests when using the @@ -312,22 +339,22 @@ #Recommended minimum configuration per scheme: #auth_param negotiate program -#auth_param negotiate children 5 +#auth_param negotiate children 20 startup=0 idle=1 #auth_param negotiate keep_alive on # #auth_param ntlm program -#auth_param ntlm children 5 +#auth_param ntlm children 20 startup=0 idle=1 #auth_param ntlm keep_alive on # #auth_param digest program -#auth_param digest children 5 +#auth_param digest children 20 startup=0 idle=1 #auth_param digest realm Squid proxy-caching web server #auth_param digest nonce_garbage_interval 5 minutes #auth_param digest nonce_max_duration 30 minutes #auth_param digest nonce_max_count 50 # #auth_param basic program -#auth_param basic children 5 +#auth_param basic children 5 stratup=5 idle=1 #auth_param basic realm Squid proxy-caching web server #auth_param basic credentialsttl 2 hours DOC_END @@ -390,14 +417,24 @@ negative_ttl=n TTL for cached negative lookups (default same as ttl) - children=n Number of acl helper processes spawn to service - external acl lookups of this type. (default 5) + children-max=n + Maximum number of acl helper processes spawned to service + external acl lookups of this type. (default 20) + children-startup=n + Minimum number of acl helper processes to spawn during + startup and reconfigure to service external acl lookups + of this type. (default 0) + children-idle=n + Number of acl helper processes to keep ahead of traffic + loads. Squid will spawn this many at once whenever load + rises above the capabilities of existing processes. + Up to the value of children-max. (default 1) concurrency=n concurrency level per process. Only used with helpers capable of processing more than one query at a time. - cache=n result cache size, 0 is unbounded (default) + cache=n limit the result cache size, default is unbounded. grace=n Percentage remaining of TTL where a refresh of a cached entry should be initiated without needing to - wait for a new reply. (default 0 for no grace period) + wait for a new reply. (default is for no grace period) protocol=2.5 Compatibility mode for Squid-2.5 external acl helpers ipv4 / ipv6 IP-mode used to communicate to this helper. For compatability with older configurations and helpers @@ -3158,21 +3195,36 @@ DOC_END NAME: url_rewrite_children redirect_children -TYPE: int -DEFAULT: 5 +TYPE: HelperChildConfig +DEFAULT: 20 startup=0 idle=1 concurrency=0 LOC: Config.redirectChildren DOC_START - The number of redirector processes to spawn. If you start - too few Squid will have to wait for them to process a backlog of - URLs, slowing it down. If you start too many they will use RAM - and other system resources. -DOC_END - -NAME: url_rewrite_concurrency redirect_concurrency -TYPE: int -DEFAULT: 0 -LOC: Config.redirectConcurrency -DOC_START + The maximum number of redirector processes to spawn. If you limit + it too few Squid will have to wait for them to process a backlog of + URLs, slowing it down. If you allow too many they will use RAM + and other system resources noticably. + + The startup= and idle= options allow some measure of skew in your + tuning. + + startup= + + Sets a minimum of how many processes are to be spawned when Squid + starts or reconfigures. When set to zero the first request will + cause spawning of the first child process to handle it. + + Starting too few will cause an initial slowdown in traffic as Squid + attempts to simultaneously spawn enough processes to cope. + + idle= + + Sets a minimum of how many processes Squid is to try and keep available + at all times. When traffic begins to rise above what the existing + processes can handle this many more will be spawned up to the maximum + configured. A minimum setting of 1 is required. + + concurrency= + The number of requests each redirector helper can handle in parallel. Defaults to 0 which indicates the redirector is a old-style single threaded redirector. @@ -6133,17 +6185,35 @@ DOC_END NAME: dns_children -TYPE: int +TYPE: HelperChildConfig IFDEF: USE_DNSSERVERS -DEFAULT: 5 +DEFAULT: 32 startup=1 idle=1 LOC: Config.dnsChildren DOC_START - The number of processes spawn to service DNS name lookups. - For heavily loaded caches on large servers, you should - probably increase this value to at least 10. The maximum - is 32. The default is 5. - - You must have at least one dnsserver process. + The maximum number of processes spawn to service DNS name lookups. + If you limit it too few Squid will have to wait for them to process + a backlog of requests, slowing it down. If you allow too many they + will use RAM and other system resources noticably. + The maximum this may be safely set to is 32. + + The startup= and idle= options allow some measure of skew in your + tuning. + + startup= + + Sets a minimum of how many processes are to be spawned when Squid + starts or reconfigures. When set to zero the first request will + cause spawning of the first child process to handle it. + + Starting too few will cause an initial slowdown in traffic as Squid + attempts to simultaneously spawn enough processes to cope. + + idle= + + Sets a minimum of how many processes Squid is to try and keep available + at all times. When traffic begins to rise above what the existing + processes can handle this many more will be spawned up to the maximum + configured. A minimum setting of 1 is required. DOC_END NAME: dns_retransmit_interval @@ -6154,7 +6224,6 @@ DOC_START Initial retransmit interval for DNS queries. The interval is doubled each time all configured DNS servers have been tried. - DOC_END NAME: dns_timeout === modified file 'src/dns.cc' --- src/dns.cc 2009-01-21 03:47:47 +0000 +++ src/dns.cc 2009-11-29 05:08:07 +0000 @@ -75,7 +75,7 @@ if (dnsservers == NULL) dnsservers = helperCreate("dnsserver"); - dnsservers->n_to_start = Config.dnsChildren; + dnsservers->childs = Config.dnsChildren; dnsservers->ipc_type = IPC_STREAM; @@ -119,7 +119,11 @@ static time_t first_warn = 0; snprintf(buf, 256, "%s\n", lookup); - if (dnsservers->stats.queue_size >= dnsservers->n_running * 2) { + if (dnsservers->stats.queue_size >= dnsservers->childs.n_active && dnsservers->childs.needNew() > 0) { + helperOpenServers(dnsservers); + } + + if (dnsservers->stats.queue_size >= dnsservers->childs.n_running * 2) { if (first_warn == 0) first_warn = squid_curtime; === modified file 'src/external_acl.cc' --- src/external_acl.cc 2009-11-09 11:25:11 +0000 +++ src/external_acl.cc 2009-11-26 10:22:44 +0000 @@ -107,9 +107,7 @@ wordlist *cmdline; - int children; - - int concurrency; + HelperChildConfig children; helper *theHelper; @@ -295,11 +293,12 @@ /* set defaults */ a->ttl = DEFAULT_EXTERNAL_ACL_TTL; a->negative_ttl = -1; - a->children = DEFAULT_EXTERNAL_ACL_CHILDREN; + a->children.n_max = DEFAULT_EXTERNAL_ACL_CHILDREN; + a->children.n_startup = a->children.n_max; + a->children.n_idle = 99999999; // big to detect if the user sets their own. a->local_addr.SetLocalhost(); a->quote = external_acl::QUOTE_METHOD_URL; - token = strtok(NULL, w_space); if (!token) @@ -316,9 +315,16 @@ } else if (strncmp(token, "negative_ttl=", 13) == 0) { a->negative_ttl = atoi(token + 13); } else if (strncmp(token, "children=", 9) == 0) { - a->children = atoi(token + 9); + a->children.n_max = atoi(token + 9); + debugs(0, 0, "WARNING: external_acl_type option children=N has been deprecated in favor of children-max=N and children-startup=N"); + } else if (strncmp(token, "children-max=", 13) == 0) { + a->children.n_max = atoi(token + 13); + } else if (strncmp(token, "children-startup=", 17) == 0) { + a->children.n_startup = atoi(token + 17); + } else if (strncmp(token, "children-idle=", 14) == 0) { + a->children.n_idle = atoi(token + 14); } else if (strncmp(token, "concurrency=", 12) == 0) { - a->concurrency = atoi(token + 12); + a->children.concurrency = atoi(token + 12); } else if (strncmp(token, "cache=", 6) == 0) { a->cache_size = atoi(token + 6); } else if (strncmp(token, "grace=", 6) == 0) { @@ -351,6 +357,11 @@ token = strtok(NULL, w_space); } + /* our default idle is huge on purpose, make it sane when we know whether the user has set their own. */ + if (a->children.n_idle > a->children.n_max - a->children.n_startup) + a->children.n_idle = max(1, (int)(a->children.n_max - a->children.n_startup)); + + if (a->negative_ttl == -1) a->negative_ttl = a->ttl; @@ -483,11 +494,17 @@ if (node->grace) storeAppendPrintf(sentry, " grace=%d", node->grace); - if (node->children != DEFAULT_EXTERNAL_ACL_CHILDREN) - storeAppendPrintf(sentry, " children=%d", node->children); - - if (node->concurrency) - storeAppendPrintf(sentry, " concurrency=%d", node->concurrency); + if (node->children.n_max != DEFAULT_EXTERNAL_ACL_CHILDREN) + storeAppendPrintf(sentry, " children-max=%d", node->children.n_max); + + if (node->children.n_startup != 1) + storeAppendPrintf(sentry, " children-startup=%d", node->children.n_startup); + + if (node->children.n_idle != (node->children.n_max + node->children.n_startup) ) + storeAppendPrintf(sentry, " children-idle=%d", node->children.n_idle); + + if (node->children.concurrency) + storeAppendPrintf(sentry, " concurrency=%d", node->children.concurrency); if (node->cache) storeAppendPrintf(sentry, " cache=%d", node->cache_size); @@ -744,7 +761,7 @@ debugs(82, 2, "aclMatchExternal: \"" << key << "\": entry=@" << entry << ", age=" << (entry ? (long int) squid_curtime - entry->date : 0)); - if (acl->def->theHelper->stats.queue_size <= acl->def->theHelper->n_running) { + if (acl->def->theHelper->stats.queue_size <= acl->def->theHelper->childs.n_active) { debugs(82, 2, "aclMatchExternal: \"" << key << "\": queueing a call."); ch->changeState (ExternalACLLookup::Instance()); @@ -1331,7 +1348,7 @@ } else { /* Check for queue overload */ - if (def->theHelper->stats.queue_size >= def->theHelper->n_running) { + if (def->theHelper->stats.queue_size >= def->theHelper->childs.n_running) { debugs(82, 1, "externalAclLookup: '" << def->name << "' queue overload (ch=" << ch << ")"); cbdataFree(state); callback(callback_data, entry); @@ -1411,9 +1428,7 @@ p->theHelper->cmdline = p->cmdline; - p->theHelper->n_to_start = p->children; - - p->theHelper->concurrency = p->concurrency; + p->theHelper->childs = p->children; p->theHelper->ipc_type = IPC_TCP_SOCKET; === modified file 'src/forward.cc' --- src/forward.cc 2009-11-25 17:10:52 +0000 +++ src/forward.cc 2009-12-03 07:14:52 +0000 @@ -1000,7 +1000,8 @@ break; if (o->cmsg_level == SOL_IP && o->cmsg_type == IP_TOS) { - clientFde->upstreamTOS = (unsigned char)(*(int*)CMSG_DATA(o)); + int *tmp = (int*)CMSG_DATA(o); + clientFde->upstreamTOS = (unsigned char)*tmp; break; } pbuf += CMSG_LEN(o->cmsg_len); === modified file 'src/globals.h' --- src/globals.h 2009-08-23 09:30:49 +0000 +++ src/globals.h 2009-11-25 20:09:05 +0000 @@ -118,6 +118,7 @@ //MOVED:snmp_core.cc extern IpAddress theOutSNMPAddr; extern struct timeval squid_start; + extern int starting_up; /* 1 */ extern int shutting_down; /* 0 */ extern int reconfiguring; /* 0 */ extern unsigned long store_swap_size; /* 0 */ === modified file 'src/helper.cc' --- src/helper.cc 2009-12-02 22:39:11 +0000 +++ src/helper.cc 2009-12-04 00:33:12 +0000 @@ -101,10 +101,10 @@ else shortname = xstrdup(progname); - /* dont ever start more than hlp->n_to_start processes. */ - int need_new = hlp->n_to_start - hlp->n_active; + /* figure out how many new child are actually needed. */ + int need_new = hlp->childs.needNew(); - debugs(84, 1, "helperOpenServers: Starting " << need_new << "/" << hlp->n_to_start << " '" << shortname << "' processes"); + debugs(84, 1, "helperOpenServers: Starting " << need_new << "/" << hlp->childs.n_max << " '" << shortname << "' processes"); if (need_new < 1) { debugs(84, 1, "helperOpenServers: No '" << shortname << "' processes needed."); @@ -140,8 +140,8 @@ continue; } - hlp->n_running++; - hlp->n_active++; + hlp->childs.n_running++; + hlp->childs.n_active++; CBDATA_INIT_TYPE(helper_server); srv = cbdataAlloc(helper_server); srv->hIpc = hIpc; @@ -153,7 +153,7 @@ srv->rbuf = (char *)memAllocBuf(BUF_8KB, &srv->rbuf_sz); srv->wqueue = new MemBuf; srv->roffset = 0; - srv->requests = (helper_request **)xcalloc(hlp->concurrency ? hlp->concurrency : 1, sizeof(*srv->requests)); + srv->requests = (helper_request **)xcalloc(hlp->childs.concurrency ? hlp->childs.concurrency : 1, sizeof(*srv->requests)); srv->parent = cbdataReference(hlp); dlinkAddTail(srv, &srv->link, &hlp->servers); @@ -207,11 +207,10 @@ else shortname = xstrdup(progname); - /* dont ever start more than hlp->n_to_start processes. */ - /* n_active are the helpers which have not been shut down. */ - int need_new = hlp->n_to_start - hlp->n_active; + /* figure out haw mant new helpers are needed. */ + int need_new = hlp->childs.needNew(); - debugs(84, 1, "helperOpenServers: Starting " << need_new << "/" << hlp->n_to_start << " '" << shortname << "' processes"); + debugs(84, 1, "helperOpenServers: Starting " << need_new << "/" << hlp->childs.n_max << " '" << shortname << "' processes"); if (need_new < 1) { debugs(84, 1, "helperStatefulOpenServers: No '" << shortname << "' processes needed."); @@ -249,8 +248,8 @@ continue; } - hlp->n_running++; - hlp->n_active++; + hlp->childs.n_running++; + hlp->childs.n_active++; CBDATA_INIT_TYPE(helper_stateful_server); helper_stateful_server *srv = cbdataAlloc(helper_stateful_server); srv->hIpc = hIpc; @@ -404,7 +403,7 @@ storeAppendPrintf(sentry, "program: %s\n", hlp->cmdline->key); storeAppendPrintf(sentry, "number active: %d of %d (%d shutting down)\n", - hlp->n_active, hlp->n_to_start, (hlp->n_running - hlp->n_active) ); + hlp->childs.n_active, hlp->childs.n_max, (hlp->childs.n_running - hlp->childs.n_active) ); storeAppendPrintf(sentry, "requests sent: %d\n", hlp->stats.requests); storeAppendPrintf(sentry, "replies received: %d\n", @@ -457,7 +456,7 @@ storeAppendPrintf(sentry, "program: %s\n", hlp->cmdline->key); storeAppendPrintf(sentry, "number active: %d of %d (%d shutting down)\n", - hlp->n_active, hlp->n_to_start, (hlp->n_running - hlp->n_active) ); + hlp->childs.n_active, hlp->childs.n_max, (hlp->childs.n_running - hlp->childs.n_active) ); storeAppendPrintf(sentry, "requests sent: %d\n", hlp->stats.requests); storeAppendPrintf(sentry, "replies received: %d\n", @@ -524,8 +523,8 @@ continue; } - hlp->n_active--; - assert(hlp->n_active >= 0); + assert(hlp->childs.n_active > 0); + hlp->childs.n_active--; srv->flags.shutdown = 1; /* request it to shut itself down */ if (srv->flags.closing) { @@ -592,8 +591,8 @@ continue; } - hlp->n_active--; - assert(hlp->n_active >= 0); + assert(hlp->childs.n_active > 0); + hlp->childs.n_active--; srv->flags.shutdown = 1; /* request it to shut itself down */ if (srv->flags.busy) { @@ -709,7 +708,7 @@ helper_server *srv = (helper_server *)data; helper *hlp = srv->parent; helper_request *r; - int i, concurrency = hlp->concurrency; + int i, concurrency = hlp->childs.concurrency; if (!concurrency) concurrency = 1; @@ -748,24 +747,21 @@ dlinkDelete(&srv->link, &hlp->servers); - hlp->n_running--; - - assert(hlp->n_running >= 0); + assert(hlp->childs.n_running > 0); + hlp->childs.n_running--; if (!srv->flags.shutdown) { - hlp->n_active--; - assert(hlp->n_active >= 0); - debugs(84, 0, "WARNING: " << hlp->id_name << " #" << srv->index + 1 << - " (FD " << fd << ") exited"); - - if (hlp->n_active < hlp->n_to_start / 2) { - debugs(80, 0, "Too few " << hlp->id_name << " processes are running"); - - if (hlp->last_restart > squid_curtime - 30) + assert(hlp->childs.n_active > 0); + hlp->childs.n_active--; + debugs(84, DBG_CRITICAL, "WARNING: " << hlp->id_name << " #" << srv->index + 1 << " (FD " << fd << ") exited"); + + if (hlp->childs.needNew() > 0) { + debugs(80, 1, "Too few " << hlp->id_name << " processes are running (need " << hlp->childs.needNew() << "/" << hlp->childs.n_max << ")"); + + if (hlp->childs.n_active < hlp->childs.n_startup && hlp->last_restart > squid_curtime - 30) fatalf("The %s helpers are crashing too rapidly, need help!\n", hlp->id_name); - debugs(80, 0, "Starting new helpers"); - + debugs(80, 1, "Starting new helpers"); helperOpenServers(hlp); } } @@ -810,23 +806,21 @@ dlinkDelete(&srv->link, &hlp->servers); - hlp->n_running--; - - assert(hlp->n_running >= 0); + assert(hlp->childs.n_running > 0); + hlp->childs.n_running--; if (!srv->flags.shutdown) { - hlp->n_active--; - assert( hlp->n_active >= 0); + assert( hlp->childs.n_active > 0); + hlp->childs.n_active--; debugs(84, 0, "WARNING: " << hlp->id_name << " #" << srv->index + 1 << " (FD " << fd << ") exited"); - if (hlp->n_active <= hlp->n_to_start / 2) { - debugs(80, 0, "Too few " << hlp->id_name << " processes are running"); + if (hlp->childs.needNew() > 0) { + debugs(80, 1, "Too few " << hlp->id_name << " processes are running (need " << hlp->childs.needNew() << "/" << hlp->childs.n_max << ")"); - if (hlp->last_restart > squid_curtime - 30) + if (hlp->childs.n_active < hlp->childs.n_startup && hlp->last_restart > squid_curtime - 30) fatalf("The %s helpers are crashing too rapidly, need help!\n", hlp->id_name); - debugs(80, 0, "Starting new helpers"); - + debugs(80, 1, "Starting new helpers"); helperStatefulOpenServers(hlp); } } @@ -893,7 +887,7 @@ *t++ = '\0'; - if (hlp->concurrency) { + if (hlp->childs.concurrency) { i = strtol(msg, &msg, 10); while (*msg && xisspace(*msg)) @@ -1040,7 +1034,14 @@ dlinkAddTail(r, link, &hlp->queue); hlp->stats.queue_size++; - if (hlp->stats.queue_size < hlp->n_running) + /* do this first so idle=N has a chance to grow the child pool before it hits critical. */ + if (hlp->childs.needNew() > 0) { + debugs(84, 0, "Starting new " << hlp->id_name << " helpers..."); + helperOpenServers(hlp); + return; + } + + if (hlp->stats.queue_size < hlp->childs.n_running) return; if (squid_curtime - hlp->last_queue_warn < 600) @@ -1053,13 +1054,10 @@ debugs(84, 0, "WARNING: All " << hlp->id_name << " processes are busy."); debugs(84, 0, "WARNING: " << hlp->stats.queue_size << " pending requests queued"); - - - if (hlp->stats.queue_size > hlp->n_running * 2) + debugs(84, 0, "WARNING: Consider increasing the number of " << hlp->id_name << " processes in your config file."); + + if (hlp->stats.queue_size > hlp->childs.n_running * 2) fatalf("Too many queued %s requests", hlp->id_name); - - debugs(84, 1, "Consider increasing the number of " << hlp->id_name << " processes in your config file."); - } static void @@ -1069,10 +1067,17 @@ dlinkAddTail(r, link, &hlp->queue); hlp->stats.queue_size++; - if (hlp->stats.queue_size < hlp->n_running) - return; - - if (hlp->stats.queue_size > hlp->n_running * 2) + /* do this first so idle=N has a chance to grow the child pool before it hits critical. */ + if (hlp->childs.needNew() > 0) { + debugs(84, 0, "Starting new " << hlp->id_name << " helpers..."); + helperStatefulOpenServers(hlp); + return; + } + + if (hlp->stats.queue_size < hlp->childs.n_running) + return; + + if (hlp->stats.queue_size > hlp->childs.n_running * 2) fatalf("Too many queued %s requests", hlp->id_name); if (squid_curtime - hlp->last_queue_warn < 600) @@ -1084,10 +1089,8 @@ hlp->last_queue_warn = squid_curtime; debugs(84, 0, "WARNING: All " << hlp->id_name << " processes are busy."); - debugs(84, 0, "WARNING: " << hlp->stats.queue_size << " pending requests queued"); - debugs(84, 1, "Consider increasing the number of " << hlp->id_name << " processes in your config file."); - + debugs(84, 0, "WARNING: Consider increasing the number of " << hlp->id_name << " processes in your config file."); } static helper_request * @@ -1129,7 +1132,7 @@ helper_server *srv; helper_server *selected = NULL; - if (hlp->n_running == 0) + if (hlp->childs.n_running == 0) return NULL; /* Find "least" loaded helper (approx) */ @@ -1157,7 +1160,7 @@ if (!selected) return NULL; - if (selected->stats.pending >= (hlp->concurrency ? hlp->concurrency : 1)) + if (selected->stats.pending >= (hlp->childs.concurrency ? hlp->childs.concurrency : 1)) return NULL; return selected; @@ -1168,9 +1171,9 @@ { dlink_node *n; helper_stateful_server *srv = NULL; - debugs(84, 5, "StatefulGetFirstAvailable: Running servers " << hlp->n_running); + debugs(84, 5, "StatefulGetFirstAvailable: Running servers " << hlp->childs.n_running); - if (hlp->n_running == 0) + if (hlp->childs.n_running == 0) return NULL; for (n = hlp->servers.head; n != NULL; n = n->next) { @@ -1238,7 +1241,7 @@ return; } - for (slot = 0; slot < (hlp->concurrency ? hlp->concurrency : 1); slot++) { + for (slot = 0; slot < (hlp->childs.concurrency ? hlp->childs.concurrency : 1); slot++) { if (!srv->requests[slot]) { ptr = &srv->requests[slot]; break; @@ -1253,7 +1256,7 @@ if (srv->wqueue->isNull()) srv->wqueue->init(); - if (hlp->concurrency) + if (hlp->childs.concurrency) srv->wqueue->Printf("%d %s", slot, r->buf); else srv->wqueue->append(r->buf, strlen(r->buf)); === modified file 'src/helper.h' --- src/helper.h 2009-08-04 14:39:10 +0000 +++ src/helper.h 2009-11-26 02:01:41 +0000 @@ -36,6 +36,7 @@ #include "squid.h" #include "cbdata.h" #include "ip/IpAddress.h" +#include "HelperChildConfig.h" class helper_request; @@ -58,12 +59,9 @@ dlink_list servers; dlink_list queue; const char *id_name; - int n_to_start; ///< Configuration setting of how many helper children should be running - int n_running; ///< Total helper children objects currently existing - int n_active; ///< Count of helper children active (not shutting down) + HelperChildConfig childs; ///< Configuration settings for number running. int ipc_type; IpAddress addr; - unsigned int concurrency; time_t last_queue_warn; time_t last_restart; @@ -80,9 +78,7 @@ dlink_list servers; dlink_list queue; const char *id_name; - int n_to_start; ///< Configuration setting of how many helper children should be running - int n_running; ///< Total helper children objects currently existing - int n_active; ///< Count of helper children active (not shutting down) + HelperChildConfig childs; ///< Configuration settings for number running. int ipc_type; IpAddress addr; MemAllocator *datapool; === modified file 'src/icmp/Makefile.am' --- src/icmp/Makefile.am 2009-11-12 01:12:50 +0000 +++ src/icmp/Makefile.am 2009-11-30 11:07:27 +0000 @@ -55,7 +55,8 @@ $(top_builddir)/src/globals.cc \ $(top_srcdir)/src/time.cc \ $(top_srcdir)/src/SquidConfig.cc \ - $(top_srcdir)/src/SquidNew.cc + $(top_srcdir)/src/SquidNew.cc \ + $(top_srcdir)/src/tests/stub_HelperChildConfig.cc pinger_LDFLAGS = $(LIBADD_DL) pinger_LDADD=\ === modified file 'src/main.cc' --- src/main.cc 2009-11-18 12:37:56 +0000 +++ src/main.cc 2009-11-25 20:13:50 +0000 @@ -1370,6 +1370,9 @@ mainLoop.setTimeService(&time_engine); + /* at this point we are finished the synchronous startup. */ + starting_up = 0; + mainLoop.run(); if (mainLoop.errcount == 10) === modified file 'src/redirect.cc' --- src/redirect.cc 2009-11-09 11:25:11 +0000 +++ src/redirect.cc 2009-11-26 10:40:40 +0000 @@ -198,9 +198,7 @@ redirectors->cmdline = Config.Program.redirect; - redirectors->n_to_start = Config.redirectChildren; - - redirectors->concurrency = Config.redirectConcurrency; + redirectors->childs = Config.redirectChildren; redirectors->ipc_type = IPC_STREAM; === modified file 'src/structs.h' --- src/structs.h 2009-11-22 20:37:27 +0000 +++ src/structs.h 2009-11-26 02:17:51 +0000 @@ -128,6 +128,8 @@ #include "ip/QosConfig.h" #endif +#include "HelperChildConfig.h" + /* forward decl for SquidConfig, see RemovalPolicy.h */ class RemovalPolicySettings; @@ -301,11 +303,10 @@ } Program; #if USE_DNSSERVERS - int dnsChildren; + HelperChildConfig dnsChildren; #endif - int redirectChildren; - int redirectConcurrency; + HelperChildConfig redirectChildren; time_t authenticateGCInterval; time_t authenticateTTL; time_t authenticateIpTTL; === added file 'src/tests/stub_HelperChildConfig.cc' --- src/tests/stub_HelperChildConfig.cc 1970-01-01 00:00:00 +0000 +++ src/tests/stub_HelperChildConfig.cc 2009-11-30 11:07:00 +0000 @@ -0,0 +1,52 @@ +#include "config.h" +#include "HelperChildConfig.h" +#include "globals.h" + +#include + +HelperChildConfig::HelperChildConfig() : + n_max(0), + n_startup(1), + n_idle(1), + concurrency(1), + n_running(0), + n_active(0) +{} + +HelperChildConfig::HelperChildConfig(const unsigned int m, const unsigned int s, const unsigned int i, const unsigned int cc) : + n_max(m), + n_startup(s), + n_idle(i), + concurrency(cc), + n_running(0), + n_active(0) +{} + +HelperChildConfig::~HelperChildConfig() +{} + +HelperChildConfig & +HelperChildConfig::operator =(const HelperChildConfig &rhs) +{ + memcpy(this, &rhs, sizeof(HelperChildConfig)); + return *this; +} + +const int +HelperChildConfig::needNew() const { + /* during the startup and reconfigure use our special amount... */ + if (starting_up || reconfiguring) return n_startup; + + /* keep a minimum of n_idle helpers free... */ + if ( (n_active + n_idle) < n_max) return n_idle; + + /* dont ever start more than n_max processes. */ + return (n_max - n_active); +} + +void +HelperChildConfig::parseConfig() +{ + fprintf(stderr, "HelperChildConfig::parseConfig not implemented."); + exit(1); +} === modified file 'src/tests/stub_tools.cc' --- src/tests/stub_tools.cc 2009-01-21 03:47:47 +0000 +++ src/tests/stub_tools.cc 2009-11-30 11:06:23 +0000 @@ -31,7 +31,7 @@ * */ -#include "squid.h" +#include "config.h" int percent(int a, int b) @@ -43,5 +43,6 @@ void death(int sig) { - fatal ("Not implemented"); + fprintf(stderr, "Not implemented"); + exit(1); } === modified file 'test-suite/testheaders.sh' --- test-suite/testheaders.sh 2009-11-19 23:32:21 +0000 +++ test-suite/testheaders.sh 2009-11-29 02:29:48 +0000 @@ -16,9 +16,11 @@ dir="${2}" fi +PWD=`pwd` for f in `cd ${dir} && ls -1 *.h 2>/dev/null`; do echo -n "Testing ${dir}/${f} ..." hdr=`echo "${f}" | sed s/.h//` + echo " ${dir}/${f} -nt ${PWD}/testHeaderDeps_${hdr}.o" if [ ! -e ./testHeaderDeps_${hdr}.o -o ${dir}/${f} -nt ./testHeaderDeps_${hdr}.o ]; then ( echo "/* This file is AUTOMATICALLY GENERATED. DO NOT ALTER IT */" echo "#include \"${dir}/${f}\" " @@ -29,6 +31,8 @@ # DEBUG: echo "TRY: ${cc} -o testHeaderDeps.o ./testHeaderDeps_${hdr}.cc" ${cc} -c -o testHeaderDeps_${hdr}.o ./testHeaderDeps_${hdr}.cc rm ./testHeaderDeps_${hdr}.cc + else + echo -n "(no change) ..." fi if [ ! -f testHeaderDeps_${hdr}.o ]; then rm testHeaders