=== modified file 'src/ClientRequestContext.h' --- src/ClientRequestContext.h 2012-11-30 11:08:47 +0000 +++ src/ClientRequestContext.h 2012-12-05 12:04:18 +0000 @@ -18,71 +18,76 @@ class ClientRequestContext : public RefCountable { public: void *operator new(size_t); void operator delete(void *); ClientRequestContext(ClientHttpRequest *); ~ClientRequestContext(); bool httpStateIsValid(); void hostHeaderVerify(); void hostHeaderIpVerify(const ipcache_addrs* ia, const DnsLookupDetails &dns); void hostHeaderVerifyFailed(const char *A, const char *B); void clientAccessCheck(); void clientAccessCheck2(); void clientAccessCheckDone(const allow_t &answer); void clientRedirectStart(); void clientRedirectDone(const HelperReply &reply); + void clientStoreIdStart(); + void clientStoreIdDone(const HelperReply &reply); void checkNoCache(); void checkNoCacheDone(const allow_t &answer); #if USE_ADAPTATION void adaptationAccessCheck(); #endif #if USE_SSL /** * Initiates and start the acl checklist to check if the a CONNECT * request must be bumped. \retval true if the acl check scheduled, false if no ssl-bump required */ bool sslBumpAccessCheck(); /// The callback function for ssl-bump access check list void sslBumpAccessCheckDone(const allow_t &answer); #endif ClientHttpRequest *http; ACLChecklist *acl_checklist; /* need ptr back so we can unreg if needed */ int redirect_state; + int store_id_state; /** * URL-rewrite/redirect helper may return BH for internal errors. * We attempt to recover by trying the lookup again, but limit the * number of retries to prevent lag and lockups. * This tracks the number of previous failures for the current context. */ uint8_t redirect_fail_count; + uint8_t store_id_fail_count; bool host_header_verify_done; bool http_access_done; bool adapted_http_access_done; #if USE_ADAPTATION bool adaptation_acl_check_done; #endif bool redirect_done; + bool store_id_done; bool no_cache_done; bool interpreted_req_hdrs; bool tosToClientDone; bool nfmarkToClientDone; #if USE_SSL bool sslBumpCheckDone; #endif ErrorState *error; ///< saved error page for centralized/delayed processing bool readNextRequest; ///< whether Squid should read after error handling private: CBDATA_CLASS(ClientRequestContext); }; #endif /* SQUID_CLIENTREQUESTCONTEXT_H */ === modified file 'src/HelperReply.cc' --- src/HelperReply.cc 2012-11-28 01:13:21 +0000 +++ src/HelperReply.cc 2012-12-05 12:08:21 +0000 @@ -1,65 +1,71 @@ /* * DEBUG: section 84 Helper process maintenance * AUTHOR: Amos Jeffries */ #include "squid.h" #include "ConfigParser.h" #include "HelperReply.h" #include "helper.h" #include "rfc1738.h" #include "SquidString.h" +#include "Debug.h" HelperReply::HelperReply(char *buf, size_t len) : result(HelperReply::Unknown), whichServer(NULL) { parse(buf,len); } void HelperReply::parse(char *buf, size_t len) { + debugs(84, 3, HERE << "Parsing helper buffer"); // check we have something to parse if (!buf || len < 1) { // for now ensure that legacy handlers are not presented with NULL strings. + debugs(84, 3, HERE << "Buff lenth is smaller then 1 or no at all "); other_.init(1,1); other_.terminate(); return; } char *p = buf; - // optimization: do not consider parsing result code if the response is short. // URL-rewriter may return relative URLs or empty response for a large portion // of its replies. if (len >= 2) { + debugs(84, 3, HERE <<"Buff lenth is larger then 2"); // some helper formats (digest auth, URL-rewriter) just send a data string // we must also check for the ' ' character after the response token (if anything) if (!strncmp(p,"OK",2) && (len == 2 || p[2] == ' ')) { - result = HelperReply::Okay; + debugs(84, 3, HERE <<"helper Result = OK"); + result = HelperReply::Okay; p+=2; } else if (!strncmp(p,"ERR",3) && (len == 3 || p[3] == ' ')) { - result = HelperReply::Error; + debugs(84, 3, HERE <<"helper Result = ERR"); + result = HelperReply::Error; p+=3; } else if (!strncmp(p,"BH",2) && (len == 2 || p[2] == ' ')) { - result = HelperReply::BrokenHelper; + debugs(84, 3, HERE <<"helper Result = BrokenHelper"); + result = HelperReply::BrokenHelper; p+=2; } else if (!strncmp(p,"TT ",3)) { // NTLM challenge token result = HelperReply::TT; p+=3; // followed by an auth token char *w1 = strwordtok(NULL, &p); if (w1 != NULL) { MemBuf authToken; authToken.init(); authToken.append(w1, strlen(w1)); notes.add("token",authToken.content()); } else { // token field is mandatory on this response code result = HelperReply::BrokenHelper; notes.add("message","Missing 'token' data"); } } else if (!strncmp(p,"AF ",3)) { // NTLM/Negotate OK response === modified file 'src/HttpRequest.cc' --- src/HttpRequest.cc 2012-12-02 23:10:23 +0000 +++ src/HttpRequest.cc 2012-12-05 12:13:01 +0000 @@ -671,20 +671,35 @@ bool HttpRequest::canHandle1xx() const { // old clients do not support 1xx unless they sent Expect: 100-continue // (we reject all other HDR_EXPECT values so just check for HDR_EXPECT) if (http_ver <= HttpVersion(1,0) && !header.has(HDR_EXPECT)) return false; // others must support 1xx control messages return true; } ConnStateData * HttpRequest::pinnedConnection() { if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned) return clientConnectionManager.get(); return NULL; } +/* + * The method return the current storeID of the request. + * returns canonicalUrl of the reuest in a case StoreID not present. + */ +const char *HttpRequest::storeId() +{ + String can = urlCanonical(static_cast(this)); + debugs(73, 3, HERE << "storeId: is being used"); + if (store_id.size() != 0){ + debugs(73, 3,HERE << "storeId sent back store_id:" << store_id); + return store_id.termedBuf(); + } + debugs(73, 3,HERE << "storeId sent back canonicalUrl:" << can); + return can.termedBuf(); +} === modified file 'src/HttpRequest.h' --- src/HttpRequest.h 2012-11-30 22:57:54 +0000 +++ src/HttpRequest.h 2012-12-05 12:17:08 +0000 @@ -148,40 +148,42 @@ int host_is_numeric; #if USE_ADAPTATION mutable Adaptation::History::Pointer adaptHistory_; ///< per-HTTP transaction info #endif #if ICAP_CLIENT mutable Adaptation::Icap::History::Pointer icapHistory_; ///< per-HTTP transaction info #endif public: Ip::Address host_addr; #if USE_AUTH Auth::UserRequest::Pointer auth_user_request; #endif unsigned short port; String urlpath; char *canonical; + String store_id; /* Storage of StoreID for the specific cases that the request not avalile */ + RequestFlags flags; HttpHdrRange *range; time_t ims; int imslen; Ip::Address client_addr; #if FOLLOW_X_FORWARDED_FOR Ip::Address indirect_client_addr; #endif /* FOLLOW_X_FORWARDED_FOR */ Ip::Address my_addr; HierarchyLogEntry hier; int dnsWait; ///< sum of DNS lookup delays in milliseconds, for %dt @@ -224,40 +226,42 @@ int parseHeader(const char *parse_start, int len); virtual bool expectingBody(const HttpRequestMethod& unused, int64_t&) const; bool bodyNibbled() const; // the request has a [partially] consumed body int prefixLen(); void swapOut(StoreEntry * e); void pack(Packer * p); static void httpRequestPack(void *obj, Packer *p); static HttpRequest * CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method); static HttpRequest * CreateFromUrl(char * url); ConnStateData *pinnedConnection(); + const char *storeId(); /* Returns the current StoreID for the reuest*/ + /** * The client connection manager, if known; * Used for any response actions needed directly to the client. * ie 1xx forwarding or connection pinning state changes */ CbcPointer clientConnectionManager; int64_t getRangeOffsetLimit(); /* the result of this function gets cached in rangeOffsetLimit */ private: const char *packableURI(bool full_uri) const; mutable int64_t rangeOffsetLimit; /* caches the result of getRangeOffsetLimit */ protected: virtual void packFirstLineInto(Packer * p, bool full_uri) const; virtual bool sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, http_status *error); virtual void hdrCacheInit(); === modified file 'src/SquidConfig.h' --- src/SquidConfig.h 2012-10-29 04:59:58 +0000 +++ src/SquidConfig.h 2012-12-04 18:10:46 +0000 @@ -186,57 +186,59 @@ char *swap; CustomLog *accesslogs; #if ICAP_CLIENT CustomLog *icaplogs; #endif int rotateNumber; } Log; char *adminEmail; char *EmailFrom; char *EmailProgram; char *effectiveUser; char *visible_appname_string; char *effectiveGroup; struct { #if USE_DNSHELPER char *dnsserver; #endif wordlist *redirect; + wordlist *store_id; #if USE_UNLINKD char *unlinkd; #endif char *diskd; #if USE_SSL char *ssl_password; #endif } Program; #if USE_DNSHELPER HelperChildConfig dnsChildren; #endif HelperChildConfig redirectChildren; + HelperChildConfig storeIdChildren; time_t authenticateGCInterval; time_t authenticateTTL; time_t authenticateIpTTL; struct { char *surrogate_id; } Accel; char *appendDomain; size_t appendDomainLen; char *pidFilename; char *netdbFilename; char *mimeTablePathname; char *etcHostsPath; char *visibleHostname; char *uniqueHostname; wordlist *hostnameAliases; char *errHtmlText; struct { char *host; @@ -301,40 +303,41 @@ int buffered_logs; int common_log; int log_mime_hdrs; int log_fqdn; int announce; int mem_pools; int test_reachability; int half_closed_clients; int refresh_all_ims; #if USE_HTTP_VIOLATIONS int reload_into_ims; #endif int offline; int redir_rewrites_host; int prefer_direct; int nonhierarchical_direct; int strip_query_terms; int redirector_bypass; + int store_id_bypass; int ignore_unknown_nameservers; int client_pconns; int server_pconns; int error_pconns; #if USE_CACHE_DIGESTS int digest_generation; #endif int ie_refresh; int vary_ignore_expire; int pipeline_prefetch; int surrogate_is_remote; int request_entities; int detect_broken_server_pconns; int balance_on_multiple_ip; int relaxed_header_parser; int check_hostnames; int allow_underscore; int via; @@ -364,40 +367,41 @@ class ACL *aclList; struct { acl_access *http; acl_access *adapted_http; acl_access *icp; acl_access *miss; acl_access *NeverDirect; acl_access *AlwaysDirect; acl_access *ASlists; acl_access *noCache; acl_access *log; #if SQUID_SNMP acl_access *snmp; #endif #if USE_HTTP_VIOLATIONS acl_access *brokenPosts; #endif acl_access *redirector; + acl_access *store_id; acl_access *reply; AclAddress *outgoing_address; #if USE_HTCP acl_access *htcp; acl_access *htcp_clr; #endif #if USE_SSL acl_access *ssl_bump; #endif #if FOLLOW_X_FORWARDED_FOR acl_access *followXFF; #endif /* FOLLOW_X_FORWARDED_FOR */ #if ICAP_CLIENT acl_access* icap; #endif } accessList; AclDenyInfoList *denyInfoList; === modified file 'src/cache_cf.cc' --- src/cache_cf.cc 2012-11-17 12:58:59 +0000 +++ src/cache_cf.cc 2012-12-04 17:37:42 +0000 @@ -684,40 +684,47 @@ Config.onoff.announce = 0; } if (Config.onoff.httpd_suppress_version_string) visible_appname_string = (char *)appname_string; else visible_appname_string = (char const *)APP_FULLNAME; #if USE_DNSHELPER if (Config.dnsChildren.n_max < 1) fatal("No DNS helpers allocated"); #endif if (Config.Program.redirect) { if (Config.redirectChildren.n_max < 1) { Config.redirectChildren.n_max = 0; wordlistDestroy(&Config.Program.redirect); } } + if (Config.Program.store_id) { + if (Config.storeIdChildren.n_max < 1) { + Config.storeIdChildren.n_max = 0; + wordlistDestroy(&Config.Program.store_id); + } + } + if (Config.appendDomain) if (*Config.appendDomain != '.') fatal("append_domain must begin with a '.'"); if (Config.errHtmlText == NULL) Config.errHtmlText = xstrdup(null_string); #if !HAVE_SETRLIMIT || !defined(RLIMIT_NOFILE) if (Config.max_filedescriptors > 0) { debugs(0, DBG_IMPORTANT, "WARNING: max_filedescriptors disabled. Operating System setrlimit(RLIMIT_NOFILE) is missing."); } #elif USE_SELECT || USE_SELECT_WIN32 if (Config.max_filedescriptors > FD_SETSIZE) { debugs(0, DBG_IMPORTANT, "WARNING: max_filedescriptors limited to " << FD_SETSIZE << " by select() algorithm."); } #endif storeConfigure(); snprintf(ThisCache, sizeof(ThisCache), "%s (%s)", @@ -746,40 +753,43 @@ else Config.appendDomainLen = 0; if (Config.connect_retries > 10) { debugs(0,DBG_CRITICAL, "WARNING: connect_retries cannot be larger than 10. Resetting to 10."); Config.connect_retries = 10; } requirePathnameExists("MIME Config Table", Config.mimeTablePathname); #if USE_DNSHELPER requirePathnameExists("cache_dns_program", Config.Program.dnsserver); #endif #if USE_UNLINKD requirePathnameExists("unlinkd_program", Config.Program.unlinkd); #endif requirePathnameExists("logfile_daemon", Log::TheConfig.logfile_daemon); if (Config.Program.redirect) requirePathnameExists("redirect_program", Config.Program.redirect->key); + if (Config.Program.store_id) + requirePathnameExists("store_id_program", Config.Program.store_id->key); + requirePathnameExists("Icon Directory", Config.icons.directory); if (Config.errorDirectory) requirePathnameExists("Error Directory", Config.errorDirectory); #if USE_HTTP_VIOLATIONS { const RefreshPattern *R; for (R = Config.Refresh; R; R = R->next) { if (!R->flags.override_expire) continue; debugs(22, DBG_IMPORTANT, "WARNING: use of 'override-expire' in 'refresh_pattern' violates HTTP"); break; } for (R = Config.Refresh; R; R = R->next) { === modified file 'src/cf.data.pre' --- src/cf.data.pre 2012-11-11 05:10:59 +0000 +++ src/cf.data.pre 2012-12-05 12:29:45 +0000 @@ -4263,40 +4263,167 @@ See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. DOC_END NAME: url_rewrite_bypass redirector_bypass TYPE: onoff LOC: Config.onoff.redirector_bypass DEFAULT: off DOC_START When this is 'on', a request will not go through the redirector if all redirectors are busy. If this is 'off' and the redirector queue grows too large, Squid will exit with a FATAL error and ask you to increase the number of redirectors. You should only enable this if the redirectors are not critical to your caching system. If you use redirectors for access control, and you enable this option, users may have access to pages they should not be allowed to request. DOC_END COMMENT_START + OPTIONS FOR STORE ID + ----------------------------------------------------------------------------- +COMMENT_END + +NAME: store_id_program storeurl_rewrite_program +TYPE: wordlist +LOC: Config.Program.store_id +DEFAULT: none +DOC_START + Specify the location of the executable StoreID helper to use. + Since they can perform almost any function there isn't one included. + + For each requested URL, the helper will receive on line with the format + + [channel-ID ] URL client_ip "/" fqdn user method [ kv-pairs] + + + After processing the request the helper must reply using the following format: + + [channel-ID ] result [ kv-pairs] + + The result code can be: + + OK store-id="..." + Use the StoreID supplied in 'store-id='. + + ERR + Do not change the StoreID. + + BH + An internal error occured in the helper, preventing + a result being identified. + + + In the future, the interface protocol will be extended with + key=value pairs ("kv-pairs" shown above). Helper programs + should be prepared to receive and possibly ignore additional + whitespace-separated tokens on each input line. + + When using the concurrency= option the protocol is changed by + introducing a query channel tag in front of the request/response. + The query channel tag is a number between 0 and concurrency-1. + This value must be echoed back unchanged to Squid as the first part + of the response relating to its request. + + WARNING: + + Re-write creates a difference in the state held by the client + and server. Possibly causing confusion when the server response + contains snippets of its view state. Embeded URLs, response + and content Location headers, etc. are not re-written by this + interface. + + By default, a StoreID helper is not used. +DOC_END + +NAME: store_id_children storeurl_rewrite_children +TYPE: HelperChildConfig +DEFAULT: 20 startup=0 idle=1 concurrency=0 +LOC: Config.storeIdChildren +DOC_START + The maximum number of StoreID helper processes to spawn. If you limit + it too few Squid will have to wait for them to process a backlog of + URLs, slowing it down. If you allow too many they will use RAM + and other system resources noticably. + + The startup= and idle= options allow some measure of skew in your + tuning. + + startup= + + Sets a minimum of how many processes are to be spawned when Squid + starts or reconfigures. When set to zero the first request will + cause spawning of the first child process to handle it. + + Starting too few will cause an initial slowdown in traffic as Squid + attempts to simultaneously spawn enough processes to cope. + + idle= + + Sets a minimum of how many processes Squid is to try and keep available + at all times. When traffic begins to rise above what the existing + processes can handle this many more will be spawned up to the maximum + configured. A minimum setting of 1 is required. + + concurrency= + + The number of requests each storeID helper can handle in + parallel. Defaults to 0 which indicates the helper + is a old-style single threaded program. + + When this directive is set to a value >= 1 then the protocol + used to communicate with the helper is modified to include + a request ID in front of the request/response. The request + ID from the request must be echoed back with the response + to that request. +DOC_END + +NAME: store_id_access storeurl_rewrite_access +TYPE: acl_access +DEFAULT: none +LOC: Config.accessList.store_id +DOC_START + If defined, this access list specifies which requests are + sent to the StoreID processes. By default all requests + are sent. + + This clause supports both fast and slow acl types. + See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. +DOC_END + +NAME: store_id_bypass storeurl_rewrite_bypass +TYPE: onoff +LOC: Config.onoff.store_id_bypass +DEFAULT: off +DOC_START + When this is 'on', a request will not go through the + helper if all helpers are busy. If this is 'off' + and the helper queue grows too large, Squid will exit + with a FATAL error and ask you to increase the number of + helpers. You should only enable this if the helperss + are not critical to your caching system. If you use + helpers for critical caching components, and you enable this + option, users may not get objects from cache. +DOC_END + +COMMENT_START OPTIONS FOR TUNING THE CACHE ----------------------------------------------------------------------------- COMMENT_END NAME: cache no_cache TYPE: acl_access DEFAULT: none LOC: Config.accessList.noCache DOC_START A list of ACL elements which, if matched and denied, cause the request to not be satisfied from the cache and the reply to not be cached. In other words, use this to force certain objects to never be cached. You must use the words 'allow' or 'deny' to indicate whether items matching the ACL should be allowed or denied into the cache. Default is to allow all to be cached. This clause supports both fast and slow acl types. See http://wiki.squid-cache.org/SquidFaq/SquidAcl for details. === modified file 'src/client_side.cc' --- src/client_side.cc 2012-11-30 11:08:47 +0000 +++ src/client_side.cc 2012-12-05 12:36:56 +0000 @@ -2803,41 +2803,40 @@ HTTP_REQUEST_ENTITY_TOO_LARGE, Http::METHOD_NONE, NULL, conn->clientConnection->remote, http->request, NULL, NULL); assert(context->http->out.offset == 0); context->pullData(); goto finish; } // We may stop producing, comm_close, and/or call setReplyToError() // below, so quit on errors to avoid http->doCallouts() if (!conn->handleRequestBodyData()) goto finish; if (!request->body_pipe->productionEnded()) { debugs(33, 5, HERE << "need more request body"); context->mayUseConnection(true); assert(conn->flags.readMore); } } http->calloutContext = new ClientRequestContext(http); - http->doCallouts(); finish: if (!notedUseOfBuffer) connNoteUseOfBuffer(conn, http->req_sz); /* * DPW 2007-05-18 * Moved the TCP_RESET feature from clientReplyContext::sendMoreData * to here because calling comm_reset_close() causes http to * be freed and the above connNoteUseOfBuffer() would hit an * assertion, not to mention that we were accessing freed memory. */ if (request && request->flags.resetTcp && Comm::IsConnOpen(conn->clientConnection)) { debugs(33, 3, HERE << "Sending TCP RST on " << conn->clientConnection); conn->flags.readMore = false; comm_reset_close(conn->clientConnection); } } === modified file 'src/client_side_reply.cc' --- src/client_side_reply.cc 2012-11-12 02:25:12 +0000 +++ src/client_side_reply.cc 2012-12-05 14:27:08 +0000 @@ -241,41 +241,41 @@ /* This function is wrong - the client parameters don't include the * header offset */ void clientReplyContext::triggerInitialStoreRead() { /* when confident, 0 becomes reqofs, and then this factors into * startSendProcess */ assert(reqofs == 0); StoreIOBuffer localTempBuffer (next()->readBuffer.length, 0, next()->readBuffer.data); storeClientCopy(sc, http->storeEntry(), localTempBuffer, SendMoreData, this); } /* there is an expired entry in the store. * setup a temporary buffer area and perform an IMS to the origin */ void clientReplyContext::processExpired() { - char *url = http->uri; + const char *url = (http->store_id.size() != 0) ? http->store_id.rawBuf() : http->uri; /* The rawBug is fine here?*/ StoreEntry *entry = NULL; debugs(88, 3, "clientReplyContext::processExpired: '" << http->uri << "'"); assert(http->storeEntry()->lastmod >= 0); /* * check if we are allowed to contact other servers * @?@: Instead of a 504 (Gateway Timeout) reply, we may want to return * a stale entry *if* it matches client requirements */ if (http->onlyIfCached()) { processOnlyIfCachedMiss(); return; } http->request->flags.refresh = 1; #if STORE_CLIENT_LIST_DEBUG /* Prevent a race with the store client memory free routines */ assert(storeClientIsThisAClient(sc, this)); #endif @@ -480,42 +480,42 @@ if (result.length == 0) { /* the store couldn't get enough data from the file for us to id the * object */ /* treat as a miss */ http->logType = LOG_TCP_MISS; processMiss(); return; } assert(!EBIT_TEST(e->flags, ENTRY_ABORTED)); /* update size of the request */ reqsize = result.length + reqofs; /* * Got the headers, now grok them */ assert(http->logType == LOG_TCP_HIT); - if (strcmp(e->mem_obj->url, urlCanonical(r)) != 0) { - debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->url << "' != '" << urlCanonical(r) << "'"); + if (strcmp(e->mem_obj->url, http->request->storeId()) != 0) { + debugs(33, DBG_IMPORTANT, "clientProcessHit: URL mismatch, '" << e->mem_obj->url << "' != '" << r->storeId() << "'"); processMiss(); return; } switch (varyEvaluateMatch(e, r)) { case VARY_NONE: /* No variance detected. Continue as normal */ break; case VARY_MATCH: /* This is the correct entity for this request. Continue */ debugs(88, 2, "clientProcessHit: Vary MATCH!"); break; case VARY_OTHER: /* This is not the correct entity for this request. We need * to requery the cache. */ removeClientStoreReference(&sc, http); @@ -859,41 +859,41 @@ } void clientReplyContext::purgeFoundObject(StoreEntry *entry) { assert (entry && !entry->isNull()); if (EBIT_TEST(entry->flags, ENTRY_SPECIAL)) { http->logType = LOG_TCP_DENIED; ErrorState *err = clientBuildError(ERR_ACCESS_DENIED, HTTP_FORBIDDEN, NULL, http->getConn()->clientConnection->remote, http->request); startError(err); return; } StoreIOBuffer localTempBuffer; /* Swap in the metadata */ http->storeEntry(entry); http->storeEntry()->lock(); - http->storeEntry()->createMemObject(http->uri, http->log_uri); + http->storeEntry()->createMemObject( http->store_id.size() < 0 ? http->store_id.termedBuf() : http->uri, http->log_uri); http->storeEntry()->mem_obj->method = http->request->method; sc = storeClientListAdd(http->storeEntry(), this); http->logType = LOG_TCP_HIT; reqofs = 0; localTempBuffer.offset = http->out.offset; localTempBuffer.length = next()->readBuffer.length; localTempBuffer.data = next()->readBuffer.data; storeClientCopy(sc, http->storeEntry(), localTempBuffer, CacheHit, this); } void @@ -1719,42 +1719,50 @@ } void clientReplyContext::doGetMoreData() { /* We still have to do store logic processing - vary, cache hit etc */ if (http->storeEntry() != NULL) { /* someone found the object in the cache for us */ StoreIOBuffer localTempBuffer; http->storeEntry()->lock(); if (http->storeEntry()->mem_obj == NULL) { /* * This if-block exists because we don't want to clobber * a preexiting mem_obj->method value if the mem_obj * already exists. For example, when a HEAD request * is a cache hit for a GET response, we want to keep * the method as GET. */ - http->storeEntry()->createMemObject(http->uri, http->log_uri); + + /* + * NOTE: there is a problem when using the http->request->storeId() + * getting relpy of "HTTP/". + * the next debug section is important since it affects a whole chain + * of events later. + */ + http->storeEntry()->createMemObject((http->store_id.size() != 0 )? http->store_id.termedBuf() : http->uri , http->log_uri); http->storeEntry()->mem_obj->method = http->request->method; + debugs(88, 3, HERE << "What mem-obj->url contains ?: "<< http->storeEntry()->mem_obj->url ); } sc = storeClientListAdd(http->storeEntry(), this); #if USE_DELAY_POOLS sc->setDelayId(DelayId::DelayClient(http)); #endif assert(http->logType == LOG_TCP_HIT); reqofs = 0; /* guarantee nothing has been sent yet! */ assert(http->out.size == 0); assert(http->out.offset == 0); if (Ip::Qos::TheConfig.isHitTosActive()) { Ip::Qos::doTosLocalHit(http->getConn()->clientConnection); } if (Ip::Qos::TheConfig.isHitNfmarkActive()) { Ip::Qos::doNfmarkLocalHit(http->getConn()->clientConnection); } @@ -2143,41 +2151,41 @@ holdingBuffer = result; processReplyAccess(); return; } /* Using this breaks the client layering just a little! */ void clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags) { assert(http != NULL); /* * For erroneous requests, we might not have a h->request, * so make a fake one. */ if (http->request == NULL) http->request = HTTPMSGLOCK(new HttpRequest(m, AnyP::PROTO_NONE, null_string)); - StoreEntry *e = storeCreateEntry(http->uri, http->log_uri, reqFlags, m); + StoreEntry *e = storeCreateEntry((http->store_id.size() != 0) ? http->store_id.termedBuf(): http->uri, http->log_uri, reqFlags, m); sc = storeClientListAdd(e, this); #if USE_DELAY_POOLS sc->setDelayId(DelayId::DelayClient(http)); #endif reqofs = 0; reqsize = 0; /* I don't think this is actually needed! -- adrian */ /* http->reqbuf = http->norm_reqbuf; */ // assert(http->reqbuf == http->norm_reqbuf); /* The next line is illegal because we don't know if the client stream * buffers have been set up */ // storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf, // SendMoreData, this); /* So, we mark the store logic as complete */ === modified file 'src/client_side_request.cc' --- src/client_side_request.cc 2012-12-01 01:20:32 +0000 +++ src/client_side_request.cc 2012-12-05 13:19:20 +0000 @@ -115,65 +115,68 @@ ClientRequestContext *result = cbdataAlloc(ClientRequestContext); return result; } void ClientRequestContext::operator delete (void *address) { ClientRequestContext *t = static_cast(address); cbdataFree(t); } /* Local functions */ /* other */ static void clientAccessCheckDoneWrapper(allow_t, void *); #if USE_SSL static void sslBumpAccessCheckDoneWrapper(allow_t, void *); #endif static int clientHierarchical(ClientHttpRequest * http); static void clientInterpretRequestHeaders(ClientHttpRequest * http); static HLPCB clientRedirectDoneWrapper; +static HLPCB clientStoreIdDoneWrapper; static void checkNoCacheDoneWrapper(allow_t, void *); SQUIDCEXTERN CSR clientGetMoreData; SQUIDCEXTERN CSS clientReplyStatus; SQUIDCEXTERN CSD clientReplyDetach; static void checkFailureRatio(err_type, hier_code); ClientRequestContext::~ClientRequestContext() { /* * Release our "lock" on our parent, ClientHttpRequest, if we * still have one */ if (http) cbdataReferenceDone(http); delete error; debugs(85,3, HERE << this << " ClientRequestContext destructed"); } -ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), error(NULL), readNextRequest(false) +ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), store_id_state(REDIRECT_NONE),error(NULL), readNextRequest(false) { http_access_done = false; redirect_done = false; redirect_fail_count = 0; + store_id_done = false; + store_id_fail_count = 0; no_cache_done = false; interpreted_req_hdrs = false; #if USE_SSL sslBumpCheckDone = false; #endif debugs(85,3, HERE << this << " ClientRequestContext constructed"); } CBDATA_CLASS_INIT(ClientHttpRequest); void * ClientHttpRequest::operator new (size_t size) { assert (size == sizeof (ClientHttpRequest)); CBDATA_INIT_TYPE(ClientHttpRequest); ClientHttpRequest *result = cbdataAlloc(ClientHttpRequest); return result; } void @@ -903,40 +906,76 @@ if (answer == ACCESS_ALLOWED) redirectStart(http, clientRedirectDoneWrapper, context); else { HelperReply nilReply; context->clientRedirectDone(nilReply); } } void ClientRequestContext::clientRedirectStart() { debugs(33, 5, HERE << "'" << http->uri << "'"); if (Config.accessList.redirector) { acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http); acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this); } else redirectStart(http, clientRedirectDoneWrapper, this); } +/* The methods handles Access checks result of StoreId access list. + * Will handle as ERROR(no change) in a case Access is not allowed. + */ +static void +clientStoreIdAccessCheckDone(allow_t answer, void *data) +{ + ClientRequestContext *context = (ClientRequestContext *)data; + ClientHttpRequest *http = context->http; + context->acl_checklist = NULL; + + if (answer == ACCESS_ALLOWED) + storeIdStart(http, clientStoreIdDoneWrapper, context); + else { + /* The nilReply is marked as Unknown if Will not set as Error. + */ + HelperReply nilReply; + nilReply.result = HelperReply::Error; + context->clientStoreIdDone(nilReply); + } +} + +/* The methods starts the helper chain. + * + */ +void +ClientRequestContext::clientStoreIdStart() +{ + debugs(33, 5, HERE << "'" << http->uri << "'"); + + if (Config.accessList.store_id) { + acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http); + acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this); + } else + storeIdStart(http, clientStoreIdDoneWrapper, this); +} + static int clientHierarchical(ClientHttpRequest * http) { const char *url = http->uri; HttpRequest *request = http->request; HttpRequestMethod method = request->method; const wordlist *p = NULL; // intercepted requests MUST NOT (yet) be sent to peers unless verified if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.spoofClientIp)) return 0; /* * IMS needs a private key, so we can use the hierarchy for IMS only if our * neighbors support private keys */ if (request->flags.ims && !neighbors_do_private_keys) return 0; @@ -1179,40 +1218,54 @@ debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " << (request->flags.noCache ? "SET" : "NOT SET")); debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " << (request->flags.cachable ? "SET" : "NOT SET")); debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " << (request->flags.hierarchical ? "SET" : "NOT SET")); } void clientRedirectDoneWrapper(void *data, const HelperReply &result) { ClientRequestContext *calloutContext = (ClientRequestContext *)data; if (!calloutContext->httpStateIsValid()) return; calloutContext->clientRedirectDone(result); } +/* Wrapping method for StoreID done callouts. + * + */ +void +clientStoreIdDoneWrapper(void *data, const HelperReply &result) +{ + ClientRequestContext *calloutContext = (ClientRequestContext *)data; + + if (!calloutContext->httpStateIsValid()) + return; + + calloutContext->clientStoreIdDone(result); +} + void ClientRequestContext::clientRedirectDone(const HelperReply &reply) { HttpRequest *old_request = http->request; debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply); assert(redirect_state == REDIRECT_PENDING); redirect_state = REDIRECT_DONE; // copy the URL rewriter response Notes to the HTTP request for logging // do it early to ensure that no matter what the outcome the notes are present. // TODO put them straight into the transaction state record (ALE?) eventually if (!old_request->helperNotes) old_request->helperNotes = new Notes; old_request->helperNotes->add(reply.notes); switch (reply.result) { case HelperReply::Unknown: case HelperReply::TT: // Handler in redirect.cc should have already mapped Unknown // IF it contained valid entry for the old URL-rewrite helper protocol @@ -1296,40 +1349,95 @@ debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " << old_request->method << " " << urlNote->firstValue() << " " << old_request->http_ver); delete new_request; } } } } break; } /* FIXME PIPELINE: This is innacurate during pipelining */ if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection)) fd_note(http->getConn()->clientConnection->fd, http->uri); assert(http->uri); http->doCallouts(); } +/* + * The method handles the different replies from StoreID helper. + */ +void +ClientRequestContext::clientStoreIdDone(const HelperReply &reply) +{ + HttpRequest *old_request = http->request; + debugs(85, 5, "'" << http->uri << "' result=" << reply); + assert(store_id_state == REDIRECT_PENDING); + store_id_state = REDIRECT_DONE; + + // copy the helper response Notes to the HTTP request for logging + // do it early to ensure that no matter what the outcome the notes are present. + // TODO put them straight into the transaction state record (ALE?) eventually + if (!old_request->helperNotes) + old_request->helperNotes = new Notes; + old_request->helperNotes->add(reply.notes); + + switch(reply.result) { + case HelperReply::Unknown: + case HelperReply::TT: + // Handler in redirect.cc should have already mapped Unknown + // IF it contained valid entry for the old helper protocol + debugs(85, DBG_IMPORTANT, "ERROR: store-URL helper returned invalid result code. Wrong helper? " << reply); + break; + + case HelperReply::BrokenHelper: + debugs(85, DBG_IMPORTANT, "ERROR: store-URL helper: " << reply << ", attempt #" << (store_id_fail_count+1) << " of 2"); + if (store_id_fail_count < 2) { // XXX: make this configurable ? + ++store_id_fail_count; + // reset state flag to try StoreID again from scratch. + store_id_done = false; + } + break; + + case HelperReply::Error: + // no change to be done. + break; + + case HelperReply::Okay: { + Note::Pointer urlNote = reply.notes.find("store-id"); + + // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write. + if (urlNote != NULL && strcmp(urlNote->firstValue(), http->uri)) { + /* storing the result in the HttpRequest */ + http->request->store_id = urlNote->firstValue(); + http->store_id = urlNote->firstValue(); + } + } + break; + } + + http->doCallouts(); +} + /** Test cache allow/deny configuration * Sets flags.cachable=1 if caching is not denied. */ void ClientRequestContext::checkNoCache() { if (Config.accessList.noCache) { acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http); acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this); } else { /* unless otherwise specified, we try to cache. */ checkNoCacheDone(ACCESS_ALLOWED); } } static void checkNoCacheDoneWrapper(allow_t answer, void *data) { ClientRequestContext *calloutContext = (ClientRequestContext *) data; @@ -1625,40 +1733,53 @@ if (!calloutContext->redirect_done) { calloutContext->redirect_done = true; assert(calloutContext->redirect_state == REDIRECT_NONE); if (Config.Program.redirect) { debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()"); calloutContext->redirect_state = REDIRECT_PENDING; calloutContext->clientRedirectStart(); return; } } if (!calloutContext->adapted_http_access_done) { debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()"); calloutContext->adapted_http_access_done = true; calloutContext->clientAccessCheck2(); return; } + + if (!calloutContext->store_id_done) { + calloutContext->store_id_done = true; + assert(calloutContext->store_id_state == REDIRECT_NONE); + + if (Config.Program.store_id) { + debugs(83, 3, HERE << "Doing calloutContext->clientStoreIdStart()"); + calloutContext->store_id_state = REDIRECT_PENDING; + calloutContext->clientStoreIdStart(); + return; + } + } + if (!calloutContext->interpreted_req_hdrs) { debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()"); calloutContext->interpreted_req_hdrs = 1; clientInterpretRequestHeaders(this); } if (!calloutContext->no_cache_done) { calloutContext->no_cache_done = true; if (Config.accessList.noCache && request->flags.cachable) { debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()"); calloutContext->checkNoCache(); return; } } } // if !calloutContext->error if (!calloutContext->tosToClientDone) { calloutContext->tosToClientDone = true; if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) { @@ -1678,41 +1799,41 @@ ch.src_addr = request->client_addr; ch.my_addr = request->my_addr; nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch); if (mark) Ip::Qos::setSockNfmark(getConn()->clientConnection, mark); } } #if USE_SSL // We need to check for SslBump even if the calloutContext->error is set // because bumping may require delaying the error until after CONNECT. if (!calloutContext->sslBumpCheckDone) { calloutContext->sslBumpCheckDone = true; if (calloutContext->sslBumpAccessCheck()) return; /* else no ssl bump required*/ } #endif if (calloutContext->error) { - const char *uri = urlCanonical(request); + const char *uri = request->storeId(); /* using StoreID to create a storeEntry by the StoreID or canonicalUrl*/ StoreEntry *e= storeCreateEntry(uri, uri, request->flags, request->method); #if USE_SSL if (sslBumpNeeded()) { // set final error but delay sending until we bump Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e); errorAppendEntry(e, calloutContext->error); calloutContext->error = NULL; getConn()->setServerBump(srvBump); e->unlock(); } else #endif { // send the error to the client now clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data; clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); assert (repContext); repContext->setReplyToStoreEntry(e); errorAppendEntry(e, calloutContext->error); calloutContext->error = NULL; if (calloutContext->readNextRequest) === modified file 'src/client_side_request.h' --- src/client_side_request.h 2012-11-04 12:27:49 +0000 +++ src/client_side_request.h 2012-12-05 14:19:45 +0000 @@ -81,40 +81,41 @@ void processRequest(); void httpStart(); bool onlyIfCached()const; bool gotEnough() const; _SQUID_INLINE_ StoreEntry *storeEntry() const; void storeEntry(StoreEntry *); _SQUID_INLINE_ StoreEntry *loggingEntry() const; void loggingEntry(StoreEntry *); _SQUID_INLINE_ ConnStateData * getConn() const; _SQUID_INLINE_ void setConn(ConnStateData *); /** Details of the client socket which produced us. * Treat as read-only for the lifetime of this HTTP request. */ Comm::ConnectionPointer clientConnection; HttpRequest *request; /* Parsed URL ... */ char *uri; char *log_uri; + String store_id; /* A more persistent store_id for parts of squid who dosnt have access to the request */ struct { int64_t offset; int64_t size; size_t headers_sz; } out; HttpHdrRangeIter range_iter; /* data for iterating thru range specs */ size_t req_sz; /* raw request size on input, not current request size */ log_type logType; struct timeval start_time; AccessLogEntry::Pointer al; ///< access.log entry struct { unsigned int accel:1; unsigned int intercepted:1; unsigned int spoof_client_ip:1; unsigned int internal:1; unsigned int done_copying:1; === modified file 'src/helper.cc' --- src/helper.cc 2012-11-24 14:30:02 +0000 +++ src/helper.cc 2012-12-05 14:34:46 +0000 @@ -831,84 +831,78 @@ helperStatefulRequestFree(r); srv->request = NULL; } if (srv->data != NULL) hlp->datapool->freeOne(srv->data); cbdataReferenceDone(srv->parent); cbdataFree(srv); } /// Calls back with a pointer to the buffer with the helper output static void helperReturnBuffer(int request_number, helper_server * srv, helper * hlp, char * msg, char * msg_end) { helper_request *r = srv->requests[request_number]; if (r) { -// TODO: parse the reply into new helper reply object -// pass that to the callback instead of msg - HLPCB *callback = r->callback; srv->requests[request_number] = NULL; r->callback = NULL; void *cbdata = NULL; if (cbdataReferenceValidDone(r->data, &cbdata)) { HelperReply response(msg, (msg_end-msg)); callback(cbdata, response); } -- srv->stats.pending; ++ srv->stats.replies; ++ hlp->stats.replies; srv->answer_time = current_time; srv->dispatch_time = r->dispatch_time; hlp->stats.avg_svc_time = Math::intAverage(hlp->stats.avg_svc_time, tvSubMsec(r->dispatch_time, current_time), hlp->stats.replies, REDIRECT_AV_FACTOR); helperRequestFree(r); } else { debugs(84, DBG_IMPORTANT, "helperHandleRead: unexpected reply on channel " << request_number << " from " << hlp->id_name << " #" << srv->index + 1 << " '" << srv->rbuf << "'"); } - srv->roffset -= (msg_end - srv->rbuf); - memmove(srv->rbuf, msg_end, srv->roffset + 1); if (!srv->flags.shutdown) { helperKickQueue(hlp); } else if (!srv->flags.closing && !srv->stats.pending) { srv->flags.closing=1; srv->writePipe->close(); - return; } } static void helperHandleRead(const Comm::ConnectionPointer &conn, char *buf, size_t len, comm_err_t flag, int xerrno, void *data) { char *t = NULL; helper_server *srv = (helper_server *)data; helper *hlp = srv->parent; assert(cbdataReferenceValid(data)); /* Bail out early on COMM_ERR_CLOSING - close handlers will tidy up for us */ if (flag == COMM_ERR_CLOSING) { return; } assert(conn->fd == srv->readPipe->fd); debugs(84, 5, "helperHandleRead: " << len << " bytes from " << hlp->id_name << " #" << srv->index + 1); @@ -919,57 +913,63 @@ } srv->roffset += len; srv->rbuf[srv->roffset] = '\0'; debugs(84, 9, "helperHandleRead: '" << srv->rbuf << "'"); if (!srv->stats.pending) { /* someone spoke without being spoken to */ debugs(84, DBG_IMPORTANT, "helperHandleRead: unexpected read from " << hlp->id_name << " #" << srv->index + 1 << ", " << (int)len << " bytes '" << srv->rbuf << "'"); srv->roffset = 0; srv->rbuf[0] = '\0'; } while ((t = strchr(srv->rbuf, hlp->eom))) { /* end of reply found */ char *msg = srv->rbuf; int i = 0; + int skip = 1; debugs(84, 3, "helperHandleRead: end of reply found"); - if (t > srv->rbuf && t[-1] == '\r' && hlp->eom == '\n') - t[-1] = '\0'; + if (t > srv->rbuf && t[-1] == '\r' && hlp->eom == '\n') { + t = '\0'; + // rewind to the \r octet which is the real terminal now + // and remember that we have to skip forward 2 places now. + skip = 2; + --t; + } *t = '\0'; if (hlp->childs.concurrency) { i = strtol(msg, &msg, 10); while (*msg && xisspace(*msg)) ++msg; } helperReturnBuffer(i, srv, hlp, msg, t); - // only skip off the \0 _after_ passing its location to helperReturnBuffer - ++t; + srv->roffset -= (t - srv->rbuf) + skip; + memmove(srv->rbuf, t, srv->roffset); } if (Comm::IsConnOpen(srv->readPipe)) { int spaceSize = srv->rbuf_sz - srv->roffset - 1; assert(spaceSize >= 0); // grow the input buffer if needed and possible if (!spaceSize && srv->rbuf_sz + 4096 <= ReadBufMaxSize) { srv->rbuf = (char *)memReallocBuf(srv->rbuf, srv->rbuf_sz + 4096, &srv->rbuf_sz); debugs(84, 3, HERE << "Grew read buffer to " << srv->rbuf_sz); spaceSize = srv->rbuf_sz - srv->roffset - 1; assert(spaceSize >= 0); } // quit reading if there is no space left if (!spaceSize) { debugs(84, DBG_IMPORTANT, "ERROR: Disconnecting from a " << "helper that overflowed " << srv->rbuf_sz << "-byte " << "Squid input buffer: " << hlp->id_name << " #" << (srv->index + 1)); @@ -1032,40 +1032,46 @@ // rewind to the \r octet which is the real terminal now // and remember that we have to skip forward 2 places now. skip = 2; --t; } *t = '\0'; if (r && cbdataReferenceValid(r->data)) { HelperReply res(srv->rbuf, (t - srv->rbuf)); res.whichServer = srv; r->callback(r->data, res); } else { debugs(84, DBG_IMPORTANT, "StatefulHandleRead: no callback data registered"); called = 0; } // only skip off the \0's _after_ passing its location in HelperReply above t += skip; srv->flags.busy = 0; + /** + * BUG: the below assumes that only one response per read() was received and discards any octets remaining. + * Doing this prohibits concurrency support with multiple replies per read(). + * TODO: check that read() setup on these buffers pays attention to roffest!=0 + * TODO: check that replies bigger than the buffer are discarded and do not to affect future replies + */ srv->roffset = 0; helperStatefulRequestFree(r); srv->request = NULL; -- srv->stats.pending; ++ srv->stats.replies; ++ hlp->stats.replies; srv->answer_time = current_time; hlp->stats.avg_svc_time = Math::intAverage(hlp->stats.avg_svc_time, tvSubMsec(srv->dispatch_time, current_time), hlp->stats.replies, REDIRECT_AV_FACTOR); if (called) helperStatefulServerDone(srv); else helperStatefulReleaseServer(srv); } === modified file 'src/redirect.cc' --- src/redirect.cc 2012-11-30 11:08:47 +0000 +++ src/redirect.cc 2012-12-05 13:31:49 +0000 @@ -49,43 +49,46 @@ #include "auth/UserRequest.h" #endif #if USE_SSL #include "ssl/support.h" #endif /// url maximum lengh + extra informations passed to redirector #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024) typedef struct { void *data; char *orig_url; Ip::Address client_addr; const char *client_ident; const char *method_s; HLPCB *handler; } redirectStateData; static HLPCB redirectHandleReply; +static HLPCB storeIdHandleReply; static void redirectStateFree(redirectStateData * r); static helper *redirectors = NULL; +static helper *storeIds = NULL; static OBJH redirectStats; +static OBJH storeIdStats; static int n_bypassed = 0; CBDATA_TYPE(redirectStateData); static void redirectHandleReply(void *data, const HelperReply &reply) { redirectStateData *r = static_cast(data); debugs(61, 5, HERE << "reply=" << reply); // XXX: This function is now kept only to check for and display the garbage use-case // and to map the old helper response format(s) into new format result code and key=value pairs // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format if (reply.result == HelperReply::Unknown) { // BACKWARD COMPATIBILITY 2012-06-15: // Some nasty old helpers send back the entire input line including extra format keys. // This is especially bad for simple perl search-replace filter scripts. // // * trim all but the first word off the response. // * warn once every 50 responses that this will stop being fixed-up soon. @@ -138,62 +141,99 @@ newReply.notes.add("rewrite-url", reply.other().content()); } void *cbdata; if (cbdataReferenceValidDone(r->data, &cbdata)) r->handler(cbdata, newReply); redirectStateFree(r); return; } } } void *cbdata; if (cbdataReferenceValidDone(r->data, &cbdata)) r->handler(cbdata, reply); redirectStateFree(r); } +/* + * Handle couple reply stuff for StoreID helper. + */ +static void +storeIdHandleReply(void *data, const HelperReply &reply) +{ + redirectStateData *r = static_cast(data); + debugs(61, 5, HERE << "StoreId helper: reply=" << reply); + + // XXX: This function is now kept only to check for and display the garbage use-case + // and to map the old helper response format(s) into new format result code and key=value pairs + // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format + void *cbdata; + if (cbdataReferenceValidDone(r->data, &cbdata)) + r->handler(cbdata, reply); + + redirectStateFree(r); +} + static void redirectStateFree(redirectStateData * r) { safe_free(r->orig_url); cbdataFree(r); } static void redirectStats(StoreEntry * sentry) { if (redirectors == NULL) { storeAppendPrintf(sentry, "No redirectors defined\n"); return; } helperStats(sentry, redirectors, "Redirector Statistics"); if (Config.onoff.redirector_bypass) storeAppendPrintf(sentry, "\nNumber of requests bypassed " "because all redirectors were busy: %d\n", n_bypassed); } +/* + * Handles storeID statistics. + */ +static void +storeIdStats(StoreEntry * sentry) +{ + if (storeIds == NULL) { + storeAppendPrintf(sentry, "No StoreIds defined\n"); + return; + } + + helperStats(sentry, storeIds, "StoreIds Statistics"); + + if (Config.onoff.store_id_bypass) + storeAppendPrintf(sentry, "\nNumber of requests bypassed " + "because all store_Ids were busy: %d\n", n_bypassed); +} + /**** PUBLIC FUNCTIONS ****/ void redirectStart(ClientHttpRequest * http, HLPCB * handler, void *data) { ConnStateData * conn = http->getConn(); redirectStateData *r = NULL; const char *fqdn; char buf[MAX_REDIRECTOR_REQUEST_STRLEN]; int sz; http_status status; char claddr[MAX_IPSTRLEN]; char myaddr[MAX_IPSTRLEN]; assert(http); assert(handler); debugs(61, 5, "redirectStart: '" << http->uri << "'"); if (Config.onoff.redirector_bypass && redirectors->stats.queue_size) { /* Skip redirector if there is one request queued */ ++n_bypassed; @@ -277,67 +317,216 @@ http->getConn() != NULL && http->getConn()->clientConnection != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr, http->request, NULL, #if USE_AUTH http->getConn() != NULL && http->getConn()->auth_user_request != NULL ? http->getConn()->auth_user_request : http->request->auth_user_request); #else NULL); #endif node = (clientStreamNode *)http->client_stream.tail->data; clientStreamRead(node, http, node->readBuffer); return; } debugs(61,6, HERE << "sending '" << buf << "' to the helper"); helperSubmit(redirectors, buf, redirectHandleReply, r); } +/* + * Handles the StoreID feature helper starting. + * For now it cannot be done using the redirectStart method. + */ +void +storeIdStart(ClientHttpRequest * http, HLPCB * handler, void *data) +{ + ConnStateData * conn = http->getConn(); + redirectStateData *r = NULL; + const char *fqdn; + char buf[MAX_REDIRECTOR_REQUEST_STRLEN]; + int sz; + http_status status; + char claddr[MAX_IPSTRLEN]; + char myaddr[MAX_IPSTRLEN]; + assert(http); + assert(handler); + debugs(61, 5, "storeIdStart: '" << http->uri << "'"); + + if (Config.onoff.store_id_bypass && storeIds->stats.queue_size) { + /* Skip StoreID Helper if there is one request queued */ + ++n_bypassed; + HelperReply bypassReply; + + bypassReply.result = HelperReply::Okay; + + bypassReply.notes.add("message","StoreIds queue too long. Bypassed."); + handler(data, bypassReply); + return; + } + + r = cbdataAlloc(redirectStateData); + r->orig_url = xstrdup(http->uri); + if (conn != NULL) + r->client_addr = conn->log_addr; + else + r->client_addr.SetNoAddr(); + r->client_ident = NULL; +#if USE_AUTH + if (http->request->auth_user_request != NULL) { + r->client_ident = http->request->auth_user_request->username(); + debugs(61, 5, HERE << "auth-user=" << (r->client_ident?r->client_ident:"NULL")); + } +#endif + + // HttpRequest initializes with null_string. So we must check both defined() and size() + if (!r->client_ident && http->request->extacl_user.defined() && http->request->extacl_user.size()) { + r->client_ident = http->request->extacl_user.termedBuf(); + debugs(61, 5, HERE << "acl-user=" << (r->client_ident?r->client_ident:"NULL")); + } + + if (!r->client_ident && conn != NULL && conn->clientConnection != NULL && conn->clientConnection->rfc931[0]) { + r->client_ident = conn->clientConnection->rfc931; + debugs(61, 5, HERE << "ident-user=" << (r->client_ident?r->client_ident:"NULL")); + } + +#if USE_SSL + + if (!r->client_ident && conn != NULL && Comm::IsConnOpen(conn->clientConnection)) { + r->client_ident = sslGetUserEmail(fd_table[conn->clientConnection->fd].ssl); + debugs(61, 5, HERE << "ssl-user=" << (r->client_ident?r->client_ident:"NULL")); + } +#endif + + if (!r->client_ident) + r->client_ident = dash_str; + + r->method_s = RequestMethodStr(http->request->method); + + r->handler = handler; + + r->data = cbdataReference(data); + + if ((fqdn = fqdncache_gethostbyaddr(r->client_addr, 0)) == NULL) + fqdn = dash_str; + + sz = snprintf(buf, MAX_REDIRECTOR_REQUEST_STRLEN, "%s %s/%s %s %s myip=%s myport=%d\n", + r->orig_url, + r->client_addr.NtoA(claddr,MAX_IPSTRLEN), + fqdn, + r->client_ident[0] ? rfc1738_escape(r->client_ident) : dash_str, + r->method_s, + http->request->my_addr.NtoA(myaddr,MAX_IPSTRLEN), + http->request->my_addr.GetPort()); + + if ((sz<=0) || (sz>=MAX_REDIRECTOR_REQUEST_STRLEN)) { + if (sz<=0) { + status = HTTP_INTERNAL_SERVER_ERROR; + debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Can not build request to be passed to redirector. Request ABORTED."); + } else { + status = HTTP_REQUEST_URI_TOO_LARGE; + debugs(61, DBG_CRITICAL, "ERROR: Gateway Failure. Request passed to redirector exceeds MAX_REDIRECTOR_REQUEST_STRLEN (" << MAX_REDIRECTOR_REQUEST_STRLEN << "). Request ABORTED."); + } + + clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data; + clientReplyContext *repContext = dynamic_cast(node->data.getRaw()); + assert (repContext); + Ip::Address tmpnoaddr; + tmpnoaddr.SetNoAddr(); + repContext->setReplyToError(ERR_GATEWAY_FAILURE, status, + http->request->method, NULL, + http->getConn() != NULL && http->getConn()->clientConnection != NULL ? + http->getConn()->clientConnection->remote : tmpnoaddr, + http->request, + NULL, +#if USE_AUTH + http->getConn() != NULL && http->getConn()->auth_user_request != NULL ? + http->getConn()->auth_user_request : http->request->auth_user_request); +#else + NULL); +#endif + + node = (clientStreamNode *)http->client_stream.tail->data; + clientStreamRead(node, http, node->readBuffer); + return; + } + + debugs(61,6, HERE << "sending '" << buf << "' to the helper"); + helperSubmit(storeIds, buf, storeIdHandleReply, r); +} + static void redirectRegisterWithCacheManager(void) { Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1); + Mgr::RegisterAction("store_id", "Store IDs Stats", storeIdStats, 0, 1); /* registering the new StoreID statistics in Mgr*/ } void redirectInit(void) { static int init = 0; redirectRegisterWithCacheManager(); - if (!Config.Program.redirect) + /* A new handling for all helpers startup */ + if (!Config.Program.redirect && !Config.Program.store_id) return; - if (redirectors == NULL) - redirectors = new helper("redirector"); + if (Config.Program.redirect){ + + if (redirectors == NULL) + redirectors = new helper("redirector"); + + redirectors->cmdline = Config.Program.redirect; + + redirectors->childs.updateLimits(Config.redirectChildren); - redirectors->cmdline = Config.Program.redirect; + redirectors->ipc_type = IPC_STREAM; - redirectors->childs.updateLimits(Config.redirectChildren); + helperOpenServers(redirectors); + } + + if (Config.Program.store_id){ + + if (storeIds == NULL) + storeIds = new helper("store_id"); - redirectors->ipc_type = IPC_STREAM; + storeIds->cmdline = Config.Program.store_id; - helperOpenServers(redirectors); + storeIds->childs.updateLimits(Config.storeIdChildren); + + storeIds->ipc_type = IPC_STREAM; + + helperOpenServers(storeIds); + } if (!init) { init = 1; CBDATA_INIT_TYPE(redirectStateData); } } void redirectShutdown(void) { - if (!redirectors) + /* A new handling for all helpers Shutdown */ + if (!storeIds && !redirectors) return; - helperShutdown(redirectors); + if (redirectors) + helperShutdown(redirectors); + + if (storeIds) + helperShutdown(storeIds); if (!shutting_down) return; delete redirectors; redirectors = NULL; + + delete storeIds; + storeIds = NULL; + } === modified file 'src/redirect.h' --- src/redirect.h 2012-11-04 12:27:49 +0000 +++ src/redirect.h 2012-12-04 17:37:42 +0000 @@ -23,22 +23,23 @@ * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "helper.h" class ClientHttpRequest; void redirectInit(void); void redirectShutdown(void); void redirectStart(ClientHttpRequest *, HLPCB *, void *); +void storeIdStart(ClientHttpRequest *, HLPCB *, void *); #endif /* SQUID_REDIRECT_H_ */ === modified file 'src/store.cc' --- src/store.cc 2012-10-26 11:36:45 +0000 +++ src/store.cc 2012-12-05 14:42:36 +0000 @@ -775,52 +775,58 @@ if (vary.defined()) { /* Again, we own this structure layout */ rep->header.putStr(HDR_X_ACCELERATOR_VARY, vary.termedBuf()); vary.clean(); } #endif pe->replaceHttpReply(rep); pe->timestampsSet(); pe->makePublic(); pe->complete(); pe->unlock(); } newkey = storeKeyPublicByRequest(mem_obj->request); - } else - newkey = storeKeyPublic(mem_obj->url, mem_obj->method); - + } else{ + debugs(20, 3, HERE << "used with StoreID: " << mem_obj->url); /* importatnt for debuging StoreID*/ + newkey = storeKeyPublic(mem_obj->url, mem_obj->method); + } if ((e2 = (StoreEntry *) hash_lookup(store_table, newkey))) { debugs(20, 3, "StoreEntry::setPublicKey: Making old '" << mem_obj->url << "' private."); e2->setPrivateKey(); e2->release(); + /* TODO: in the next newkey should be a debug section to help debug storeID. + * A proper section should be used in the right place, maybe in storeKeyPublic* ? + */ if (mem_obj->request) newkey = storeKeyPublicByRequest(mem_obj->request); - else - newkey = storeKeyPublic(mem_obj->url, mem_obj->method); + else{ + newkey = storeKeyPublic(mem_obj->url, mem_obj->method); + } + } if (key) hashDelete(); EBIT_CLR(flags, KEY_PRIVATE); hashInsert(newkey); if (swap_filen > -1) storeDirSwapLog(this, SWAP_LOG_ADD); } StoreEntry * storeCreateEntry(const char *url, const char *log_url, const RequestFlags &flags, const HttpRequestMethod& method) { StoreEntry *e = NULL; MemObject *mem = NULL; debugs(20, 3, "storeCreateEntry: '" << url << "'"); @@ -1662,40 +1668,42 @@ --hot_obj_count; } mem_status = new_status; } const char * StoreEntry::url() const { if (this == NULL) return "[null_entry]"; else if (mem_obj == NULL) return "[null_mem_obj]"; else return mem_obj->url; } void StoreEntry::createMemObject(const char *aUrl, const char *aLogUrl) { + debugs(20, 3, HERE << "A mem-obj created using :" << aUrl); /* important for StoreID debug*/ + if (mem_obj) return; if (hidden_mem_obj) { debugs(20, 3, HERE << "restoring " << hidden_mem_obj); mem_obj = hidden_mem_obj; hidden_mem_obj = NULL; mem_obj->resetUrls(aUrl, aLogUrl); return; } mem_obj = new MemObject(aUrl, aLogUrl); } /* this just sets DELAY_SENDING */ void StoreEntry::buffer() { EBIT_SET(flags, DELAY_SENDING); } === modified file 'src/store_key_md5.cc' --- src/store_key_md5.cc 2012-09-01 14:38:36 +0000 +++ src/store_key_md5.cc 2012-12-05 13:33:49 +0000 @@ -122,41 +122,41 @@ unsigned char m = (unsigned char) method.id(); SquidMD5_CTX M; SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); SquidMD5Final(digest, &M); return digest; } const cache_key * storeKeyPublicByRequest(HttpRequest * request) { return storeKeyPublicByRequestMethod(request, request->method); } const cache_key * storeKeyPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method) { static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; unsigned char m = (unsigned char) method.id(); - const char *url = urlCanonical(request); + const char *url = request->storeId(); /* storeId returns the right storeID\canonical URL for the md5 calc */ SquidMD5_CTX M; SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); if (request->vary_headers) SquidMD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers)); SquidMD5Final(digest, &M); return digest; } cache_key * storeKeyDup(const cache_key * key) { cache_key *dup = (cache_key *)memAllocate(MEM_MD5_DIGEST); memcpy(dup, key, SQUID_MD5_DIGEST_LENGTH); return dup; } === modified file 'src/store_swapmeta.cc' --- src/store_swapmeta.cc 2012-09-01 14:38:36 +0000 +++ src/store_swapmeta.cc 2012-12-05 13:41:16 +0000 @@ -50,42 +50,49 @@ while ((t = n) != NULL) { n = t->next; xfree(t->value); delete t; } } /* * Build a TLV list for a StoreEntry */ tlv * storeSwapMetaBuild(StoreEntry * e) { tlv *TLV = NULL; /* we'll return this */ tlv **T = &TLV; const char *url; const char *vary; assert(e->mem_obj != NULL); const int64_t objsize = e->mem_obj->expectedReplySize(); assert(e->swap_status == SWAPOUT_WRITING); - url = e->url(); - debugs(20, 3, "storeSwapMetaBuild: " << url ); + + /* uses the avalible storeID for meta data*/ + if (e->mem_obj->request) + url = e->mem_obj->request->storeId(); + else + url = e->url(); + + debugs(20, 3, "storeSwapMetaBuild URL: " << url ); /* important to debug StoreID */ + tlv *t = StoreMeta::Factory (STORE_META_KEY,SQUID_MD5_DIGEST_LENGTH, e->key); if (!t) { storeSwapTLVFree(TLV); return NULL; } T = StoreMeta::Add(T, t); t = StoreMeta::Factory(STORE_META_STD_LFS,STORE_HDR_METASIZE,&e->timestamp); if (!t) { storeSwapTLVFree(TLV); return NULL; } T = StoreMeta::Add(T, t); t = StoreMeta::Factory(STORE_META_URL, strlen(url) + 1, url); if (!t) { storeSwapTLVFree(TLV);