=== modified file 'src/ClientRequestContext.h' --- src/ClientRequestContext.h 2012-11-30 11:08:47 +0000 +++ src/ClientRequestContext.h 2012-12-02 06:14:44 +0000 @@ -17,72 +17,77 @@ class HelperReply; class ClientRequestContext : public RefCountable { public: void *operator new(size_t); void operator delete(void *); ClientRequestContext(ClientHttpRequest *); ~ClientRequestContext(); bool httpStateIsValid(); void hostHeaderVerify(); void hostHeaderIpVerify(const ipcache_addrs* ia, const DnsLookupDetails &dns); void hostHeaderVerifyFailed(const char *A, const char *B); void clientAccessCheck(); void clientAccessCheck2(); void clientAccessCheckDone(const allow_t &answer); void clientRedirectStart(); + void clientStoreUrlStart(); void clientRedirectDone(const HelperReply &reply); + void clientStoreUrlDone(const HelperReply &reply); void checkNoCache(); void checkNoCacheDone(const allow_t &answer); #if USE_ADAPTATION void adaptationAccessCheck(); #endif #if USE_SSL /** * Initiates and start the acl checklist to check if the a CONNECT * request must be bumped. \retval true if the acl check scheduled, false if no ssl-bump required */ bool sslBumpAccessCheck(); /// The callback function for ssl-bump access check list void sslBumpAccessCheckDone(const allow_t &answer); #endif ClientHttpRequest *http; ACLChecklist *acl_checklist; /* need ptr back so we can unreg if needed */ int redirect_state; + int store_url_state; /** * URL-rewrite/redirect helper may return BH for internal errors. * We attempt to recover by trying the lookup again, but limit the * number of retries to prevent lag and lockups. * This tracks the number of previous failures for the current context. */ uint8_t redirect_fail_count; + uint8_t storeurl_fail_count; bool host_header_verify_done; bool http_access_done; bool adapted_http_access_done; #if USE_ADAPTATION bool adaptation_acl_check_done; #endif bool redirect_done; + bool storeurl_rewrite_done; bool no_cache_done; bool interpreted_req_hdrs; bool tosToClientDone; bool nfmarkToClientDone; #if USE_SSL bool sslBumpCheckDone; #endif ErrorState *error; ///< saved error page for centralized/delayed processing bool readNextRequest; ///< whether Squid should read after error handling private: CBDATA_CLASS(ClientRequestContext); }; #endif /* SQUID_CLIENTREQUESTCONTEXT_H */ === modified file 'src/HttpRequest.cc' --- src/HttpRequest.cc 2012-11-30 22:54:46 +0000 +++ src/HttpRequest.cc 2012-12-02 06:05:02 +0000 @@ -130,40 +130,42 @@ #if USE_ADAPTATION adaptHistory_ = NULL; #endif #if ICAP_CLIENT icapHistory_ = NULL; #endif rangeOffsetLimit = -2; //a value of -2 means not checked yet } void HttpRequest::clean() { // we used to assert that the pipe is NULL, but now the request only // points to a pipe that is owned and initiated by another object. body_pipe = NULL; #if USE_AUTH auth_user_request = NULL; #endif safe_free(canonical); + safe_free(store_url); + safe_free(vary_headers); urlpath.clean(); header.clean(); if (cache_control) { delete cache_control; cache_control = NULL; } if (range) { delete range; range = NULL; } myportname.clean(); if (helperNotes) { delete helperNotes; @@ -669,20 +671,28 @@ bool HttpRequest::canHandle1xx() const { // old clients do not support 1xx unless they sent Expect: 100-continue // (we reject all other HDR_EXPECT values so just check for HDR_EXPECT) if (http_ver <= HttpVersion(1,0) && !header.has(HDR_EXPECT)) return false; // others must support 1xx control messages return true; } ConnStateData * HttpRequest::pinnedConnection() { if (clientConnectionManager.valid() && clientConnectionManager->pinning.pinned) return clientConnectionManager.get(); return NULL; } + +const char *HttpRequest::urlStoreUrl() +{ + if (store_url) + return store_url; + else + return urlCanonical((HttpRequest*)this); +} === modified file 'src/HttpRequest.h' --- src/HttpRequest.h 2012-11-30 22:57:54 +0000 +++ src/HttpRequest.h 2012-12-02 05:05:02 +0000 @@ -148,40 +148,42 @@ int host_is_numeric; #if USE_ADAPTATION mutable Adaptation::History::Pointer adaptHistory_; ///< per-HTTP transaction info #endif #if ICAP_CLIENT mutable Adaptation::Icap::History::Pointer icapHistory_; ///< per-HTTP transaction info #endif public: Ip::Address host_addr; #if USE_AUTH Auth::UserRequest::Pointer auth_user_request; #endif unsigned short port; String urlpath; char *canonical; + char *store_url; + RequestFlags flags; HttpHdrRange *range; time_t ims; int imslen; Ip::Address client_addr; #if FOLLOW_X_FORWARDED_FOR Ip::Address indirect_client_addr; #endif /* FOLLOW_X_FORWARDED_FOR */ Ip::Address my_addr; HierarchyLogEntry hier; int dnsWait; ///< sum of DNS lookup delays in milliseconds, for %dt @@ -224,40 +226,42 @@ int parseHeader(const char *parse_start, int len); virtual bool expectingBody(const HttpRequestMethod& unused, int64_t&) const; bool bodyNibbled() const; // the request has a [partially] consumed body int prefixLen(); void swapOut(StoreEntry * e); void pack(Packer * p); static void httpRequestPack(void *obj, Packer *p); static HttpRequest * CreateFromUrlAndMethod(char * url, const HttpRequestMethod& method); static HttpRequest * CreateFromUrl(char * url); ConnStateData *pinnedConnection(); + const char *urlStoreUrl(); + /** * The client connection manager, if known; * Used for any response actions needed directly to the client. * ie 1xx forwarding or connection pinning state changes */ CbcPointer clientConnectionManager; int64_t getRangeOffsetLimit(); /* the result of this function gets cached in rangeOffsetLimit */ private: const char *packableURI(bool full_uri) const; mutable int64_t rangeOffsetLimit; /* caches the result of getRangeOffsetLimit */ protected: virtual void packFirstLineInto(Packer * p, bool full_uri) const; virtual bool sanityCheckStartLine(MemBuf *buf, const size_t hdr_len, http_status *error); virtual void hdrCacheInit(); === modified file 'src/SquidConfig.h' --- src/SquidConfig.h 2012-10-29 04:59:58 +0000 +++ src/SquidConfig.h 2012-12-01 11:10:48 +0000 @@ -186,57 +186,59 @@ char *swap; CustomLog *accesslogs; #if ICAP_CLIENT CustomLog *icaplogs; #endif int rotateNumber; } Log; char *adminEmail; char *EmailFrom; char *EmailProgram; char *effectiveUser; char *visible_appname_string; char *effectiveGroup; struct { #if USE_DNSHELPER char *dnsserver; #endif wordlist *redirect; + wordlist *store_url; #if USE_UNLINKD char *unlinkd; #endif char *diskd; #if USE_SSL char *ssl_password; #endif } Program; #if USE_DNSHELPER HelperChildConfig dnsChildren; #endif HelperChildConfig redirectChildren; + HelperChildConfig storeUrlChildren; time_t authenticateGCInterval; time_t authenticateTTL; time_t authenticateIpTTL; struct { char *surrogate_id; } Accel; char *appendDomain; size_t appendDomainLen; char *pidFilename; char *netdbFilename; char *mimeTablePathname; char *etcHostsPath; char *visibleHostname; char *uniqueHostname; wordlist *hostnameAliases; char *errHtmlText; struct { char *host; @@ -301,40 +303,41 @@ int buffered_logs; int common_log; int log_mime_hdrs; int log_fqdn; int announce; int mem_pools; int test_reachability; int half_closed_clients; int refresh_all_ims; #if USE_HTTP_VIOLATIONS int reload_into_ims; #endif int offline; int redir_rewrites_host; int prefer_direct; int nonhierarchical_direct; int strip_query_terms; int redirector_bypass; + int store_url_bypass; int ignore_unknown_nameservers; int client_pconns; int server_pconns; int error_pconns; #if USE_CACHE_DIGESTS int digest_generation; #endif int ie_refresh; int vary_ignore_expire; int pipeline_prefetch; int surrogate_is_remote; int request_entities; int detect_broken_server_pconns; int balance_on_multiple_ip; int relaxed_header_parser; int check_hostnames; int allow_underscore; int via; @@ -364,40 +367,41 @@ class ACL *aclList; struct { acl_access *http; acl_access *adapted_http; acl_access *icp; acl_access *miss; acl_access *NeverDirect; acl_access *AlwaysDirect; acl_access *ASlists; acl_access *noCache; acl_access *log; #if SQUID_SNMP acl_access *snmp; #endif #if USE_HTTP_VIOLATIONS acl_access *brokenPosts; #endif acl_access *redirector; + acl_access *store_url; acl_access *reply; AclAddress *outgoing_address; #if USE_HTCP acl_access *htcp; acl_access *htcp_clr; #endif #if USE_SSL acl_access *ssl_bump; #endif #if FOLLOW_X_FORWARDED_FOR acl_access *followXFF; #endif /* FOLLOW_X_FORWARDED_FOR */ #if ICAP_CLIENT acl_access* icap; #endif } accessList; AclDenyInfoList *denyInfoList; === modified file 'src/client_side_reply.cc' --- src/client_side_reply.cc 2012-11-12 02:25:12 +0000 +++ src/client_side_reply.cc 2012-12-02 06:11:34 +0000 @@ -780,41 +780,44 @@ // TODO: can we use purgeAllCached() here instead of doing the // getPublicByRequestMethod() dance? StoreEntry::getPublicByRequestMethod(this, http->request, Http::METHOD_GET); } // Purges all entries with a given url // TODO: move to SideAgent parent, when we have one /* * We probably cannot purge Vary-affected responses because their MD5 * keys depend on vary headers. */ void purgeEntriesByUrl(HttpRequest * req, const char *url) { #if USE_HTCP bool get_or_head_sent = false; #endif for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) { if (m.respMaybeCacheable()) { - if (StoreEntry *entry = storeGetPublic(url, m)) { + /* TODO: fix things to allow store_url requests to be deleted. + * is it the place? + */ + if (StoreEntry *entry = storeGetPublic(url, m)) { debugs(88, 5, "purging " << RequestMethodStr(m) << ' ' << url); #if USE_HTCP neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION); if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) { get_or_head_sent = true; } #endif entry->release(); } } } #if USE_HTCP if (!get_or_head_sent) { neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION); } #endif } void @@ -2143,41 +2146,47 @@ holdingBuffer = result; processReplyAccess(); return; } /* Using this breaks the client layering just a little! */ void clientReplyContext::createStoreEntry(const HttpRequestMethod& m, RequestFlags reqFlags) { assert(http != NULL); /* * For erroneous requests, we might not have a h->request, * so make a fake one. */ if (http->request == NULL) http->request = HTTPMSGLOCK(new HttpRequest(m, AnyP::PROTO_NONE, null_string)); - StoreEntry *e = storeCreateEntry(http->uri, http->log_uri, reqFlags, m); + const char *url; + if (http->request->store_url) + url = http->request->store_url; + else + url = http->uri; + + StoreEntry *e = storeCreateEntry(url, http->log_uri, reqFlags, m); sc = storeClientListAdd(e, this); #if USE_DELAY_POOLS sc->setDelayId(DelayId::DelayClient(http)); #endif reqofs = 0; reqsize = 0; /* I don't think this is actually needed! -- adrian */ /* http->reqbuf = http->norm_reqbuf; */ // assert(http->reqbuf == http->norm_reqbuf); /* The next line is illegal because we don't know if the client stream * buffers have been set up */ // storeClientCopy(http->sc, e, 0, HTTP_REQBUF_SZ, http->reqbuf, // SendMoreData, this); /* So, we mark the store logic as complete */ === modified file 'src/client_side_request.cc' --- src/client_side_request.cc 2012-12-01 01:20:32 +0000 +++ src/client_side_request.cc 2012-12-02 06:15:36 +0000 @@ -115,61 +115,62 @@ ClientRequestContext *result = cbdataAlloc(ClientRequestContext); return result; } void ClientRequestContext::operator delete (void *address) { ClientRequestContext *t = static_cast(address); cbdataFree(t); } /* Local functions */ /* other */ static void clientAccessCheckDoneWrapper(allow_t, void *); #if USE_SSL static void sslBumpAccessCheckDoneWrapper(allow_t, void *); #endif static int clientHierarchical(ClientHttpRequest * http); static void clientInterpretRequestHeaders(ClientHttpRequest * http); static HLPCB clientRedirectDoneWrapper; +static HLPCB clientStoreUrlDoneWrapper; static void checkNoCacheDoneWrapper(allow_t, void *); SQUIDCEXTERN CSR clientGetMoreData; SQUIDCEXTERN CSS clientReplyStatus; SQUIDCEXTERN CSD clientReplyDetach; static void checkFailureRatio(err_type, hier_code); ClientRequestContext::~ClientRequestContext() { /* * Release our "lock" on our parent, ClientHttpRequest, if we * still have one */ if (http) cbdataReferenceDone(http); delete error; debugs(85,3, HERE << this << " ClientRequestContext destructed"); } -ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), error(NULL), readNextRequest(false) +ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), store_url_state (REDIRECT_NONE), error(NULL), readNextRequest(false) { http_access_done = false; redirect_done = false; redirect_fail_count = 0; no_cache_done = false; interpreted_req_hdrs = false; #if USE_SSL sslBumpCheckDone = false; #endif debugs(85,3, HERE << this << " ClientRequestContext constructed"); } CBDATA_CLASS_INIT(ClientHttpRequest); void * ClientHttpRequest::operator new (size_t size) { assert (size == sizeof (ClientHttpRequest)); CBDATA_INIT_TYPE(ClientHttpRequest); ClientHttpRequest *result = cbdataAlloc(ClientHttpRequest); @@ -903,40 +904,67 @@ if (answer == ACCESS_ALLOWED) redirectStart(http, clientRedirectDoneWrapper, context); else { HelperReply nilReply; context->clientRedirectDone(nilReply); } } void ClientRequestContext::clientRedirectStart() { debugs(33, 5, HERE << "'" << http->uri << "'"); if (Config.accessList.redirector) { acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http); acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this); } else redirectStart(http, clientRedirectDoneWrapper, this); } +static void +clientStoreUrlAccessCheckDone(allow_t answer, void *data) +{ + ClientRequestContext *context = (ClientRequestContext *)data; + ClientHttpRequest *http = context->http; + context->acl_checklist = NULL; + + if (answer == ACCESS_ALLOWED) + redirectStart(http, clientStoreUrlDoneWrapper, context); + else { + HelperReply nilReply; + context->clientStoreUrlDone(nilReply); + } +} + +void +ClientRequestContext::clientStoreUrlStart() +{ + debugs(33, 5, HERE << "'" << http->uri << "'"); + + if (Config.accessList.store_url) { + acl_checklist = clientAclChecklistCreate(Config.accessList.store_url, http); + acl_checklist->nonBlockingCheck(clientStoreUrlAccessCheckDone, this); + } else + redirectStart(http, clientStoreUrlDoneWrapper, this); +} + static int clientHierarchical(ClientHttpRequest * http) { const char *url = http->uri; HttpRequest *request = http->request; HttpRequestMethod method = request->method; const wordlist *p = NULL; // intercepted requests MUST NOT (yet) be sent to peers unless verified if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.spoofClientIp)) return 0; /* * IMS needs a private key, so we can use the hierarchy for IMS only if our * neighbors support private keys */ if (request->flags.ims && !neighbors_do_private_keys) return 0; @@ -1236,41 +1264,41 @@ // #1: redirect with a specific status code OK status=NNN url="..." // #2: redirect with a default status code OK url="..." // #3: re-write the URL OK rewrite-url="..." Note::Pointer statusNote = reply.notes.find("status"); Note::Pointer urlNote = reply.notes.find("url"); if (urlNote != NULL) { // HTTP protocol redirect to be done. // TODO: change default redirect status for appropriate requests // Squid defaults to 302 status for now for better compatibility with old clients. // HTTP/1.0 client should get 302 (HTTP_MOVED_TEMPORARILY) // HTTP/1.1 client contacting reverse-proxy should get 307 (HTTP_TEMPORARY_REDIRECT) // HTTP/1.1 client being diverted by forward-proxy should get 303 (HTTP_SEE_OTHER) http_status status = HTTP_MOVED_TEMPORARILY; if (statusNote != NULL) { const char * result = statusNote->firstValue(); status = (http_status) atoi(result); } - + /* not needed for store url*/ if (status == HTTP_MOVED_PERMANENTLY || status == HTTP_MOVED_TEMPORARILY || status == HTTP_SEE_OTHER || status == HTTP_PERMANENT_REDIRECT || status == HTTP_TEMPORARY_REDIRECT) { http->redirect.status = status; http->redirect.location = xstrdup(urlNote->firstValue()); // TODO: validate the URL produced here is RFC 2616 compliant absolute URI } else { debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote->firstValue()); } } else { // URL-rewrite wanted. Ew. urlNote = reply.notes.find("rewrite-url"); // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write. if (urlNote != NULL && strcmp(urlNote->firstValue(), http->uri)) { // XXX: validate the URL properly *without* generating a whole new request object right here. // XXX: the clone() should be done only AFTER we know the new URL is valid. HttpRequest *new_request = old_request->clone(); @@ -1296,40 +1324,103 @@ debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " << old_request->method << " " << urlNote->firstValue() << " " << old_request->http_ver); delete new_request; } } } } break; } /* FIXME PIPELINE: This is innacurate during pipelining */ if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection)) fd_note(http->getConn()->clientConnection->fd, http->uri); assert(http->uri); http->doCallouts(); } +void +clientStoreUrlDoneWrapper(void *data, const HelperReply &result) +{ + ClientRequestContext *calloutContext = (ClientRequestContext *)data; + + if (!calloutContext->httpStateIsValid()) + return; + + calloutContext->clientStoreUrlDone(result); +} + +void +ClientRequestContext::clientStoreUrlDone(const HelperReply &reply) +{ + HttpRequest *old_request = http->request; + debugs(85, 5, "'" << http->uri << "' result=" << reply); + assert(store_url_state == REDIRECT_PENDING); + store_url_state = REDIRECT_DONE; + + // copy the helper response Notes to the HTTP request for logging + // do it early to ensure that no matter what the outcome the notes are present. + // TODO put them straight into the transaction state record (ALE?) eventually + if (!old_request->helperNotes) + old_request->helperNotes = new Notes; + old_request->helperNotes->add(reply.notes); + + switch(reply.result) { + case HelperReply::Unknown: + case HelperReply::TT: + // Handler in redirect.cc should have already mapped Unknown + // IF it contained valid entry for the old URL-rewrite helper protocol + debugs(85, DBG_IMPORTANT, "ERROR: store-URL helper returned invalid result code. Wrong helper? " << reply); + break; + + case HelperReply::BrokenHelper: + debugs(85, DBG_IMPORTANT, "ERROR: store-URL helper: " << reply << ", attempt #" << (storeurl_fail_count+1) << " of 2"); + if (storeurl_fail_count < 2) { // XXX: make this configurable ? + ++storeurl_fail_count; + // reset state flag to try redirector again from scratch. + storeurl_rewrite_done = false; + } + break; + + case HelperReply::Error: + // no change to be done. + break; + + case HelperReply::Okay: { + Note::Pointer urlNote = reply.notes.find("rewrite-url"); + + // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write. + if (urlNote != NULL && strcmp(urlNote->firstValue(), http->uri)) { + + /* storing the result in the HttpRequest */ + http->request->store_url= xstrdup(urlNote->firstValue()); + } + } + break; + } + + http->doCallouts(); +} + /** Test cache allow/deny configuration * Sets flags.cachable=1 if caching is not denied. */ void ClientRequestContext::checkNoCache() { if (Config.accessList.noCache) { acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http); acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this); } else { /* unless otherwise specified, we try to cache. */ checkNoCacheDone(ACCESS_ALLOWED); } } static void checkNoCacheDoneWrapper(allow_t answer, void *data) { ClientRequestContext *calloutContext = (ClientRequestContext *) data; @@ -1625,40 +1716,52 @@ if (!calloutContext->redirect_done) { calloutContext->redirect_done = true; assert(calloutContext->redirect_state == REDIRECT_NONE); if (Config.Program.redirect) { debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()"); calloutContext->redirect_state = REDIRECT_PENDING; calloutContext->clientRedirectStart(); return; } } if (!calloutContext->adapted_http_access_done) { debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()"); calloutContext->adapted_http_access_done = true; calloutContext->clientAccessCheck2(); return; } + if (!calloutContext->storeurl_rewrite_done) { + calloutContext->storeurl_rewrite_done = true; + assert(calloutContext->store_url_state == REDIRECT_NONE); + + if (Config.Program.store_url) { + debugs(83, 3, HERE << "Doing calloutContext->clientStoreUrlStart()"); + calloutContext->store_url_state = REDIRECT_PENDING; + calloutContext->clientStoreUrlStart(); + return; + } + } + if (!calloutContext->interpreted_req_hdrs) { debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()"); calloutContext->interpreted_req_hdrs = 1; clientInterpretRequestHeaders(this); } if (!calloutContext->no_cache_done) { calloutContext->no_cache_done = true; if (Config.accessList.noCache && request->flags.cachable) { debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()"); calloutContext->checkNoCache(); return; } } } // if !calloutContext->error if (!calloutContext->tosToClientDone) { calloutContext->tosToClientDone = true; if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) { === modified file 'src/client_side_request.h' --- src/client_side_request.h 2012-11-04 12:27:49 +0000 +++ src/client_side_request.h 2012-12-01 10:09:27 +0000 @@ -80,40 +80,41 @@ bool multipartRangeRequest() const; void processRequest(); void httpStart(); bool onlyIfCached()const; bool gotEnough() const; _SQUID_INLINE_ StoreEntry *storeEntry() const; void storeEntry(StoreEntry *); _SQUID_INLINE_ StoreEntry *loggingEntry() const; void loggingEntry(StoreEntry *); _SQUID_INLINE_ ConnStateData * getConn() const; _SQUID_INLINE_ void setConn(ConnStateData *); /** Details of the client socket which produced us. * Treat as read-only for the lifetime of this HTTP request. */ Comm::ConnectionPointer clientConnection; HttpRequest *request; /* Parsed URL ... */ char *uri; + char *store_url; char *log_uri; struct { int64_t offset; int64_t size; size_t headers_sz; } out; HttpHdrRangeIter range_iter; /* data for iterating thru range specs */ size_t req_sz; /* raw request size on input, not current request size */ log_type logType; struct timeval start_time; AccessLogEntry::Pointer al; ///< access.log entry struct { unsigned int accel:1; unsigned int intercepted:1; unsigned int spoof_client_ip:1; unsigned int internal:1; === modified file 'src/redirect.cc' --- src/redirect.cc 2012-11-30 11:08:47 +0000 +++ src/redirect.cc 2012-12-02 06:18:09 +0000 @@ -51,41 +51,43 @@ #if USE_SSL #include "ssl/support.h" #endif /// url maximum lengh + extra informations passed to redirector #define MAX_REDIRECTOR_REQUEST_STRLEN (MAX_URL + 1024) typedef struct { void *data; char *orig_url; Ip::Address client_addr; const char *client_ident; const char *method_s; HLPCB *handler; } redirectStateData; static HLPCB redirectHandleReply; static void redirectStateFree(redirectStateData * r); static helper *redirectors = NULL; +static helper *storeUrls = NULL; static OBJH redirectStats; +static OBJH storeUrlStats; static int n_bypassed = 0; CBDATA_TYPE(redirectStateData); static void redirectHandleReply(void *data, const HelperReply &reply) { redirectStateData *r = static_cast(data); debugs(61, 5, HERE << "reply=" << reply); // XXX: This function is now kept only to check for and display the garbage use-case // and to map the old helper response format(s) into new format result code and key=value pairs // it can be removed when the helpers are all updated to the normalized "OK/ERR kv-pairs" format if (reply.result == HelperReply::Unknown) { // BACKWARD COMPATIBILITY 2012-06-15: // Some nasty old helpers send back the entire input line including extra format keys. // This is especially bad for simple perl search-replace filter scripts. // // * trim all but the first word off the response. // * warn once every 50 responses that this will stop being fixed-up soon. @@ -160,40 +162,55 @@ { safe_free(r->orig_url); cbdataFree(r); } static void redirectStats(StoreEntry * sentry) { if (redirectors == NULL) { storeAppendPrintf(sentry, "No redirectors defined\n"); return; } helperStats(sentry, redirectors, "Redirector Statistics"); if (Config.onoff.redirector_bypass) storeAppendPrintf(sentry, "\nNumber of requests bypassed " "because all redirectors were busy: %d\n", n_bypassed); } +static void +storeUrlStats(StoreEntry * sentry) +{ + if (storeUrls == NULL) { + storeAppendPrintf(sentry, "No StoreUrls defined\n"); + return; + } + + helperStats(sentry, storeUrls, "StoreUrls Statistics"); + + if (Config.onoff.store_url_bypass) + storeAppendPrintf(sentry, "\nNumber of requests bypassed " + "because all storeUrls were busy: %d\n", n_bypassed); +} + /**** PUBLIC FUNCTIONS ****/ void redirectStart(ClientHttpRequest * http, HLPCB * handler, void *data) { ConnStateData * conn = http->getConn(); redirectStateData *r = NULL; const char *fqdn; char buf[MAX_REDIRECTOR_REQUEST_STRLEN]; int sz; http_status status; char claddr[MAX_IPSTRLEN]; char myaddr[MAX_IPSTRLEN]; assert(http); assert(handler); debugs(61, 5, "redirectStart: '" << http->uri << "'"); if (Config.onoff.redirector_bypass && redirectors->stats.queue_size) { /* Skip redirector if there is one request queued */ ++n_bypassed; @@ -281,63 +298,89 @@ #if USE_AUTH http->getConn() != NULL && http->getConn()->auth_user_request != NULL ? http->getConn()->auth_user_request : http->request->auth_user_request); #else NULL); #endif node = (clientStreamNode *)http->client_stream.tail->data; clientStreamRead(node, http, node->readBuffer); return; } debugs(61,6, HERE << "sending '" << buf << "' to the helper"); helperSubmit(redirectors, buf, redirectHandleReply, r); } static void redirectRegisterWithCacheManager(void) { Mgr::RegisterAction("redirector", "URL Redirector Stats", redirectStats, 0, 1); + Mgr::RegisterAction("storeurl", "Store URL rewriter Stats", storeUrlStats, 0, 1); } void redirectInit(void) { static int init = 0; redirectRegisterWithCacheManager(); - if (!Config.Program.redirect) - return; + if (!Config.Program.redirect && !Config.Program.store_url) + return; - if (redirectors == NULL) - redirectors = new helper("redirector"); + if (Config.Program.redirect){ - redirectors->cmdline = Config.Program.redirect; + if (redirectors == NULL) + redirectors = new helper("redirector"); - redirectors->childs.updateLimits(Config.redirectChildren); + redirectors->cmdline = Config.Program.redirect; - redirectors->ipc_type = IPC_STREAM; + redirectors->childs.updateLimits(Config.redirectChildren); - helperOpenServers(redirectors); + redirectors->ipc_type = IPC_STREAM; + + helperOpenServers(redirectors); + } + + if (Config.Program.store_url){ + + if (storeUrls == NULL) + storeUrls = new helper("store_url"); + + storeUrls->cmdline = Config.Program.store_url; + + storeUrls->childs.updateLimits(Config.storeUrlChildren); + + storeUrls->ipc_type = IPC_STREAM; + + helperOpenServers(storeUrls); + } if (!init) { init = 1; CBDATA_INIT_TYPE(redirectStateData); } } void redirectShutdown(void) { - if (!redirectors) + if (!storeUrls && !redirectors) return; - helperShutdown(redirectors); + if (redirectors) + helperShutdown(redirectors); + + if (storeUrls) + helperShutdown(storeUrls); if (!shutting_down) return; delete redirectors; redirectors = NULL; + + delete storeUrls; + storeUrls = NULL; + } === modified file 'src/store_key_md5.cc' --- src/store_key_md5.cc 2012-09-01 14:38:36 +0000 +++ src/store_key_md5.cc 2012-12-02 06:18:45 +0000 @@ -122,41 +122,41 @@ unsigned char m = (unsigned char) method.id(); SquidMD5_CTX M; SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); SquidMD5Final(digest, &M); return digest; } const cache_key * storeKeyPublicByRequest(HttpRequest * request) { return storeKeyPublicByRequestMethod(request, request->method); } const cache_key * storeKeyPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method) { static cache_key digest[SQUID_MD5_DIGEST_LENGTH]; unsigned char m = (unsigned char) method.id(); - const char *url = urlCanonical(request); + const char *url = request->urlStoreUrl(); SquidMD5_CTX M; SquidMD5Init(&M); SquidMD5Update(&M, &m, sizeof(m)); SquidMD5Update(&M, (unsigned char *) url, strlen(url)); if (request->vary_headers) SquidMD5Update(&M, (unsigned char *) request->vary_headers, strlen(request->vary_headers)); SquidMD5Final(digest, &M); return digest; } cache_key * storeKeyDup(const cache_key * key) { cache_key *dup = (cache_key *)memAllocate(MEM_MD5_DIGEST); memcpy(dup, key, SQUID_MD5_DIGEST_LENGTH); return dup; }