=== modified file 'src/client_db.cc' --- src/client_db.cc 2011-08-04 03:21:06 +0000 +++ src/client_db.cc 2011-09-11 06:40:41 +0000 @@ -17,41 +17,41 @@ * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "squid.h" #include "event.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "ClientInfo.h" #include "ip/Address.h" #include "mgr/Registration.h" #include "SquidMath.h" #include "SquidTime.h" #include "Store.h" static hash_table *client_table = NULL; static ClientInfo *clientdbAdd(const Ip::Address &addr); static FREE clientdbFreeItem; static void clientdbStartGC(void); static void clientdbScheduledGC(void *); #if USE_DELAY_POOLS static int max_clients = 32768; #else static int max_clients = 32; #endif === modified file 'src/client_side_reply.cc' --- src/client_side_reply.cc 2011-08-10 15:54:51 +0000 +++ src/client_side_reply.cc 2011-09-14 06:26:24 +0000 @@ -38,41 +38,41 @@ /* old includes without reasons given. */ #include "squid.h" #include "acl/FilledChecklist.h" #include "acl/Gadgets.h" #if USE_AUTH #include "auth/UserRequest.h" #endif #include "client_side.h" #include "client_side_reply.h" #include "clientStream.h" #if USE_DELAY_POOLS #include "DelayPools.h" #endif #include "errorpage.h" #if USE_SQUID_ESI #include "esi/Esi.h" #endif #include "fde.h" #include "forward.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpReply.h" #include "HttpRequest.h" #include "ip/QosConfig.h" #include "ipcache.h" #include "MemObject.h" #include "ProtoPort.h" #include "SquidTime.h" #include "StoreClient.h" #include "Store.h" CBDATA_CLASS_INIT(clientReplyContext); /* Local functions */ extern "C" CSS clientReplyStatus; extern ErrorState *clientBuildError(err_type, http_status, char const *, Ip::Address &, HttpRequest *); /* privates */ clientReplyContext::~clientReplyContext() { === modified file 'src/client_side_request.cc' --- src/client_side_request.cc 2011-11-04 23:28:49 +0000 +++ src/client_side_request.cc 2011-11-04 23:30:16 +0000 @@ -49,41 +49,41 @@ #include "adaptation/AccessCheck.h" #include "adaptation/Answer.h" #include "adaptation/Iterator.h" #include "adaptation/Service.h" #if ICAP_CLIENT #include "adaptation/icap/History.h" #endif #endif #if USE_AUTH #include "auth/UserRequest.h" #endif #include "clientStream.h" #include "client_side.h" #include "client_side_reply.h" #include "client_side_request.h" #include "ClientRequestContext.h" #include "comm/Connection.h" #include "comm/Write.h" #include "compat/inet_pton.h" #include "fde.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpHdrCc.h" #include "HttpReply.h" #include "HttpRequest.h" #include "ip/QosConfig.h" #include "MemObject.h" #include "ProtoPort.h" #include "Store.h" #include "SquidTime.h" #include "wordlist.h" #include "err_detail_type.h" #if USE_SSL #include "ssl/support.h" #endif #if LINGERING_CLOSE #define comm_close comm_lingering_close #endif static const char *const crlf = "\r\n"; === added file 'src/format/ByteCode.h' --- src/format/ByteCode.h 1970-01-01 00:00:00 +0000 +++ src/format/ByteCode.h 2011-09-23 00:58:53 +0000 @@ -0,0 +1,209 @@ +#ifndef _SQUID_FMT_BYTECODE_H +#define _SQUID_FMT_BYTECODE_H + +/* + * Squid configuration allows users to define custom formats in + * several components. + * - logging + * - external ACL input + * - deny page URL + * + * These enumerations and classes define the API for parsing of + * format directives to define these patterns. Along with output + * functionality to produce formatted buffers. + */ + +namespace Format +{ + +/* + * Bytecodes for the configureable format stuff + */ +typedef enum { + LFT_NONE, /* dummy */ + + /* arbitrary string between tokens */ + LFT_STRING, + + /* client TCP connection remote end details */ + LFT_CLIENT_IP_ADDRESS, + LFT_CLIENT_FQDN, + LFT_CLIENT_PORT, + LFT_CLIENT_EUI, + + /* client TCP connection local end details */ + LFT_CLIENT_LOCAL_IP, + LFT_CLIENT_LOCAL_PORT, + /*LFT_CLIENT_LOCAL_FQDN, (rDNS) */ + + /* client connection local squid.conf details */ + LFT_LOCAL_LISTENING_IP, + LFT_LOCAL_LISTENING_PORT, + /*LFT_LOCAL_LISTENING_NAME, (myportname) */ + + /* server TCP connection remote end details */ + LFT_SERVER_IP_ADDRESS, + LFT_SERVER_FQDN_OR_PEER_NAME, + LFT_SERVER_PORT, + + /* server TCP connection local end details */ + LFT_SERVER_LOCAL_IP, + LFT_SERVER_LOCAL_IP_OLD_27, + LFT_SERVER_LOCAL_PORT, + + /* original Request-Line details recieved from client */ + LFT_CLIENT_REQ_METHOD, + LFT_CLIENT_REQ_URI, + LFT_CLIENT_REQ_URLPATH, + /* LFT_CLIENT_REQ_QUERY, */ + LFT_CLIENT_REQ_VERSION, + + /* Request-Line details recieved from client (legacy, filtered) */ + LFT_REQUEST_METHOD, + LFT_REQUEST_URI, + LFT_REQUEST_URLPATH_OLD_31, + /*LFT_REQUEST_QUERY, */ + LFT_REQUEST_VERSION_OLD_2X, + LFT_REQUEST_VERSION, + + /* request header details pre-adaptation */ + LFT_REQUEST_HEADER, + LFT_REQUEST_HEADER_ELEM, + LFT_REQUEST_ALL_HEADERS, + + /* request header details post-adaptation */ + LFT_ADAPTED_REQUEST_HEADER, + LFT_ADAPTED_REQUEST_HEADER_ELEM, + LFT_ADAPTED_REQUEST_ALL_HEADERS, + + /* Request-Line details sent to the server/peer */ + LFT_SERVER_REQ_METHOD, + LFT_SERVER_REQ_URI, + LFT_SERVER_REQ_URLPATH, + /*LFT_SERVER_REQ_QUERY, */ + LFT_SERVER_REQ_VERSION, + + /* request meta details */ + LFT_REQUEST_SIZE_TOTAL, + /*LFT_REQUEST_SIZE_LINE, */ + LFT_REQUEST_SIZE_HEADERS, + /*LFT_REQUEST_SIZE_BODY, */ + /*LFT_REQUEST_SIZE_BODY_NO_TE, */ + + /* original Status-Line details recieved from server */ + // XXX: todo + + /* Status-Line details sent to the client */ + // XXX: todo + + /* response Status-Line details (legacy, filtered) */ + LFT_HTTP_SENT_STATUS_CODE_OLD_30, + LFT_HTTP_SENT_STATUS_CODE, + LFT_HTTP_RECEIVED_STATUS_CODE, + /*LFT_HTTP_STATUS, */ + LFT_HTTP_BODY_BYTES_READ, + + /* response header details pre-adaptation */ + LFT_REPLY_HEADER, + LFT_REPLY_HEADER_ELEM, + LFT_REPLY_ALL_HEADERS, + + /* response header details post-adaptation */ + /* LFT_ADAPTED_REPLY_HEADER, */ + /* LFT_ADAPTED_REPLY_HEADER_ELEM, */ + /* LFT_ADAPTED_REPLY_ALL_HEADERS, */ + + /* response meta details */ + LFT_REPLY_SIZE_TOTAL, + LFT_REPLY_HIGHOFFSET, + LFT_REPLY_OBJECTSIZE, + /*LFT_REPLY_SIZE_LINE, */ + LFT_REPLY_SIZE_HEADERS, + /*LFT_REPLY_SIZE_BODY, */ + /*LFT_REPLY_SIZE_BODY_NO_TE, */ + + /* client credentials */ + LFT_USER_NAME, /* any source will do */ + LFT_USER_LOGIN, + LFT_USER_IDENT, + /*LFT_USER_REALM, */ + /*LFT_USER_SCHEME, */ + LFT_USER_EXTERNAL, + /* LFT_USER_SSL_CERT, */ + + /* global time details */ + LFT_TIME_SECONDS_SINCE_EPOCH, + LFT_TIME_SUBSECOND, + LFT_TIME_LOCALTIME, + LFT_TIME_GMT, + + /* processing time details */ + LFT_TIME_TO_HANDLE_REQUEST, + LFT_PEER_RESPONSE_TIME, + LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, + LFT_DNS_WAIT_TIME, + + /* Squid internal processing details */ + LFT_SQUID_STATUS, + LFT_SQUID_ERROR, + LFT_SQUID_ERROR_DETAIL, + LFT_SQUID_HIERARCHY, + + LFT_MIME_TYPE, + LFT_TAG, + LFT_IO_SIZE_TOTAL, + LFT_EXT_LOG, + + LFT_SEQUENCE_NUMBER, + +#if USE_ADAPTATION + LFT_ADAPTATION_SUM_XACT_TIMES, + LFT_ADAPTATION_ALL_XACT_TIMES, + LFT_ADAPTATION_LAST_HEADER, + LFT_ADAPTATION_LAST_HEADER_ELEM, + LFT_ADAPTATION_LAST_ALL_HEADERS, +#endif + +#if ICAP_CLIENT + + LFT_ICAP_TOTAL_TIME, + + LFT_ICAP_ADDR, + LFT_ICAP_SERV_NAME, + LFT_ICAP_REQUEST_URI, + LFT_ICAP_REQUEST_METHOD, + LFT_ICAP_BYTES_SENT, + LFT_ICAP_BYTES_READ, + LFT_ICAP_BODY_BYTES_READ, + + LFT_ICAP_REQ_HEADER, + LFT_ICAP_REQ_HEADER_ELEM, + LFT_ICAP_REQ_ALL_HEADERS, + + LFT_ICAP_REP_HEADER, + LFT_ICAP_REP_HEADER_ELEM, + LFT_ICAP_REP_ALL_HEADERS, + + LFT_ICAP_TR_RESPONSE_TIME, + LFT_ICAP_IO_TIME, + LFT_ICAP_OUTCOME, + LFT_ICAP_STATUS_CODE, +#endif + + LFT_PERCENT /* special string cases for escaped chars */ +} ByteCode_t; + +/// Quoting style for a format output. +enum Quoting { + LOG_QUOTE_NONE = 0, + LOG_QUOTE_QUOTES, + LOG_QUOTE_MIMEBLOB, + LOG_QUOTE_URL, + LOG_QUOTE_RAW +}; + +extern const char *log_tags[]; + +} // namespace Format + +#endif /* _SQUID_FMT_BYTECODE_H */ === added file 'src/format/Config.cc' --- src/format/Config.cc 1970-01-01 00:00:00 +0000 +++ src/format/Config.cc 2011-11-05 11:08:25 +0000 @@ -0,0 +1,44 @@ +#include "config.h" +#include "format/Config.h" +#include "protos.h" +#include + +Format::FmtConfig Format::TheConfig; + +void +Format::FmtConfig::parseFormats() +{ + char *name, *def; + + if ((name = strtok(NULL, w_space)) == NULL) + self_destruct(); + + if ((def = strtok(NULL, "\r\n")) == NULL) { + self_destruct(); + return; + } + + debugs(3, 2, "Custom Format for '" << name << "' is '" << def << "'"); + + Format *nlf = new Format(name); + + if (!nlf->parse(def)) { + self_destruct(); + return; + } + + // add to global config list + nlf->next = formats; + formats = nlf; +} + +void +Format::FmtConfig::registerTokens(const String &nsName, TokenTableEntry const *tokenArray) +{ +// assert(tokenArray != NULL); + + if (tokenArray != NULL) + tokens.push_back(TokenNamespace(nsName, tokenArray)); + else + debugs(0,0, "BUG: format tokens for '" << nsName << "' missing!"); +} === added file 'src/format/Config.h' --- src/format/Config.h 1970-01-01 00:00:00 +0000 +++ src/format/Config.h 2011-11-15 11:26:17 +0000 @@ -0,0 +1,76 @@ +#ifndef SQUID_SRC_FORMAT_CONFIG_H +#define SQUID_SRC_FORMAT_CONFIG_H + +#include "format/Format.h" +//#include "format/TokenTableEntry.h" +#include "SquidString.h" +#include + +class StoreEntry; + +namespace Format +{ + +class TokenTableEntry; + +/// A namespace or 'set' of tokens +/// components register their namespace prefix and an array of tokens +/// which can then be embeded in any format. +class TokenNamespace +{ +public: + TokenNamespace(const String &nsName, TokenTableEntry const *tSet) : prefix(nsName), tokenSet(tSet) {} + + /// prefix namespace name (excluding '::') + String prefix; + + /// array of tokens inside this namespace + /// The set of tokens may change, but the location of it pointed to from here must not. + TokenTableEntry const *tokenSet; +}; + +/// The set of custom formats defined in squid.conf +/// +class FmtConfig +{ +public: + /// Parse a log format directive line (logfile_format) + void parseFormats(); + + /// Dump/display the formats currently known to the provided StoreEntry object + void dumpFormats(StoreEntry *e, const char *name) { + formats->dump(e, name); + } + + /* Register a namespace set of tokens to be accepted by the format parser. + * Multiple arrays can be registered, they will be scanned for + * in order registered. So care needs to be taken that arrays registered + * first do not overlap or consume tokens registered later for a namespace. + */ + void registerTokens(const String &nsName, TokenTableEntry const *tokenArray); + + /// Linked list of custom formats + Format *formats; + + /// list of token namespaces registered + std::list tokens; + +#if USE_ADAPTATION + bool hasAdaptToken; +#endif + +#if ICAP_CLIENT + bool hasIcapToken; +#endif +}; + +extern FmtConfig TheConfig; + +} // namespace Format + +// Legacy parsing wrappers +#define parse_format(X) (X)->parseFormats() +#define free_format(X) do{ delete (*(X)).formats; (*(X)).formats=NULL; }while(false) +#define dump_format(E,N,D) (D).dumpFormats((E),(N)) + +#endif === modified file 'src/format/Format.cc' --- src/format/Format.cc 2011-10-14 01:49:17 +0000 +++ src/format/Format.cc 2011-10-16 05:00:24 +0000 @@ -1,28 +1,28 @@ #include "config.h" #include "AccessLogEntry.h" #include "comm/Connection.h" #include "err_detail_type.h" #include "errorpage.h" #include "format/Format.h" #include "format/Quoting.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpRequest.h" #include "MemBuf.h" #include "rfc1738.h" #include "SquidTime.h" #include "Store.h" /// Convert a string to NULL pointer if it is "" #define strOrNull(s) ((s)==NULL||(s)[0]=='\0'?NULL:(s)) Format::Format::Format(const char *n) : format(NULL), next(NULL) { name = xstrdup(n); } Format::Format::~Format() { // erase the list without consuming stack space while (next) { @@ -499,50 +499,50 @@ if (al->request) sb = al->adapted_request->header.getByName(fmt->data.header.header); out = sb.termedBuf(); quote = 1; break; case LFT_REPLY_HEADER: if (al->reply) sb = al->reply->header.getByName(fmt->data.header.header); out = sb.termedBuf(); quote = 1; break; #if USE_ADAPTATION - case LTF_ADAPTATION_SUM_XACT_TIMES: + case LFT_ADAPTATION_SUM_XACT_TIMES: if (al->request) { Adaptation::History::Pointer ah = al->request->adaptHistory(); if (ah != NULL) ah->sumLogString(fmt->data.string, sb); out = sb.termedBuf(); } break; - case LTF_ADAPTATION_ALL_XACT_TIMES: + case LFT_ADAPTATION_ALL_XACT_TIMES: if (al->request) { Adaptation::History::Pointer ah = al->request->adaptHistory(); if (ah != NULL) ah->allLogString(fmt->data.string, sb); out = sb.termedBuf(); } break; case LFT_ADAPTATION_LAST_HEADER: if (al->request) { const Adaptation::History::Pointer ah = al->request->adaptHistory(); if (ah != NULL) // XXX: add adapt::allMeta.getByName(fmt->data.header.header); } // XXX: here and elsewhere: move such code inside the if guard out = sb.termedBuf(); quote = 1; === modified file 'src/format/Makefile.am' --- src/format/Makefile.am 2011-08-04 03:21:06 +0000 +++ src/format/Makefile.am 2011-09-11 06:58:13 +0000 @@ -1,13 +1,17 @@ include $(top_srcdir)/src/Common.am include $(top_srcdir)/src/TestHeaders.am noinst_LTLIBRARIES = libformat.la libformat_la_SOURCES = \ + ByteCode.h \ + Config.cc \ + Config.h \ Format.cc \ Format.h \ Quoting.cc \ Quoting.h \ - Tokens.cc \ - Tokens.h + Token.cc \ + Token.h \ + TokenTableEntry.h === renamed file 'src/format/Tokens.cc' => 'src/format/Token.cc' --- src/format/Tokens.cc 2011-10-13 17:05:25 +0000 +++ src/format/Token.cc 2011-11-16 11:24:10 +0000 @@ -1,82 +1,84 @@ #include "config.h" -#include "format/Tokens.h" +#include "format/Config.h" +#include "format/Token.h" +#include "format/TokenTableEntry.h" #include "Store.h" const char *Format::log_tags[] = { "NONE", "TCP_HIT", "TCP_MISS", "TCP_REFRESH_UNMODIFIED", "TCP_REFRESH_FAIL", // same tag logged for LOG_TCP_REFRESH_FAIL_OLD and "TCP_REFRESH_FAIL", // LOG_TCP_REFRESH_FAIL_ERR for backward-compatibility "TCP_REFRESH_MODIFIED", "TCP_CLIENT_REFRESH_MISS", "TCP_IMS_HIT", "TCP_SWAPFAIL_MISS", "TCP_NEGATIVE_HIT", "TCP_MEM_HIT", "TCP_DENIED", "TCP_DENIED_REPLY", "TCP_OFFLINE_HIT", #if LOG_TCP_REDIRECTS "TCP_REDIRECT", #endif "UDP_HIT", "UDP_MISS", "UDP_DENIED", "UDP_INVALID", "UDP_MISS_NOFETCH", "ICP_QUERY", "LOG_TYPE_MAX" }; // Due to token overlaps between 1 and 2 letter tokens (Bug 3310) // We split the token table into sets determined by the token length namespace Format { /// 1-char tokens. -static struct TokenTableEntry TokenTable1C[] = { +static TokenTableEntry TokenTable1C[] = { {">a", LFT_CLIENT_IP_ADDRESS}, {">p", LFT_CLIENT_PORT}, {">A", LFT_CLIENT_FQDN}, {"h", LFT_REQUEST_HEADER}, {">h", LFT_REQUEST_ALL_HEADERS}, {"v", LFT_REQUEST_VERSION_OLD_2X}, {"%", LFT_PERCENT}, {NULL, LFT_NONE} /* this must be last */ }; /// 2-char tokens -static struct TokenTableEntry TokenTable2C[] = { +static TokenTableEntry TokenTable2C[] = { {">la", LFT_CLIENT_LOCAL_IP}, {"la", LFT_LOCAL_LISTENING_IP}, {">lp", LFT_CLIENT_LOCAL_PORT}, {"lp", LFT_LOCAL_LISTENING_PORT}, /*{ "lA", LFT_LOCAL_NAME }, */ {"sh", LFT_REQUEST_SIZE_HEADERS }, /*{ ">sb", LFT_REQUEST_SIZE_BODY }, */ /*{ ">sB", LFT_REQUEST_SIZE_BODY_NO_TE }, */ {"2 byte tokens +static TokenTableEntry TokenTableMisc[] = { + {">eui", LFT_CLIENT_EUI}, + {"err_code", LFT_SQUID_ERROR }, + {"err_detail", LFT_SQUID_ERROR_DETAIL }, + {NULL, LFT_NONE} /* this must be last */ +}; + #if USE_ADAPTATION -/// Adaptation (adapt::) tokens -static struct TokenTableEntry TokenTableAdapt[] = { - {"all_trs", LTF_ADAPTATION_ALL_XACT_TIMES}, - {"sum_trs", LTF_ADAPTATION_SUM_XACT_TIMES}, +static TokenTableEntry TokenTableAdapt[] = { + {"all_trs", LFT_ADAPTATION_ALL_XACT_TIMES}, + {"sum_trs", LFT_ADAPTATION_SUM_XACT_TIMES}, {"st", LFT_ICAP_BYTES_SENT}, - {"st", LFT_ICAP_BYTES_SENT}, + {"h", LFT_ICAP_REQ_HEADER}, {"2 byte tokens -static struct TokenTableEntry TokenTableMisc[] = { - {">eui", LFT_CLIENT_EUI}, - {"err_code", LFT_SQUID_ERROR }, - {"err_detail", LFT_SQUID_ERROR_DETAIL }, - {NULL, LFT_NONE} /* this must be last */ -}; - } // namespace Format +/// Register all components custom format tokens +void +Format::Token::Init() +{ + // TODO standard log tokens + // TODO external ACL fmt tokens + +#if USE_ADAPTATION + TheConfig.registerTokens(String("adapt"),TokenTableAdapt); +#endif +#if ICAP_CLIENT + TheConfig.registerTokens(String("icap"),TokenTableIcap); +#endif + + // TODO tokens for OpenSSL errors in "ssl::" +} + /// Scans a token table to see if the next token exists there /// returns a pointer to next unparsed byte and updates type member if found char * -Format::Token::scanForToken(const struct TokenTableEntry *table, char *cur) +Format::Token::scanForToken(TokenTableEntry const table[], char *cur) { - for (const struct TokenTableEntry *lte = table; lte->config != NULL; lte++) { - if (strncmp(lte->config, cur, strlen(lte->config)) == 0) { - type = lte->token_type; - label = lte->config; + for (TokenTableEntry const *lte = table; lte->configTag != NULL; lte++) { + if (strncmp(lte->configTag, cur, strlen(lte->configTag)) == 0) { + type = lte->tokenType; + label = lte->configTag; debugs(46, 7, HERE << "Found token '" << label << "'"); - return cur + strlen(lte->config); + return cur + strlen(lte->configTag); } } return cur; } /* parses a single token. Returns the token length in characters, * and fills in the lt item with the token information. * def is for sure null-terminated */ int Format::Token::parse(char *def, Quoting *quoting) { char *cur = def; int l; l = strcspn(cur, "%"); if (l > 0) { char *cp; @@ -305,60 +323,54 @@ widthMin = strtol(cur, &cur, 10); if (*cur == '.' && xisdigit(*(++cur))) widthMax = strtol(cur, &cur, 10); if (*cur == '{') { char *cp; cur++; l = strcspn(cur, "}"); cp = (char *)xmalloc(l + 1); xstrncpy(cp, cur, l + 1); data.string = cp; cur += l; if (*cur == '}') cur++; } type = LFT_NONE; - // Scan each token namespace - if (strncmp(cur, "icap::", 6) == 0) { -#if ICAP_CLIENT - cur += 6; - debugs(46, 5, HERE << "scan for icap:: token"); - cur = scanForToken(TokenTableIcap, cur); -#else - debugs(46, DBG_IMPORTANT, "ERROR: Format uses icap:: token. ICAP disabled!"); -#endif - } else if (strncmp(cur, "adapt::", 7) == 0) { -#if USE_ADAPTATION - cur += 7; - debugs(46, 5, HERE << "scan for adapt:: token"); - cur = scanForToken(TokenTableAdapt, cur); -#else - debugs(46, DBG_IMPORTANT, "ERROR: Format uses adapt:: token. Adaptation disabled!"); -#endif - } else { + // Scan each registered token namespace + for (std::list::const_iterator itr = TheConfig.tokens.begin(); itr != TheConfig.tokens.end(); itr++) { + const size_t len = itr->prefix.size(); + if (itr->prefix.cmp(cur, len) != 0 && cur[len+1] == ':' && cur[len+2] == ':') { + const char *old = cur; + cur = scanForToken(itr->tokenSet, cur); + if (old != cur) // found + break; + } + } + + if (type == LFT_NONE) { // For upward compatibility, assume "http::" prefix as default prefix - // for all log access formating codes, except those starting with a + // for all log access formatting codes, except those starting with a // "%" or a known namespace. (ie "icap::", "adapt::") if (strncmp(cur,"http::", 6) == 0 && *(cur+6) != '%' ) cur += 6; // NP: scan the sets of tokens in decreasing size to guarantee no // mistakes made with overlapping names. (Bug 3310) // Scan for various long tokens debugs(46, 5, HERE << "scan for possible Misc token"); cur = scanForToken(TokenTableMisc, cur); // scan for 2-char tokens if (type == LFT_NONE) { debugs(46, 5, HERE << "scan for possible 2C token"); cur = scanForToken(TokenTable2C, cur); } // finally scan for 1-char tokens. if (type == LFT_NONE) { debugs(46, 5, HERE << "scan for possible 1C token"); cur = scanForToken(TokenTable1C, cur); } === renamed file 'src/format/Tokens.h' => 'src/format/Token.h' --- src/format/Tokens.h 2011-10-13 17:05:25 +0000 +++ src/format/Token.h 2011-11-16 11:22:50 +0000 @@ -1,235 +1,82 @@ -#ifndef _SQUID_FMT_TOKENS_H -#define _SQUID_FMT_TOKENS_H +#ifndef _SQUID_FORMAT_TOKEN_H +#define _SQUID_FORMAT_TOKEN_H + +//#include "format/TokenTableEntry.h" +#include "format/ByteCode.h" /* * Squid configuration allows users to define custom formats in * several components. * - logging * - external ACL input * - deny page URL * * These enumerations and classes define the API for parsing of * format directives to define these patterns. Along with output * functionality to produce formatted buffers. */ namespace Format { -#define LOG_BUF_SZ (MAX_URL<<2) - -/* - * Bytecodes for the configureable format stuff - */ -typedef enum { - LFT_NONE, /* dummy */ - LFT_STRING, - - LFT_CLIENT_IP_ADDRESS, - LFT_CLIENT_FQDN, - LFT_CLIENT_PORT, - LFT_CLIENT_EUI, - - LFT_SERVER_IP_ADDRESS, - LFT_SERVER_FQDN_OR_PEER_NAME, - LFT_SERVER_PORT, - - LFT_CLIENT_LOCAL_IP, - LFT_LOCAL_LISTENING_IP, - LFT_CLIENT_LOCAL_PORT, - LFT_LOCAL_LISTENING_PORT, - /*LFT_LOCAL_NAME, */ - - LFT_SERVER_LOCAL_IP, - LFT_SERVER_LOCAL_IP_OLD_27, - LFT_SERVER_LOCAL_PORT, - - LFT_TIME_SECONDS_SINCE_EPOCH, - LFT_TIME_SUBSECOND, - LFT_TIME_LOCALTIME, - LFT_TIME_GMT, - LFT_TIME_TO_HANDLE_REQUEST, - - LFT_PEER_RESPONSE_TIME, - LFT_TOTAL_SERVER_SIDE_RESPONSE_TIME, - LFT_DNS_WAIT_TIME, - - LFT_REQUEST_HEADER, - LFT_REQUEST_HEADER_ELEM, - LFT_REQUEST_ALL_HEADERS, - - LFT_ADAPTED_REQUEST_HEADER, - LFT_ADAPTED_REQUEST_HEADER_ELEM, - LFT_ADAPTED_REQUEST_ALL_HEADERS, - - LFT_REPLY_HEADER, - LFT_REPLY_HEADER_ELEM, - LFT_REPLY_ALL_HEADERS, - - LFT_USER_NAME, - LFT_USER_LOGIN, - LFT_USER_IDENT, - /*LFT_USER_REALM, */ - /*LFT_USER_SCHEME, */ - LFT_USER_EXTERNAL, - - LFT_HTTP_SENT_STATUS_CODE_OLD_30, - LFT_HTTP_SENT_STATUS_CODE, - LFT_HTTP_RECEIVED_STATUS_CODE, - /*LFT_HTTP_STATUS, */ - LFT_HTTP_BODY_BYTES_READ, - - LFT_SQUID_STATUS, - LFT_SQUID_ERROR, - LFT_SQUID_ERROR_DETAIL, - LFT_SQUID_HIERARCHY, - - LFT_MIME_TYPE, - - /* original Request-Line details receved from client */ - LFT_CLIENT_REQ_METHOD, - LFT_CLIENT_REQ_URI, - LFT_CLIENT_REQ_URLPATH, - /* LFT_CLIENT_REQ_QUERY, */ - LFT_CLIENT_REQ_VERSION, - - /* Request-Line details receved from client (legacy, filtered) */ - LFT_REQUEST_METHOD, - LFT_REQUEST_URI, - LFT_REQUEST_URLPATH_OLD_31, - /*LFT_REQUEST_QUERY, */ - LFT_REQUEST_VERSION_OLD_2X, - LFT_REQUEST_VERSION, - - /* Request-Line details sent to the server/peer */ - LFT_SERVER_REQ_METHOD, - LFT_SERVER_REQ_URI, - LFT_SERVER_REQ_URLPATH, - /*LFT_SERVER_REQ_QUERY, */ - LFT_SERVER_REQ_VERSION, - - LFT_REQUEST_SIZE_TOTAL, - /*LFT_REQUEST_SIZE_LINE, */ - LFT_REQUEST_SIZE_HEADERS, - /*LFT_REQUEST_SIZE_BODY, */ - /*LFT_REQUEST_SIZE_BODY_NO_TE, */ - - LFT_REPLY_SIZE_TOTAL, - LFT_REPLY_HIGHOFFSET, - LFT_REPLY_OBJECTSIZE, - /*LFT_REPLY_SIZE_LINE, */ - LFT_REPLY_SIZE_HEADERS, - /*LFT_REPLY_SIZE_BODY, */ - /*LFT_REPLY_SIZE_BODY_NO_TE, */ - - LFT_TAG, - LFT_IO_SIZE_TOTAL, - LFT_EXT_LOG, - - LFT_SEQUENCE_NUMBER, - -#if USE_ADAPTATION - LTF_ADAPTATION_SUM_XACT_TIMES, - LTF_ADAPTATION_ALL_XACT_TIMES, - LFT_ADAPTATION_LAST_HEADER, - LFT_ADAPTATION_LAST_HEADER_ELEM, - LFT_ADAPTATION_LAST_ALL_HEADERS, -#endif - -#if ICAP_CLIENT - - LFT_ICAP_TOTAL_TIME, - - LFT_ICAP_ADDR, - LFT_ICAP_SERV_NAME, - LFT_ICAP_REQUEST_URI, - LFT_ICAP_REQUEST_METHOD, - LFT_ICAP_BYTES_SENT, - LFT_ICAP_BYTES_READ, - LFT_ICAP_BODY_BYTES_READ, - - LFT_ICAP_REQ_HEADER, - LFT_ICAP_REQ_HEADER_ELEM, - LFT_ICAP_REQ_ALL_HEADERS, - - LFT_ICAP_REP_HEADER, - LFT_ICAP_REP_HEADER_ELEM, - LFT_ICAP_REP_ALL_HEADERS, - - LFT_ICAP_TR_RESPONSE_TIME, - LFT_ICAP_IO_TIME, - LFT_ICAP_OUTCOME, - LFT_ICAP_STATUS_CODE, -#endif - - LFT_PERCENT /* special string cases for escaped chars */ -} ByteCode_t; - -/// Quoting style for a format output. -enum Quoting { - LOG_QUOTE_NONE = 0, - LOG_QUOTE_QUOTES, - LOG_QUOTE_MIMEBLOB, - LOG_QUOTE_URL, - LOG_QUOTE_RAW -}; +class TokenTableEntry; -struct TokenTableEntry { - const char *config; - ByteCode_t token_type; - int options; -}; +#define LOG_BUF_SZ (MAX_URL<<2) // XXX: inherit from linked list class Token { public: Token() : type(LFT_NONE), label(NULL), widthMin(-1), widthMax(-1), quote(LOG_QUOTE_NONE), left(0), space(0), zero(0), divisor(0), next(NULL) - { data.string = NULL; }; + { data.string = NULL; } + ~Token(); + /// Initialize the format token registrations + static void Init(); + /** parses a single token. Returns the token length in characters, * and fills in this item with the token information. * def is for sure null-terminated. */ int parse(char *def, enum Quoting *quote); ByteCode_t type; const char *label; union { char *string; struct { char *header; char *element; char separator; } header; char *timespec; } data; int widthMin; ///< minimum field width int widthMax; ///< maximum field width enum Quoting quote; unsigned int left:1; unsigned int space:1; unsigned int zero:1; int divisor; Token *next; /* todo: move from linked list to array */ private: - char *scanForToken(const struct TokenTableEntry *table, char *cur); + char *scanForToken(TokenTableEntry const table[], char *cur); }; extern const char *log_tags[]; } // namespace Format -#endif /* _SQUID_FMT_TOKENS_H */ +#endif /* _SQUID_FORMAT_TOKEN_H */ === added file 'src/format/TokenTableEntry.h' --- src/format/TokenTableEntry.h 1970-01-01 00:00:00 +0000 +++ src/format/TokenTableEntry.h 2011-11-15 11:33:09 +0000 @@ -0,0 +1,37 @@ +#ifndef _SQUID_FORMAT_TOKENTABLEENTRY_H +#define _SQUID_FORMAT_TOKENTABLEENTRY_H + +#include "format/ByteCode.h" + +/* + * Squid configuration allows users to define custom formats in + * several components. + * - logging + * - external ACL input + * - deny page URL + * + * These enumerations and classes define the API for parsing of + * format directives to define these patterns. Along with output + * functionality to produce formatted buffers. + */ + +namespace Format +{ + +/// One entry in a table of format tokens. +class TokenTableEntry { +public: + /// the config file ASCII representation for this token + /// just the base tag bytes, excluding any option syntax bytes + const char *configTag; + + /// the internal byte code representatio of this token + ByteCode_t tokenType; + + /// 32-bit mask? of options affecting the output display of this token + uint32_t options; +}; + +} // namespace Format + +#endif /* _SQUID_FORMAT_TOKENTABLEENTRY_H */ === modified file 'src/log/FormatHttpdCombined.cc' --- src/log/FormatHttpdCombined.cc 2011-08-04 03:21:06 +0000 +++ src/log/FormatHttpdCombined.cc 2011-09-11 06:30:44 +0000 @@ -17,41 +17,41 @@ * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "config.h" #include "AccessLogEntry.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "format/Quoting.h" #include "HttpRequest.h" #include "log/File.h" #include "log/Formats.h" #include "SquidTime.h" void Log::Format::HttpdCombined(AccessLogEntry * al, Logfile * logfile) { char clientip[MAX_IPSTRLEN]; const char *user_ident = ::Format::QuoteUrlEncodeUsername(al->cache.rfc931); const char *user_auth = ::Format::QuoteUrlEncodeUsername(al->cache.authuser); const char *referer = al->request->header.getStr(HDR_REFERER); if (!referer || *referer == '\0') referer = "-"; const char *agent = al->request->header.getStr(HDR_USER_AGENT); === modified file 'src/log/FormatHttpdCommon.cc' --- src/log/FormatHttpdCommon.cc 2011-08-04 03:21:06 +0000 +++ src/log/FormatHttpdCommon.cc 2011-09-11 06:31:17 +0000 @@ -18,41 +18,41 @@ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "config.h" #include "AccessLogEntry.h" #include "format/Quoting.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "log/File.h" #include "log/Formats.h" #include "SquidTime.h" void Log::Format::HttpdCommon(AccessLogEntry * al, Logfile * logfile) { char clientip[MAX_IPSTRLEN]; const char *user_auth = ::Format::QuoteUrlEncodeUsername(al->cache.authuser); const char *user_ident = ::Format::QuoteUrlEncodeUsername(al->cache.rfc931); logfilePrintf(logfile, "%s %s %s [%s] \"%s %s %s/%d.%d\" %d %"PRId64" %s%s:%s%s", al->cache.caddr.NtoA(clientip,MAX_IPSTRLEN), user_ident ? user_ident : dash_str, user_auth ? user_auth : dash_str, Time::FormatHttpd(squid_curtime), al->_private.method_str, al->url, AnyP::ProtocolType_str[al->http.version.protocol], al->http.version.major, al->http.version.minor, === modified file 'src/log/FormatSquidCustom.cc' --- src/log/FormatSquidCustom.cc 2011-08-04 03:21:06 +0000 +++ src/log/FormatSquidCustom.cc 2011-09-11 06:34:32 +0000 @@ -17,36 +17,35 @@ * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "config.h" #include "AccessLogEntry.h" -#include "format/Tokens.h" #include "log/File.h" #include "log/Formats.h" #include "MemBuf.h" void Log::Format::SquidCustom(AccessLogEntry * al, customlog * log) { static MemBuf mb; mb.reset(); // XXX: because we do not yet have a neutral form of transaction slab. use AccessLogEntry log->logFormat->assemble(mb, al, log->logfile->sequence_number); logfilePrintf(log->logfile, "%s\n", mb.buf); } === modified file 'src/log/FormatSquidNative.cc' --- src/log/FormatSquidNative.cc 2011-08-20 15:57:06 +0000 +++ src/log/FormatSquidNative.cc 2011-09-11 06:35:16 +0000 @@ -18,41 +18,41 @@ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "config.h" #include "AccessLogEntry.h" #include "format/Quoting.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "log/File.h" #include "log/Formats.h" #include "SquidTime.h" void Log::Format::SquidNative(AccessLogEntry * al, Logfile * logfile) { char hierHost[MAX_IPSTRLEN]; const char *user = ::Format::QuoteUrlEncodeUsername(al->cache.authuser); if (!user) user = ::Format::QuoteUrlEncodeUsername(al->cache.extuser); #if USE_SSL if (!user) user = ::Format::QuoteUrlEncodeUsername(al->cache.ssluser); #endif if (!user) === modified file 'src/log/access_log.cc' --- src/log/access_log.cc 2011-09-09 20:41:40 +0000 +++ src/log/access_log.cc 2011-09-22 11:32:19 +0000 @@ -30,41 +30,41 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "squid.h" #include "AccessLogEntry.h" // Store.h Required by configuration directives parsing/dumping only #include "Store.h" #include "errorpage.h" #include "err_detail_type.h" #include "acl/Checklist.h" #include "errorpage.h" #if USE_SQUID_EUI #include "eui/Eui48.h" #include "eui/Eui64.h" #endif -#include "format/Tokens.h" +#include "format/Token.h" #include "hier_code.h" #include "HttpReply.h" #include "HttpRequest.h" #include "log/Config.h" #include "log/File.h" #include "log/Formats.h" #include "MemBuf.h" #include "mgr/Registration.h" #include "rfc1738.h" #include "SquidTime.h" #if HEADERS_LOG static Logfile *headerslog = NULL; #endif #if MULTICAST_MISS_STREAM static int mcast_miss_fd = -1; static struct sockaddr_in mcast_miss_to; static void mcast_encode(unsigned int *, size_t, const unsigned int *); @@ -305,42 +305,42 @@ accessLogRegisterWithCacheManager(); #if USE_ADAPTATION Log::TheConfig.hasAdaptToken = false; #endif #if ICAP_CLIENT Log::TheConfig.hasIcapToken = false; #endif for (log = Config.Log.accesslogs; log; log = log->next) { if (log->type == Log::Format::CLF_NONE) continue; log->logfile = logfileOpen(log->filename, MAX_URL << 2, 1); LogfileStatus = LOG_ENABLE; #if USE_ADAPTATION for (Format::Token * curr_token = (log->logFormat?log->logFormat->format:NULL); curr_token; curr_token = curr_token->next) { - if (curr_token->type == Format::LTF_ADAPTATION_SUM_XACT_TIMES || - curr_token->type == Format::LTF_ADAPTATION_ALL_XACT_TIMES || + if (curr_token->type == Format::LFT_ADAPTATION_SUM_XACT_TIMES || + curr_token->type == Format::LFT_ADAPTATION_ALL_XACT_TIMES || curr_token->type == Format::LFT_ADAPTATION_LAST_HEADER || curr_token->type == Format::LFT_ADAPTATION_LAST_HEADER_ELEM || curr_token->type == Format::LFT_ADAPTATION_LAST_ALL_HEADERS) { Log::TheConfig.hasAdaptToken = true; } #if ICAP_CLIENT if (curr_token->type == Format::LFT_ICAP_TOTAL_TIME) { Log::TheConfig.hasIcapToken = true; } #endif } #endif } #if HEADERS_LOG headerslog = logfileOpen("/usr/local/squid/logs/headers.log", 512); assert(NULL != headerslog); === modified file 'src/main.cc' --- src/main.cc 2011-11-03 10:02:02 +0000 +++ src/main.cc 2011-11-05 09:41:52 +0000 @@ -46,40 +46,41 @@ #endif #if USE_AUTH #include "auth/Gadgets.h" #endif #include "base/RunnersRegistry.h" #include "base/Subscription.h" #include "base/TextException.h" #if USE_DELAY_POOLS #include "ClientDelayConfig.h" #endif #include "comm.h" #include "ConfigParser.h" #include "CpuAffinity.h" #if USE_DELAY_POOLS #include "DelayPools.h" #endif #include "errorpage.h" #include "event.h" #include "EventLoop.h" #include "ExternalACL.h" +#include "format/Token.h" #include "fs/Module.h" #include "PeerSelectState.h" #include "Store.h" #include "ICP.h" #include "ident/Ident.h" #include "HttpReply.h" #include "pconn.h" #include "Mem.h" #include "acl/Asn.h" #include "acl/Acl.h" #include "htcp.h" #include "StoreFileSystem.h" #include "DiskIO/DiskIOModule.h" #include "ipc/Kids.h" #include "ipc/Coordinator.h" #include "ipc/Strand.h" #include "ip/tools.h" #include "SquidTime.h" #include "SwapDir.h" #include "forward.h" @@ -1088,40 +1089,42 @@ do_mallinfo = 1; mimeInit(Config.mimeTablePathname); refreshInit(); #if USE_DELAY_POOLS DelayPools::Init(); #endif FwdState::initModule(); /* register the modules in the cache manager menus */ cbdataRegisterWithCacheManager(); /* These use separate calls so that the comm loops can eventually * coexist. */ eventInit(); // TODO: pconn is a good candidate for new-style registration // PconnModule::GetInstance()->registerWithCacheManager(); // moved to PconnModule::PconnModule() + + Format::Token::Init(); } if (IamPrimaryProcess()) { #if USE_WCCP wccpInit(); #endif #if USE_WCCPv2 wccp2Init(); #endif } serverConnectionsOpen(); neighbors_init(); // neighborsRegisterWithCacheManager(); //moved to neighbors_init() === modified file 'src/stat.cc' --- src/stat.cc 2011-10-14 16:21:48 +0000 +++ src/stat.cc 2011-10-16 05:00:24 +0000 @@ -17,41 +17,41 @@ * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "squid.h" #include "event.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "StoreClient.h" #if USE_AUTH #include "auth/UserRequest.h" #endif #include "comm/Connection.h" #include "mgr/Registration.h" #include "Store.h" #include "HttpRequest.h" #include "MemObject.h" #include "fde.h" #include "mem_node.h" #if USE_DELAY_POOLS #include "DelayId.h" #endif #include "client_side_request.h" #include "client_side.h" #include "MemBuf.h" #include "SquidMath.h" #include "SquidTime.h" #include "mgr/CountersAction.h" === modified file 'src/store_log.cc' --- src/store_log.cc 2011-08-04 03:21:06 +0000 +++ src/store_log.cc 2011-09-11 06:54:54 +0000 @@ -16,41 +16,41 @@ * incorporates software developed and/or copyrighted by other * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "squid.h" -#include "format/Tokens.h" +#include "format/Token.h" #include "HttpReply.h" #include "log/File.h" #include "MemObject.h" #include "mgr/Registration.h" #include "Store.h" #include "SquidTime.h" static const char *storeLogTags[] = { "CREATE", "SWAPIN", "SWAPOUT", "RELEASE", "SO_FAIL", }; static int storeLogTagsCounts[STORE_LOG_SWAPOUTFAIL+1]; static OBJH storeLogTagsHist; static Logfile *storelog = NULL;