=== modified file 'src/MemBlob.cc' --- src/MemBlob.cc 2010-12-03 01:56:04 +0000 +++ src/MemBlob.cc 2011-03-30 23:45:00 +0000 @@ -15,153 +15,138 @@ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. */ #include "config.h" #include "base/TextException.h" #include "Debug.h" +#include "Mem.h" #include "MemBlob.h" +#include "protos.h" + #if HAVE_IOSTREAM #include #endif -#define MEMBLOB_USES_MEM_POOLS 0 - -#if MEMBLOB_USES_MEM_POOLS -#include "protos.h" -#endif MemBlobStats MemBlob::Stats; InstanceIdDefinitions(MemBlob, "blob"); /* MemBlobStats */ MemBlobStats::MemBlobStats(): alloc(0), live(0), append(0) {} +MemBlobStats& +MemBlobStats::operator += (const MemBlobStats& s) +{ + alloc+=s.alloc; + live+=s.live; + append+=s.append; + liveBytes+=s.liveBytes; + + return *this; +} + std::ostream& MemBlobStats::dump(std::ostream &os) const { os << "MemBlob created: " << alloc << "\nMemBlob alive: " << live << "\nMemBlob append calls: " << append << "\nMemBlob currently allocated size: " << liveBytes << "\nlive MemBlob mean current allocation size: " << (static_cast(liveBytes)/(live?live:1)) << std::endl; return os; } /* MemBlob */ MemBlob::MemBlob(const MemBlob::size_type reserveSize) : mem(NULL), capacity(0), size(0) // will be set by memAlloc { debugs(MEMBLOB_DEBUGSECTION,9, HERE << "constructed, this=" << static_cast(this) << " id=" << id << " reserveSize=" << reserveSize); memAlloc(reserveSize); } MemBlob::MemBlob(const char *buffer, const MemBlob::size_type bufSize) : mem(NULL), capacity(0), size(0) // will be set by memAlloc { debugs(MEMBLOB_DEBUGSECTION,9, HERE << "constructed, this=" << static_cast(this) << " id=" << id << " buffer=" << static_cast(buffer) << " bufSize=" << bufSize); memAlloc(bufSize); append(buffer, bufSize); } MemBlob::~MemBlob() { -#if MEMBLOB_USES_MEM_POOLS - //no mempools for now - // \todo reinstate mempools use memFreeString(capacity,mem); -#else - xfree(mem); -#endif Stats.liveBytes -= capacity; --Stats.live; debugs(MEMBLOB_DEBUGSECTION,9, HERE << "destructed, this=" << static_cast(this) << " id=" << id << " capacity=" << capacity << " size=" << size); } -/** - * Given the requested minimum size, return a rounded allocation size +/** Given the requested minimum size, return a rounded allocation size * for the backing store. - * This is a stopgap call, this job is eventually expected to be handled - * by MemPools via memAllocString. */ -MemBlob::size_type +// TODO: exploit better integration with MemPools, moving much of this there. +// TODO: have MemPools do system page size magic. They don't really belong here +inline MemBlob::size_type MemBlob::calcAllocSize(const size_type sz) const { - if (sz <= 36) return 36; - if (sz <= 128) return 128; - if (sz <= 512) return 512; - if (sz <= 4096) return RoundTo(sz, 512); - // XXX: recover squidSystemPageSize functionality. It's easy for - // the main squid, harder for tests -#if 0 - return RoundTo(sz, squidSystemPageSize); -#else - return RoundTo(sz, 4096); -#endif + return sz; } /** Allocate an available space area of at least minSize bytes in size. * Must be called by constructors and only by constructors. */ void MemBlob::memAlloc(const size_type minSize) { size_t actualAlloc = calcAllocSize(minSize); Must(!mem); -#if MEMBLOB_USES_MEM_POOLS - // XXX: for now, do without mempools. In order to do it, MemPools - // need to be singletons so that initialization order can be enforced - mem = static_cast(memAllocString(minSize, &actualAlloc)); -#else - // \todo reinstate mempools use - mem = static_cast(xmalloc(actualAlloc)); -#endif + mem = static_cast(memAllocString(actualAlloc, &actualAlloc)); Must(mem); capacity = actualAlloc; size = 0; debugs(MEMBLOB_DEBUGSECTION, 8, id << " memAlloc: requested=" << minSize << ", received=" << capacity); ++Stats.live; ++Stats.alloc; Stats.liveBytes += capacity; } void MemBlob::append(const char *source, const size_type n) { if (n > 0) { // appending zero bytes is allowed but only affects the stats Must(willFit(n)); Must(source); /// \note memcpy() is safe because we copy to an unused area memcpy(mem + size, source, n); === modified file 'src/MemBlob.h' --- src/MemBlob.h 2010-10-28 15:50:51 +0000 +++ src/MemBlob.h 2011-03-30 23:45:00 +0000 @@ -29,40 +29,42 @@ */ #ifndef SQUID_MEMBLOB_H_ #define SQUID_MEMBLOB_H_ #define MEMBLOB_DEBUGSECTION 24 #include "base/InstanceId.h" #include "MemPool.h" #include "RefCount.h" /// Various MemBlob class-wide statistics. class MemBlobStats { public: MemBlobStats(); /// dumps class-wide statistics std::ostream& dump(std::ostream& os) const; + MemBlobStats& operator += (const MemBlobStats&); + public: uint64_t alloc; ///< number of MemBlob instances created so far uint64_t live; ///< number of MemBlob instances currently alive uint64_t append; ///< number of MemBlob::append() calls uint64_t liveBytes; ///< the total size of currently allocated storage }; /** Refcountable, fixed-size, content-agnostic memory buffer. * * Allocated memory block is divided into two sequential areas: * "used memory" and "available space". The used area can be filled during * construction, grows via the append() call, and can be clear()ed. * * MemBlob users can cooperate to safely share the used area. However, MemBlob * provides weak use accounting and no sharing protections besides refcounting. */ class MemBlob: public RefCountable { public: === modified file 'src/mem.cc' --- src/mem.cc 2011-03-30 16:50:18 +0000 +++ src/mem.cc 2011-03-30 23:45:00 +0000 @@ -44,66 +44,74 @@ #include "SquidTime.h" #if HAVE_IOMANIP #include #endif #if HAVE_OSTREAM #include #endif /* module globals */ const size_t squidSystemPageSize=getpagesize(); /* local prototypes */ static void memStringStats(std::ostream &); /* module locals */ static MemAllocator *MemPools[MEM_MAX]; static double xm_time = 0; static double xm_deltat = 0; +/* explicit initialization state */ +static bool MemIsInitialized=false; + /* string pools */ -#define mem_str_pool_count 3 +#define mem_str_pool_count 5 static const struct { const char *name; size_t obj_size; } - +/* note: string pools must be sorted in ascending order by size */ StrPoolsAttrs[mem_str_pool_count] = { - { "Short Strings", MemAllocator::RoundedSize(36), - }, /* to fit rfc1123 and similar */ + }, /* to fit rfc1123 and similar */ { "Medium Strings", MemAllocator::RoundedSize(128), - }, /* to fit most urls */ + }, /* to fit most urls */ { "Long Strings", MemAllocator::RoundedSize(512) - } /* other */ + }, + { + "Big Strings", MemAllocator::RoundedSize(1024) + }, + { + "Huge Strings", MemAllocator::RoundedSize(4096) + } /* other */ }; static struct { MemAllocator *pool; } - StrPools[mem_str_pool_count]; + static MemMeter StrCountMeter; static MemMeter StrVolumeMeter; static MemMeter HugeBufCountMeter; static MemMeter HugeBufVolumeMeter; /* local routines */ static void memStringStats(std::ostream &stream) { int i; int pooled_count = 0; size_t pooled_volume = 0; /* heading */ stream << "String Pool\t Impact\t\t\n \t (%strings)\t (%volume)\n"; /* table body */ for (i = 0; i < mem_str_pool_count; ++i) { const MemAllocator *pool = StrPools[i].pool; @@ -173,83 +181,86 @@ assert(MemPools[type] == NULL); MemPools[type] = memPoolCreate(name, size); MemPools[type]->zeroOnPush(zeroOnPush); } /* find appropriate pool and use it (pools always init buffer with 0s) */ void * memAllocate(mem_type type) { return MemPools[type]->alloc(); } /* give memory back to the pool */ void memFree(void *p, int type) { MemPools[type]->freeOne(p); } -/* allocate a variable size buffer using best-fit pool */ void * memAllocString(size_t net_size, size_t * gross_size) { - int i; MemAllocator *pool = NULL; assert(gross_size); + // if pools are not yet ready, make sure that + // the requested size is not poolable + if (!MemIsInitialized) + net_size=1+StrPoolsAttrs[mem_str_pool_count-1].obj_size; + + unsigned int i=0; for (i = 0; i < mem_str_pool_count; ++i) { if (net_size <= StrPoolsAttrs[i].obj_size) { pool = StrPools[i].pool; break; } } - *gross_size = pool ? StrPoolsAttrs[i].obj_size : net_size; assert(*gross_size >= net_size); memMeterInc(StrCountMeter); memMeterAdd(StrVolumeMeter, *gross_size); return pool ? pool->alloc() : xcalloc(1, net_size); } + extern size_t memStringCount(); size_t memStringCount() { size_t result = 0; for (int counter = 0; counter < mem_str_pool_count; ++counter) result += memPoolInUseCount(StrPools[counter].pool); return result; } /* free buffer allocated with memAllocString() */ void memFreeString(size_t size, void *buf) { - int i; MemAllocator *pool = NULL; - assert(size && buf); + assert(buf); - for (i = 0; i < mem_str_pool_count; ++i) { + for (unsigned int i = 0; i < mem_str_pool_count; ++i) { if (size <= StrPoolsAttrs[i].obj_size) { assert(size == StrPoolsAttrs[i].obj_size); pool = StrPools[i].pool; break; } } memMeterDec(StrCountMeter); memMeterDel(StrVolumeMeter, size); pool ? pool->freeOne(buf) : xfree(buf); } /* Find the best fit MEM_X_BUF type */ static mem_type memFindBufSizeType(size_t net_size, size_t * gross_size) { mem_type type; size_t size; if (net_size <= 2 * 1024) { @@ -318,41 +329,41 @@ *gross_size = new_gross_size; return newbuf; } /* free buffer allocated with memAllocBuf() */ void memFreeBuf(size_t size, void *buf) { mem_type type = memFindBufSizeType(size, NULL); if (type != MEM_NONE) memFree(buf, type); else { xfree(buf); memMeterDec(HugeBufCountMeter); memMeterDel(HugeBufVolumeMeter, size); } } -static double clean_interval = 15.0; /* time to live of idle chunk before release */ +static double clean_interval = 15.0; /* time to live of idle chunk before release */ void Mem::CleanIdlePools(void *unused) { MemPools::GetInstance().clean(static_cast(clean_interval)); eventAdd("memPoolCleanIdlePools", CleanIdlePools, NULL, clean_interval, 1); } void memConfigure(void) { int64_t new_pool_limit; /** Set to configured value first */ if (!Config.onoff.mem_pools) new_pool_limit = 0; else if (Config.MemPools.limit > 0) new_pool_limit = Config.MemPools.limit; else { if (Config.MemPools.limit == 0) @@ -421,40 +432,41 @@ memDataInit(MEM_DREAD_CTRL, "dread_ctrl", sizeof(dread_ctrl), 0); memDataInit(MEM_DWRITE_Q, "dwrite_q", sizeof(dwrite_q), 0); memDataInit(MEM_HTTP_HDR_CC, "HttpHdrCc", sizeof(HttpHdrCc), 0); memDataInit(MEM_HTTP_HDR_CONTENT_RANGE, "HttpHdrContRange", sizeof(HttpHdrContRange), 0); memDataInit(MEM_NETDBENTRY, "netdbEntry", sizeof(netdbEntry), 0); memDataInit(MEM_NET_DB_NAME, "net_db_name", sizeof(net_db_name), 0); memDataInit(MEM_RELIST, "relist", sizeof(relist), 0); memDataInit(MEM_CLIENT_INFO, "ClientInfo", sizeof(ClientInfo), 0); memDataInit(MEM_MD5_DIGEST, "MD5 digest", SQUID_MD5_DIGEST_LENGTH, 0); MemPools[MEM_MD5_DIGEST]->setChunkSize(512 * 1024); /** Lastly init the string pools. */ for (i = 0; i < mem_str_pool_count; ++i) { StrPools[i].pool = memPoolCreate(StrPoolsAttrs[i].name, StrPoolsAttrs[i].obj_size); StrPools[i].pool->zeroOnPush(false); if (StrPools[i].pool->objectSize() != StrPoolsAttrs[i].obj_size) debugs(13, 1, "Notice: " << StrPoolsAttrs[i].name << " is " << StrPools[i].pool->objectSize() << " bytes instead of requested " << StrPoolsAttrs[i].obj_size << " bytes"); } + MemIsInitialized=true; /** \par * finally register with the cache manager */ RegisterWithCacheManager(); } void Mem::Report() { debugs(13, 3, "Memory pools are '" << (Config.onoff.mem_pools ? "on" : "off") << "'; limit: " << std::setprecision(3) << toMB(MemPools::GetInstance().idleLimit()) << " MB"); } void Mem::RegisterWithCacheManager(void) { Mgr::RegisterAction("mem", "Memory Utilization", Mem::Stats, 0, 1); } @@ -712,45 +724,45 @@ "(#)\t (KB)\t high (KB)\t high (hrs)\t %alloc\t" "(#)\t (KB)\t high (KB)\t" "(#)\t %cnt\t %vol\t" "(#)/sec\t" "\n"; xm_deltat = current_dtime - xm_time; xm_time = current_dtime; /* Get stats for Totals report line */ memPoolGetGlobalStats(&mp_total); MemPoolStats *sortme = (MemPoolStats *) xcalloc(mp_total.tot_pools_alloc ,sizeof(*sortme)); int npools = 0; /* main table */ iter = memPoolIterate(); while ((pool = memPoolIterateNext(iter))) { pool->getStats(&mp_stats); - if (!mp_stats.pool) /* pool destroyed */ + if (!mp_stats.pool) /* pool destroyed */ continue; - if (mp_stats.pool->getMeter().gb_allocated.count > 0) /* this pool has been used */ - sortme[npools++] = mp_stats; + if (mp_stats.pool->getMeter().gb_allocated.count > 0) /* this pool has been used */ + sortme[++npools] = mp_stats; else ++not_used; } memPoolIterateDone(&iter); qsort(sortme, npools, sizeof(*sortme), MemPoolReportSorter); for (int i = 0; i< npools; ++i) { PoolReport(&sortme[i], mp_total.TheMeter, stream); } xfree(sortme); mp_stats.pool = NULL; mp_stats.label = "Total"; mp_stats.meter = mp_total.TheMeter; mp_stats.obj_size = 1; mp_stats.chunk_capacity = 0; mp_stats.chunk_size = 0;