=== modified file 'src/parser/Tokenizer.cc'
--- src/parser/Tokenizer.cc	2014-06-02 17:52:31 +0000
+++ src/parser/Tokenizer.cc	2014-06-02 18:59:23 +0000
@@ -29,12 +29,12 @@
 #endif
 
 bool
-Parser::Tokenizer::token(SBuf &returnedToken, const CharacterSet &delimiters)
+Parser::Tokenizer::token(SBuf &returnedToken, const CharacterSet &delimiters, const bool requireComplete)
 {
     const SBuf savebuf(buf_);
     skip(delimiters);
     const SBuf::size_type tokenLen = buf_.findFirstOf(delimiters); // not found = npos => consume to end
-    if (tokenLen == SBuf::npos && !delimiters['\0']) {
+    if (tokenLen == SBuf::npos && requireComplete) {
         // no delimiter found, nor is NUL/EOS/npos acceptible as one
         buf_ = savebuf;
         return false;

=== modified file 'src/parser/Tokenizer.h'
--- src/parser/Tokenizer.h	2014-06-02 00:15:10 +0000
+++ src/parser/Tokenizer.h	2014-06-02 18:59:24 +0000
@@ -42,12 +42,16 @@
      *
      *  Want to extract delimiters? Use prefix() instead.
      *
-     * At least one terminating delimiter is required. \0 may be passed
-     * as a delimiter to treat end of buffer content as the end of token.
-     *
+     * \param returnedToken output parameter, returns the token if tokenization
+     *        was successful; untouched otherwise
+     * \param delimiters the delimiters to be found (optionally before) and
+     *        after the token.
+     * \param requreComplete if true, requires that a delimiter be found after
+     *        the token for a successful parsing. If false or not provided,
+     *        end of parse buffer will be considered as an implicit delimiter
      * \return false if no terminal delimiter is found.
      */
-    bool token(SBuf &returnedToken, const CharacterSet &delimiters);
+    bool token(SBuf &returnedToken, const CharacterSet &delimiters, const bool requireComplete = false);
 
     /** Accumulates all sequential permitted characters up to an optional length limit.
      *

=== modified file 'src/parser/testTokenizer.cc'
--- src/parser/testTokenizer.cc	2014-06-02 00:15:10 +0000
+++ src/parser/testTokenizer.cc	2014-06-02 19:28:27 +0000
@@ -82,19 +82,45 @@
 void
 testTokenizer::testTokenizerToken()
 {
-    Parser::Tokenizer t(text);
-    SBuf s;
-
-    // first scenario: patterns match
-    CPPUNIT_ASSERT(t.token(s,whitespace));
-    CPPUNIT_ASSERT_EQUAL(SBuf("GET"),s);
-    CPPUNIT_ASSERT(t.token(s,whitespace));
-    CPPUNIT_ASSERT_EQUAL(SBuf("http://resource.com/path"),s);
-    CPPUNIT_ASSERT(t.token(s,whitespace));
-    CPPUNIT_ASSERT_EQUAL(SBuf("HTTP/1.1"),s);
-    CPPUNIT_ASSERT(t.token(s,whitespace));
-    CPPUNIT_ASSERT_EQUAL(SBuf("Host:"),s);
-
+    {
+        // match
+        Parser::Tokenizer t(text);
+        SBuf s;
+
+        // first scenario: patterns match
+        CPPUNIT_ASSERT(t.token(s,whitespace));
+        CPPUNIT_ASSERT_EQUAL(SBuf("GET"),s);
+        CPPUNIT_ASSERT(t.token(s,whitespace));
+        CPPUNIT_ASSERT_EQUAL(SBuf("http://resource.com/path"),s);
+        CPPUNIT_ASSERT(t.token(s,whitespace));
+        CPPUNIT_ASSERT_EQUAL(SBuf("HTTP/1.1"),s);
+        CPPUNIT_ASSERT(t.token(s,whitespace));
+        CPPUNIT_ASSERT_EQUAL(SBuf("Host:"),s);
+    }
+    {
+        // complete / incomplete tokens
+        SBuf token;
+        Parser::Tokenizer t(token);
+
+        t.reset(SBuf("  foo  "));
+        CPPUNIT_ASSERT(t.token(token,whitespace));
+        CPPUNIT_ASSERT_EQUAL(SBuf("foo"),token);
+        token.clear();
+
+        t.reset(SBuf("  foo  "));
+        CPPUNIT_ASSERT(t.token(token,whitespace,true));
+        CPPUNIT_ASSERT_EQUAL(SBuf("foo"),token);
+        token.clear();
+
+        t.reset(SBuf("  foo"));
+        CPPUNIT_ASSERT(t.token(token,whitespace));
+        CPPUNIT_ASSERT_EQUAL(SBuf("foo"),token);
+        token.clear();
+
+        t.reset(SBuf("  foo"));
+        CPPUNIT_ASSERT(!t.token(token,whitespace,true));
+        CPPUNIT_ASSERT_EQUAL(SBuf(""),token);
+    }
 }
 
 void