Keep track of the caller bestenc flags that make sense.

 2000-11-08  Not Zed  <NotZed@HelixCode.com>

	* camel-mime-message.c (find_best_encoding): Keep track of the
	caller bestenc flags that make sense.

	* camel-mime-filter-bestenc.c (filter): Added code to detect when
	we have "^From " lines in the sequence of text.
	(camel_mime_filter_bestenc_get_best_encoding): Added a new flag
	CAMEL_BESTENC_NO_FROM: if set, it will not allow any lines
	matching "^From " to appear in the output - currently forcing
	base64 encoding to achieve this.

	* camel-mime-parser.c (folder_scan_step): Call
	camel_mime-filter_complete() once we're done, rather than
	filter_filter().
	(folder_scan_content): Some fixes for state changing; so that when
	we do find another boundary it is properly flagged.  Since we
	strip the last \n off all data, we must take that into account
	too.  Sigh.  Fixes a rather nasty set of bugs where multipart
	messages could start including following messages as parts, etc.
	(struct _header_scan_stack): Added new parameter,
	boundarylenfinal, which holds the length of the final boundary, if
	it is different (e.g. for From lines, whihc aren't)
	(folder_scan_step): Setup teh boundarylenfinal value when creating
	a new boundary.
	(folder_scan_content): Hmm, if we hit the end-of-buffer sentinal,
	reset the scanner back to leave 'atleast' chars in the buffer
	still, dump that content, and retry again.  Stops us losing a
	check for a boundary on some data we haven't really looked at yet!
	(folder_scan_content): Use boundarylenfinal to calculate
	'atleast'.
	(folder_scan_header): And here too.
	(folder_boundary_check): Use the atleast value directly, dont
	truncate it.  Use the boundarylen/boundarylenfinal values directly
	too.
	(struct _header_scan_stack): Add an atleast parameter to cache the
	atleast info.
	(folder_push_part): Determine/set 'atleast', every time we add a
	new part.
	(folder_scan_header): Get the cached atleast info from the current
	part.
	(folder_scan_content): And here too.
	(folder_scan_header): Fix a problem where a part starting with
	" text" would be interpreted as a followon header wrongly.

	* camel-mime-filter-charset.c (complete): Add some assertions to
	find a bug.

svn path=/trunk/; revision=6500

7 files changed, 237 insertions, 137 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index f3e7b49d29..0caf7e1aab 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -1,3 +1,51 @@
+2000-11-08  Not Zed  <NotZed@HelixCode.com>
+
+	* camel-mime-message.c (find_best_encoding): Keep track of the
+	caller bestenc flags that make sense.
+	
+	* camel-mime-filter-bestenc.c (filter): Added code to detect when
+	we have "^From " lines in the sequence of text.
+	(camel_mime_filter_bestenc_get_best_encoding): Added a new flag
+	CAMEL_BESTENC_NO_FROM: if set, it will not allow any lines
+	matching "^From " to appear in the output - currently forcing
+	base64 encoding to achieve this.
+
+	* camel-mime-parser.c (folder_scan_step): Call
+	camel_mime-filter_complete() once we're done, rather than
+	filter_filter().
+	(folder_scan_content): Some fixes for state changing; so that when
+	we do find another boundary it is properly flagged.  Since we
+	strip the last \n off all data, we must take that into account
+	too.  Sigh.  Fixes a rather nasty set of bugs where multipart
+	messages could start including following messages as parts, etc.
+	(struct _header_scan_stack): Added new parameter,
+	boundarylenfinal, which holds the length of the final boundary, if
+	it is different (e.g. for From lines, whihc aren't)
+	(folder_scan_step): Setup teh boundarylenfinal value when creating
+	a new boundary.
+	(folder_scan_content): Hmm, if we hit the end-of-buffer sentinal,
+	reset the scanner back to leave 'atleast' chars in the buffer
+	still, dump that content, and retry again.  Stops us losing a
+	check for a boundary on some data we haven't really looked at yet!
+	(folder_scan_content): Use boundarylenfinal to calculate
+	'atleast'.
+	(folder_scan_header): And here too.
+	(folder_boundary_check): Use the atleast value directly, dont
+	truncate it.  Use the boundarylen/boundarylenfinal values directly
+	too.
+	(struct _header_scan_stack): Add an atleast parameter to cache the
+	atleast info.
+	(folder_push_part): Determine/set 'atleast', every time we add a
+	new part.
+	(folder_scan_header): Get the cached atleast info from the current
+	part.
+	(folder_scan_content): And here too.
+	(folder_scan_header): Fix a problem where a part starting with
+	" text" would be interpreted as a followon header wrongly.
+
+	* camel-mime-filter-charset.c (complete): Add some assertions to
+	find a bug.
+
 2000-11-07  Ettore Perazzoli  <ettore@helixcode.com>
 
 	* camel-service.c (get_name): Strdup the "???" string you return,
diff --git a/camel/camel-mime-filter-bestenc.c b/camel/camel-mime-filter-bestenc.c
index d3db9a2a3f..18ee7039f0 100644
--- a/camel/camel-mime-filter-bestenc.c
+++ b/camel/camel-mime-filter-bestenc.c
@@ -20,6 +20,8 @@
 
 #include "camel-mime-filter-bestenc.h"
 
+#include <string.h>
+
 static void camel_mime_filter_bestenc_class_init (CamelMimeFilterBestencClass *klass);
 static void camel_mime_filter_bestenc_init       (CamelMimeFilter *obj);
 
@@ -54,6 +56,10 @@ reset(CamelMimeFilter *mf)
 	f->total = 0;
 	f->lastc = ~0;
 	f->crlfnoorder = FALSE;
+	f->fromcount = 0;
+	f->hadfrom = FALSE;
+	f->startofline = TRUE;
+
 	camel_charset_init(&f->charset);
 }
 
@@ -63,7 +69,8 @@ filter(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out, s
 	CamelMimeFilterBestenc *f = (CamelMimeFilterBestenc *)mf;
 	register unsigned char *p, *pend;
 
-	f->total += len;
+	if (len == 0)
+		goto donothing;
 
 	if (f->flags & CAMEL_BESTENC_GET_ENCODING) {
 		register unsigned int /* hopefully reg's are assinged in the order they appear? */
@@ -73,6 +80,21 @@ filter(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out, s
 			count0=f->count0,
 			count8 = f->count8;
 
+		/* Check ^From  lines first call, or have the start of a new line waiting? */
+		if ((f->flags & CAMEL_BESTENC_NO_FROM) && !f->hadfrom
+		    && (f->fromcount > 0 || f->startofline)) {
+			if (f->fromcount + len >=5) {
+				memcpy(&f->fromsave[f->fromcount], in, 5-f->fromcount);
+				f->hadfrom = strncmp(f->fromsave, "From ", 5) == 0;
+				f->fromcount = 0;
+			} else {
+				memcpy(&f->fromsave[f->fromcount], in, len);
+				f->fromcount += len;
+			}
+		}
+
+		f->startofline = FALSE;
+
 		/* See rfc2045 section 2 for definitions of 7bit/8bit/binary */
 		p = in;
 		pend = p + len;
@@ -98,6 +120,18 @@ filter(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out, s
 					if (countline > f->maxline)
 						f->maxline = countline;
 					countline = 0;
+
+					/* Check for "^From " lines */
+					if ((f->flags & CAMEL_BESTENC_NO_FROM) && !f->hadfrom) {
+						if (pend-p >= 5) {
+							f->hadfrom = strncmp(p, "From ", 5) == 0;
+						} else if (pend-p == 0) {
+							f->startofline = TRUE;
+						} else {
+							f->fromcount = pend-p;
+							memcpy(f->fromsave, p, pend-p);
+						}
+					}
 				} else {
 					f->crlfnoorder = TRUE;
 				}
@@ -112,9 +146,12 @@ filter(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out, s
 		f->lastc = lastc;
 	}
 
+	f->total += len;
+
 	if (f->flags & CAMEL_BESTENC_GET_CHARSET)
 		camel_charset_step(&f->charset, in, len);
 
+donothing:
 	*out = in;
 	*outlen = len;
 	*outprespace = prespace;
@@ -187,6 +224,12 @@ camel_mime_filter_bestenc_get_best_encoding(CamelMimeFilterBestenc *f, CamelBest
 	printf(" %d%% require encoding?\n", (f->count0+f->count8)*100 / f->total);
 #endif
 
+	/* if we're not allowed to have From lines and we had one, use an encoding
+	   that will never let it show.  Unfortunately only base64 can at present,
+	   although qp could be modified to allow it too */
+	if ((f->flags & CAMEL_BESTENC_NO_FROM) && f->hadfrom)
+		return CAMEL_MIME_PART_ENCODING_BASE64;
+
 	/* if we need to encode, see how we do it */
 	if (required == CAMEL_BESTENC_BINARY)
 		bestenc = CAMEL_MIME_PART_ENCODING_BINARY;
diff --git a/camel/camel-mime-filter-bestenc.h b/camel/camel-mime-filter-bestenc.h
index 32eea23afe..d08f259daa 100644
--- a/camel/camel-mime-filter-bestenc.h
+++ b/camel/camel-mime-filter-bestenc.h
@@ -36,7 +36,9 @@ enum _CamelBestencRequired {
 	CAMEL_BESTENC_GET_CHARSET = 1<<1,
 
 	/* do we treat 'lf' as if it were crlf? */
-	CAMEL_BESTENC_LF_IS_CRLF = 1<<8
+	CAMEL_BESTENC_LF_IS_CRLF = 1<<8,
+	/* do we not allow "From " to appear at the start of a line in any part? */
+	CAMEL_BESTENC_NO_FROM = 1<<9,
 };
 typedef enum _CamelBestencRequired CamelBestencRequired;
 
@@ -59,6 +61,12 @@ struct _CamelMimeFilterBestenc {
 	unsigned int lastc;	/* the last character read */
 	int crlfnoorder;	/* if crlf's occured where they shouldn't have */
 
+	int startofline;	/* are we at the start of a new line? */
+
+	int fromcount;
+	char fromsave[6];	/* save a few characters if we found an \nF near the end of the buffer */
+	int hadfrom;		/* did we encounter a "\nFrom " in the data? */
+
 	unsigned int countline;	/* current count of characters on a given line */
 	unsigned int maxline;	/* max length of any line */
 
diff --git a/camel/camel-mime-filter-charset.c b/camel/camel-mime-filter-charset.c
index c91c7f061f..1fc10a1aff 100644
--- a/camel/camel-mime-filter-charset.c
+++ b/camel/camel-mime-filter-charset.c
@@ -26,6 +26,7 @@
 
 #include "camel-mime-filter-charset.h"
 
+#define d(x)
 
 static void camel_mime_filter_charset_class_init (CamelMimeFilterCharsetClass *klass);
 static void camel_mime_filter_charset_init       (CamelMimeFilterCharset *obj);
@@ -101,6 +102,9 @@ complete(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out,
 	outbuf = mf->outbuf;
 	outlen = mf->outsize;
 
+	/* temporary fix to find another bug somewhere */
+	d(memset(outbuf, 0, outlen));
+
 	if (inlen>0) {
 		converted = unicode_iconv(f->ic, &inbuf, &inlen, &outbuf, &outlen);
 		if (converted == -1) {
@@ -122,6 +126,15 @@ complete(CamelMimeFilter *mf, char *in, size_t len, size_t prespace, char **out,
 		g_warning("Conversion failed to complete: %s", strerror(errno));
 	}
 
+	/* debugging assertion - check for NUL's in output */
+	d({
+		int i;
+		
+		for (i=0;i<(mf->outsize - outlen);i++) {
+			g_assert(mf->outbuf[i]);
+		}
+	});
+
 	*out = mf->outbuf;
 	*outlenptr = mf->outsize - outlen;
 	*outprespace = mf->outpre;
diff --git a/camel/camel-mime-message.c b/camel/camel-mime-message.c
index 6423072c91..c75b83b77f 100644
--- a/camel/camel-mime-message.c
+++ b/camel/camel-mime-message.c
@@ -583,7 +583,7 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 	CamelMimeFilterBestenc *bestenc;
 	int idb, idc = -1;
 	gboolean istext;
-	unsigned int flags;
+	unsigned int flags, callerflags;
 	CamelMimePartEncodingType encoding;
 	CamelDataWrapper *content;
 
@@ -591,7 +591,7 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 	   not have to read the whole lot into memory - although i have a feeling
 	   it would make things a fair bit simpler to do so ... */
 
-	printf("starting to check part\n");
+	d(printf("starting to check part\n"));
 
 	content = camel_medium_get_content_object((CamelMedium *)part);
 	if (content == NULL) {
@@ -610,6 +610,9 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 
 	/* when building the message, any encoded parts are translated already */
 	flags |= CAMEL_BESTENC_LF_IS_CRLF;
+	/* and get any flags the caller passed in */
+	callerflags = (required & CAMEL_BESTENC_NO_FROM);
+	flags |= callerflags;
 
 	/* first a null stream, so any filtering is thrown away; we only want the sideeffects */
 	null = (CamelStream *)camel_stream_null_new();
@@ -628,7 +631,7 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 
 	bestenc = camel_mime_filter_bestenc_new(flags);
 	idb = camel_stream_filter_add(filter, (CamelMimeFilter *)bestenc);
-	printf("writing to checking stream\n");
+	d(printf("writing to checking stream\n"));
 	camel_data_wrapper_write_to_stream(content, (CamelStream *)filter);
 	camel_stream_filter_remove(filter, idb);
 	if (idc != -1) {
@@ -640,13 +643,13 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 	if (istext)
 		charsetin = camel_mime_filter_bestenc_get_best_charset(bestenc);
 
-	printf("charsetin = %s\n", charsetin);
+	d(printf("charsetin = %s\n", charsetin));
 
 	/* if we have US-ASCII, or we're not doing text, we dont need to bother with the rest */
 	if (charsetin != NULL && (required & CAMEL_BESTENC_GET_CHARSET) != 0) {
 		charset = g_strdup(charsetin);
 
-		printf("have charset, trying conversion/etc\n");
+		d(printf("have charset, trying conversion/etc\n"));
 
 		/* now the 'bestenc' can has told us what the best encoding is, we can use that to create
 		   a charset conversion filter as well, and then re-add the bestenc to filter the
@@ -663,7 +666,7 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 			/* otherwise, try another pass, converting to the real charset */
 
 			camel_mime_filter_reset((CamelMimeFilter *)bestenc);
-			camel_mime_filter_bestenc_set_flags(bestenc, CAMEL_BESTENC_GET_ENCODING|CAMEL_BESTENC_LF_IS_CRLF);
+			camel_mime_filter_bestenc_set_flags(bestenc, CAMEL_BESTENC_GET_ENCODING|CAMEL_BESTENC_LF_IS_CRLF|callerflags);
 
 			camel_stream_filter_add(filter, (CamelMimeFilter *)charenc);
 			camel_stream_filter_add(filter, (CamelMimeFilter *)bestenc);
@@ -681,7 +684,7 @@ find_best_encoding(CamelMimePart *part, CamelBestencRequired required, CamelBest
 	camel_object_unref((CamelObject *)bestenc);
 	camel_object_unref((CamelObject *)null);
 
-	printf("done, best encoding = %d\n", encoding);
+	d(printf("done, best encoding = %d\n", encoding));
 
 	if (charsetp)
 		*charsetp = charset;
@@ -719,7 +722,7 @@ best_encoding(CamelMimeMessage *msg, CamelMimePart *part, void *datap)
 				gmime_content_field_set_parameter(part->content_type, "charset", charset?charset:"us-ascii");
 				newct = header_content_type_format(part->content_type->content_type);
 				if (newct) {
-					printf("Setting content-type to %s\n", newct);
+					d(printf("Setting content-type to %s\n", newct));
 
 					camel_mime_part_set_content_type(part, newct);
 					g_free(newct);
diff --git a/camel/camel-mime-parser.c b/camel/camel-mime-parser.c
index efa808170e..6b68013401 100644
--- a/camel/camel-mime-parser.c
+++ b/camel/camel-mime-parser.c
@@ -260,7 +260,9 @@ struct _header_scan_stack {
 	int prestage;		/* used to determine if it is a pre-boundary or post-boundary data segment */
 
 	char *boundary;		/* for multipart/ * boundaries, including leading -- and trailing -- for the final part */
-	int boundarylen;	/* length of boundary, including leading -- */
+	int boundarylen;	/* actual length of boundary, including leading -- if there is one */
+	int boundarylenfinal;	/* length of boundary, including trailing -- if there is one */
+	int atleast;		/* the biggest boundary from here to the parent */
 };
 
 struct _header_scan_filter {
@@ -946,6 +948,11 @@ folder_seek(struct _header_scan_state *s, off_t offset, int whence)
 static void
 folder_push_part(struct _header_scan_state *s, struct _header_scan_stack *h)
 {
+	if (s->parts && s->parts->atleast > h->boundarylenfinal)
+		h->atleast = s->parts->atleast;
+	else
+		h->atleast = h->boundarylenfinal;
+
 	h->parent = s->parts;
 	s->parts = h;
 }
@@ -1003,30 +1010,38 @@ folder_scan_skip_line(struct _header_scan_state *s)
 
 	s->atleast = atleast;
 
-	return -1;		/* not found */
-}
+	 return -1;		/* not found */
+ }
 
 /* TODO: Is there any way to make this run faster?  It gets called a lot ... */
 static struct _header_scan_stack *
 folder_boundary_check(struct _header_scan_state *s, const char *boundary, int *lastone)
 {
 	struct _header_scan_stack *part;
-	int len = s->atleast-2;	/* make sure we dont access past the buffer */
-
+	int len = s->atleast;	/* make sure we dont access past the buffer */
+	
 	h(printf("checking boundary marker upto %d bytes\n", len));
 	part = s->parts;
 	while (part) {
 		h(printf("  boundary: %s\n", part->boundary));
-		h(printf("   against: '%.*s'\n", len, boundary));
+		h(printf("   against: '%.*s'\n", s->atleast, boundary));
 		if (part->boundary
 		    && part->boundarylen <= len
 		    && memcmp(boundary, part->boundary, part->boundarylen)==0) {
 			h(printf("matched boundary: %s\n", part->boundary));
 			/* again, make sure we're in range */
-			if (part->boundarylen <= len+2) {
-				h(printf("checking lastone\n"));
-				*lastone = (boundary[part->boundarylen]=='-'
-					    && boundary[part->boundarylen+1]=='-');
+			if (part->boundarylenfinal <= len) {
+				int extra = part->boundarylenfinal - part->boundarylen;
+				
+				/* check the extra stuff on an final boundary, normally -- for mime parts */
+				if (extra>0) {
+					*lastone = memcmp(&boundary[part->boundarylen],
+							  &part->boundary[part->boundarylen],
+							  extra) == 0;
+				} else {
+					*lastone = TRUE;
+				}
+				h(printf("checking lastone = %s\n", *lastone?"TRUE":"FALSE"));
 			} else {
 				h(printf("not enough room to check last one?\n"));
 				*lastone = FALSE;
@@ -1045,34 +1060,34 @@ header_append_mempool(struct _header_scan_state *s, struct _header_scan_stack *h
 {
 	struct _header_raw *l, *n;
 	char *content;
-
+	
 	content = strchr(header, ':');
 	if (content) {
 		register int len;
 		n = mempool_alloc(h->pool, sizeof(*n));
 		n->next = NULL;
-
+		
 		len = content-header;
 		n->name = mempool_alloc(h->pool, len+1);
 		memcpy(n->name, header, len);
 		n->name[len] = 0;
-
+		
 		content++;
-
+		
 		len = s->outptr - content;
 		n->value = mempool_alloc(h->pool, len+1);
 		memcpy(n->value, content, len);
 		n->value[len] = 0;
-
+		
 		n->offset = offset;
-
+		
 		l = (struct _header_raw *)&h->headers;
 		while (l->next) {
 			l = l->next;
 		}
 		l->next = n;
 	}
-
+	
 }
 
 #define header_raw_append_parse(a, b, c) (header_append_mempool(s, h, b, c))
@@ -1119,37 +1134,28 @@ folder_scan_header(struct _header_scan_state *s, int *lastone)
 	h->pool = mempool_new(8192, 4096);
 #endif
 
-	/* FIXME: this info should be cached ? */
 	part = s->parts;
-	s->atleast = 5;
-	while (part) {
-		if (part->boundary)
-			s->atleast = MAX(s->atleast, part->boundarylen+2);
-		part = part->parent;
-	}
-#if 0
-	s->atleast = MAX(s->atleast, 5);
-	if (s->parts)
-		s->atleast = MAX(s->atleast, s->parts->boundarylen+2);
-#endif
-
+	if (part)
+		s->atleast = part->atleast;
+	else
+		s->atleast = 1;
 	*lastone = FALSE;
 retry:
-
+	
 	while ((len = folder_read(s))>0 && len >= s->atleast) { /* ensure we have at least enough room here */
 		inptr = s->inptr;
 		inend = s->inend-s->atleast;
-
+		
 		while (inptr<=inend) {
 			/*printf("  '%.20s'\n", inptr);*/
-
+			
 			start = inptr;
-
+			
 			if (!s->midline) {
 				if ((part = folder_boundary_check(s, inptr, lastone))) {
 					if ((s->outptr>s->outbuf) || (inptr-start))
 						goto header_truncated; /* may not actually be truncated */
-
+					
 					goto normal_exit;
 				}
 
@@ -1164,15 +1170,15 @@ retry:
 					*start = ' ';
 				}
 			}
-
+			
 			/* goto next line */
 			while ((*inptr++)!='\n')
 				;
-
+			
 			g_assert(inptr<=s->inend+1);
-
+			
 			header_append(s, start, inptr-1);
-
+			
 			/* check against the real buffer end, not our 'atleast limited' end */
 			/* also make sure we have at least 1 char lookahead, so even if we found a \n at
 			   the end, well, make out we didn't, and re-scan it next pass */
@@ -1182,7 +1188,7 @@ retry:
 			} else {
 				s->midline = FALSE;
 			}
-
+			
 			h(printf("midline = %s\n", s->midline?"TRUE":"FALSE"));
 			h(printf("outbuf[0] = %02x '%c' oubuf[1] = %02x '%c'\n",
 				 s->outbuf[0], isprint(s->outbuf[0])?s->outbuf[0]:'.',
@@ -1190,7 +1196,9 @@ retry:
 			h(printf("inptr[0] = %02x '%c' inptr[1] = %02x '%c'\n",
 				 inptr[0], isprint(inptr[0])?inptr[0]:'.',
 				 inptr[1], isprint(inptr[1])?inptr[1]:'.'));
-
+			
+			/* this is wont handle no headers \n\n foobr - treats it as continuation */
+/*
 			if (!s->midline
 			    && !(inptr[0] == ' ' || inptr[0] == '\t')) {
 				h(printf("ok, checking\n"));
@@ -1200,23 +1208,34 @@ retry:
 					h(printf("header done?\n"));
 					goto header_done;
 				}
+*/
+			if (!s->midline) {
+				h(printf("ok, checking\n"));
+				if (s->outbuf == s->outptr
+				    || s->outbuf[0] == '\n'
+				    || (s->outbuf[0] == '\r' && s->outbuf[1]=='\n')) {
+					h(printf("header done?\n"));
+					goto header_done;
+				}
 
-				/* we always have at least _1_ char here ... */
-				if (s->outptr > s->outbuf && s->outptr[-1] == '\n')
-					s->outptr--;
-				s->outptr[0] = 0;
-				
-				d(printf("header '%.10s' at %d\n", s->outbuf, s->header_start));
+				if (!(inptr[0] == ' ' || inptr[0] == '\t')) {
+					/* we always have at least _1_ char here ... */
+					if (s->outptr > s->outbuf && s->outptr[-1] == '\n')
+						s->outptr--;
+					s->outptr[0] = 0;
 				
-				header_raw_append_parse(&h->headers, s->outbuf, s->header_start);
+					d(printf("header '%.10s' at %d\n", s->outbuf, s->header_start));
+
+					header_raw_append_parse(&h->headers, s->outbuf, s->header_start);
 				
-				if (inptr[0]=='\n'
-				    || (inptr[0] == '\r' && inptr[1]=='\n')) {
-					inptr++;
-					goto header_done;
+					if (inptr[0]=='\n'
+					    || (inptr[0] == '\r' && inptr[1]=='\n')) {
+						inptr++;
+						goto header_done;
+					}
+					s->outptr = s->outbuf;
+					s->header_start = -1;
 				}
-				s->outptr = s->outbuf;
-				s->header_start = -1;
 			}
 		}
 		s->inptr = inptr;
@@ -1227,43 +1246,43 @@ retry:
 	if (overpart) {
 		overpart = overpart->parent;
 		while (overpart) {
-			if (overpart->boundary && (overpart->boundarylen+2) < s->atleast) {
-				s->atleast = overpart->boundarylen+2;
+			if (overpart->boundary && (overpart->boundarylenfinal) < s->atleast) {
+				s->atleast = overpart->boundarylenfinal;
 				h(printf("Retrying next smaller part ...\n"));
 				goto retry;
 			}
 			overpart = overpart->parent;
 		}
 	}
-
+	
 	if ((s->outptr > s->outbuf) || s->inend > s->inptr) {
 		start = s->inptr;
 		inptr = s->inend;
 		goto header_truncated;
 	}
-
+	
 	s->atleast = atleast;
-
+	
 	return h;
-
+	
 header_truncated:
-
+	
 	header_append(s, start, inptr);
-
+	
 	if (s->outptr>s->outbuf && s->outptr[-1] == '\n')
 		s->outptr--;
 	s->outptr[0] = 0;
-
+	
 	if (s->outbuf[0] == '\n'
 	    || (s->outbuf[0] == '\r' && s->outbuf[1]=='\n')) {
 		goto header_done;
 	}
-
+	
 	header_raw_append_parse(&h->headers, s->outbuf, s->header_start);
-
+	
 header_done:
 	part = s->parts;
-
+	
 	s->outptr = s->outbuf;
 normal_exit:
 	s->inptr = inptr;
@@ -1281,28 +1300,16 @@ folder_scan_content(struct _header_scan_state *s, int *lastone, char **data, int
 	char *start;
 	int len;
 	struct _header_scan_stack *part, *overpart = s->parts;
-	int already_packed = FALSE;
 	int onboundary = FALSE;
 
 	c(printf("scanning content\n"));
 
-	/* FIXME: this info should be cached ? */
 	part = s->parts;
-	s->atleast = 5;
-	while (part) {
-		if (part->boundary) {
-			c(printf("boundary: %s\n", part->boundary));
-			s->atleast = MAX(s->atleast, part->boundarylen+2);
-		}
-		part = part->parent;
-	}
-/*	s->atleast = MAX(s->atleast, 5);*/
-#if 0
-	if (s->parts)
-		s->atleast = MAX(s->atleast, s->parts->boundarylen+2);
-#endif
+	if (part)
+		s->atleast = part->atleast;
+	else
+		s->atleast = 1;
 	*lastone = FALSE;
-
 retry:
 	c(printf("atleast = %d\n", s->atleast));
 	
@@ -1316,65 +1323,43 @@ retry:
 		while (inptr<=inend) {
 			if (!s->midline
 			    && (part = folder_boundary_check(s, inptr, lastone))) {
-				if ( (inptr-start) ) {
-					onboundary = TRUE;
+				onboundary = TRUE;
+
+				/* since we truncate the boundary data, we need at least 1 char here spare,
+				   to remain in the same state */
+				if ( (inptr-start) > 1)
 					goto content;
-				}
-				
-				goto normal_exit;
-			}
+
+				/* otherwise, jump to the state of the boundary we actually found */
+				goto normal_exit;			}
 
 			/* goto the next line */
 			while ((*inptr++)!='\n')
 				;
 
-			/* check against the real buffer end, not our 'atleast limited' end */
-			if (inptr> s->inend) {
-				inptr--;
+			/* check the sentinal, if we went past the atleast limit, and reset it to there */
+			if (inptr > inend+1) {
 				s->midline = TRUE;
+				inptr = inend+1;
 			} else {
 				s->midline = FALSE;
 			}
-
-			g_assert(inptr<=s->inend);
 		}
 
-		/* *sigh* so much for the beautiful simplicity of the code so far - here we
-		   have the snot to deal with the nasty end-cases that come from the read-ahead
-		   buffers we use */
-		/* what this does, is if we are somewhere near the end of the buffer,
-		   force it to the front, and re-read, ensuring we bunch as much together
-		   as possible, for the final read, without copying too much of the time */
-		/* make sure we dont loop forever, but also make sure we try smaller
-		   boundaries, if there are any, so we dont miss any. */
-		/* this is not needed for the header scanner, since it copies its own
-		   data */
-		c(printf("start offset = %d  atleast = %d\n", start-s->inbuf, s->atleast));
-		if (start > (s->inbuf + s->atleast)) {
-			/* force a re-scan of this data */
-			s->inptr = start;
-			if (already_packed)
-				goto smaller_boundary;
-			c(printf("near the end, try and bunch things up a bit first\n"));
-			already_packed = TRUE;
-		} else {
-			c(printf("dumping what i've got ...\n"));
-			/* what would be nice here, is if that we're at eof, we bunch the last
-			   little bit in the same content, but i dont think this is easy */
-			goto content_mid;
-		}
+		c(printf("ran out of input, dumping what i have (%d) bytes midline = %s\n",
+			 inptr-start, s->midline?"TRUE":"FALSE"));
+		goto content;
 	}
 
 	c(printf("length read = %d\n", len));
-smaller_boundary:
 
 	/* ok, we're at the end of the data, just make sure we're not missing out some small
 	   truncated header markers */
 	if (overpart) {
 		overpart = overpart->parent;
 		while (overpart) {
-			if (overpart->boundary && (overpart->boundarylen+2) < s->atleast) {
-				s->atleast = overpart->boundarylen+2;
+			if (overpart->boundary && (overpart->boundarylenfinal) < s->atleast) {
+				s->atleast = overpart->boundarylenfinal;
 				c(printf("Retrying next smaller part ...\n"));
 				goto retry;
 			}
@@ -1392,8 +1377,6 @@ smaller_boundary:
 	s->atleast = atleast;
 	return NULL;
 
-content_mid:
-	s->midline = TRUE;
 content:
 	part = s->parts;
 normal_exit:
@@ -1402,12 +1385,12 @@ normal_exit:
 
 	*data = start;
 	/* if we hit a boundary, we should not include the closing \n */
-	if (onboundary)
+	if (onboundary && (inptr-start)>0)
 		*length = inptr-start-1;
 	else
 		*length = inptr-start;
 
-/*	printf("got %scontent: %.*s", s->midline?"partial ":"", inptr-start, start);*/
+	/*printf("got %scontent: '%.*s'\n", s->midline?"partial ":"", inptr-start, start);*/
 
 	return part;
 }
@@ -1559,6 +1542,7 @@ tail_recurse:
 			h = g_malloc0(sizeof(*h));
 			h->boundary = g_strdup("From ");
 			h->boundarylen = strlen(h->boundary);
+			h->boundarylenfinal = h->boundarylen;
 			folder_push_part(s, h);
 			
 			h = s->parts;
@@ -1602,6 +1586,7 @@ tail_recurse:
 				if (bound) {
 					d(printf("multipart, boundary = %s\n", bound));
 					h->boundarylen = strlen(bound)+2;
+					h->boundarylenfinal = h->boundarylen+2;
 					h->boundary = g_malloc(h->boundarylen+3);
 					sprintf(h->boundary, "--%s--", bound);
 					type = HSCAN_MULTIPART;
@@ -1668,8 +1653,8 @@ tail_recurse:
 		/* check for any filter completion data */
 		if (*datalength > 0) {
 			while (f) {
-				camel_mime_filter_filter(f->filter, *databuffer, *datalength, presize,
-							 databuffer, datalength, &presize);
+				camel_mime_filter_complete(f->filter, *databuffer, *datalength, presize,
+							   databuffer, datalength, &presize);
 				f = f->next;
 			}
 		}
diff --git a/camel/camel-stream-filter.c b/camel/camel-stream-filter.c
index 7c5dec7a2a..866c8d2416 100644
--- a/camel/camel-stream-filter.c
+++ b/camel/camel-stream-filter.c
@@ -22,7 +22,7 @@
 #include "camel-stream-filter.h"
 
 #define d(x)
-#include <stdio.h>
+/*#include <stdio.h>*/
 
 struct _filter {
 	struct _filter *next;