Index: ChangeLog

Index: ChangeLog
===================================================================
RCS file: /cvs/gnome/evolution/camel/ChangeLog,v
retrieving revision 1.684
diff -r1.684 ChangeLog
0a1,34
> 2001-01-17  Not Zed  <NotZed@Ximian.com>
>
> 	* camel-folder.c (free_summary): Call
> 	camel_folder_summary_array_free() to do the work.
> 	(get_summary): Use camel_folder_summary_array() to get the array
> 	atomically.  These fixes allow folder/test8 to work again, and fix
> 	a sort of race where the summary size can change while we were
> 	making a copy of it.
>
> 	* camel-folder-summary.c (camel_folder_summary_array): Get the
> 	summary array atomically, so it can't contain empty records.
> 	(camel_folder_summary_array_free): And free it.
>
> 	* tests/lib/camel-test.c (die): If we are verbose & in threads,
> 	then goto sleep so we can debug.
>
> 	* tests/folder/test8.c (worker): Add a missing pull() for
> 	comnparing content.
>
> 	* camel-filter-search.c: Fix the symbol table, so match-all is an
> 	immediate function, as it should be.
>
> 	* tests/folder/test9.c (main): New test, tests some filtering
> 	things.
>
> 	* tests/message/test3.c (main): Dont use a boundary string with
> 	spaces in it.  Folding can corrupt it.  Maybe the folding isn't
> 	working entirely right, but anyway.
>
> 	* camel-session.c: Debug out the debug.
>
> 	* camel-filter-driver.c (camel_filter_driver_filter_folder): Plug
> 	a messageinfo leak.
>
1a36,94
>
> 	* camel-filter-search.c (header_exists): Changed to support
> 	multiple args (or'd together).
> 	(header_contains): Cleaned up to match the search code.  Why did
> 	fejj change it? I'll never know.
> 	(header_matches):
> 	(header_starts_with):
> 	(header_ends_with): Big cleanup of fejj's "i'm the cut & paste
> 	king" code.  Also properly handle or'ing of additional args to
> 	match what the folder-search code should do.
> 	(check_match): New function which does the annoying matching
> 	stuff (for header matches).
> 	(check_header): Similarly, handles or'ing of the matches together.
> 	(header_contains):
> 	(header_matches):
> 	(header_starts_with):
> 	(header_ends_with): Call check_header to do the actual work.
> 	(header_soundex): And here too.
> 	(match_all): Yeah like match-all isn't passed expression results,
> 	its passed expression terms.  Fix this so match-all works like it
> 	should, by executing the contained expression.
> 	(message_body_contains): Copied directly from
> 	camel-folder-search.c, a more robust/faster/simpler body search
> 	code.
> 	(mime_part_matches): Removed entirely.
> 	(handle_multipart): Removed entirely.
> 	(build_match_regex): Copied from camel-folder-search.  Builds a
> 	set of simple strings into a regex pattern that matches any of
> 	them (for faster & simpler matching).  Expanded to accept regex
> 	patterns itself, so it can merge them together.
> 	(body_contains): Use build match/match message to match using a
> 	built regex.
> 	(body_regex): Likewise, this time we tell it we're building a
> 	regex though.
> 	(header_full_regex): Use build_match_regex to take the drudgery
> 	out of it, and expand it to handle multiple regex's at once.
> 	(get_full_header): slightly cleaner (well i dunno, the sprintf
> 	stuff just got to me).
> 	(header_regex): Cleaned up to use build_match_Regex too, and to
> 	properly check types.
> 	(filter_message_search): Just allocate 'fms' on the stack.
>
> 	* camel-filter-driver.c (camel_filter_driver_finalise):
> 	(camel_filter_driver_init):
> 	(camel_filter_driver_class_init):
> 	(camel_filter_driver_get_type): Changed from gtk object to camel
> 	object.
> 	(camel_filter_driver_add_rule): New function to add a rule to be
> 	processed in sexp form.
> 	(camel_filter_driver_init): Init the rules list.
> 	(camel_filter_driver_finalise): Clear the rules/rules list.
> 	(camel_filter_driver_filter_message): Scan rules list directly
> 	rather than creating on the fly.
>
> 	* Makefile.am (libcamelinclude_HEADERS): Added camel-filter-driver.h
> 	(libcamel_la_SOURCES): Added camel-filter-driver.c, code taken
> 	from filter-driver, which can drive, uh, filters based on sexp's.
> 	(libcamelinclude_HEADERS):
> 	(libcamel_la_SOURCES): Added camel-filter-search.[ch]

svn path=/trunk/; revision=7560

17 files changed, 1896 insertions, 28 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index 23531a930d..7d28001cc9 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -1,3 +1,37 @@
+2001-01-17  Not Zed  <NotZed@Ximian.com>
+
+	* camel-folder.c (free_summary): Call
+	camel_folder_summary_array_free() to do the work.
+	(get_summary): Use camel_folder_summary_array() to get the array
+	atomically.  These fixes allow folder/test8 to work again, and fix
+	a sort of race where the summary size can change while we were
+	making a copy of it.
+
+	* camel-folder-summary.c (camel_folder_summary_array): Get the
+	summary array atomically, so it can't contain empty records.
+	(camel_folder_summary_array_free): And free it.
+
+	* tests/lib/camel-test.c (die): If we are verbose & in threads,
+	then goto sleep so we can debug.
+
+	* tests/folder/test8.c (worker): Add a missing pull() for
+	comnparing content.
+
+	* camel-filter-search.c: Fix the symbol table, so match-all is an
+	immediate function, as it should be.
+
+	* tests/folder/test9.c (main): New test, tests some filtering
+	things.
+
+	* tests/message/test3.c (main): Dont use a boundary string with
+	spaces in it.  Folding can corrupt it.  Maybe the folding isn't
+	working entirely right, but anyway.
+
+	* camel-session.c: Debug out the debug.
+
+	* camel-filter-driver.c (camel_filter_driver_filter_folder): Plug
+	a messageinfo leak.
+
 2001-01-16  Dan Winship  <danw@ximian.com>
 
 	Delayed loading of IMAP message parts.
@@ -92,6 +126,65 @@
 
 2001-01-16  Not Zed  <NotZed@Ximian.com>
 
+	* camel-filter-search.c (header_exists): Changed to support
+	multiple args (or'd together).
+	(header_contains): Cleaned up to match the search code.  Why did
+	fejj change it? I'll never know.
+	(header_matches): 
+	(header_starts_with): 
+	(header_ends_with): Big cleanup of fejj's "i'm the cut & paste
+	king" code.  Also properly handle or'ing of additional args to
+	match what the folder-search code should do.
+	(check_match): New function which does the annoying matching
+	stuff (for header matches).
+	(check_header): Similarly, handles or'ing of the matches together.
+	(header_contains):
+	(header_matches): 
+	(header_starts_with): 
+	(header_ends_with): Call check_header to do the actual work.
+	(header_soundex): And here too.
+	(match_all): Yeah like match-all isn't passed expression results,
+	its passed expression terms.  Fix this so match-all works like it
+	should, by executing the contained expression.
+	(message_body_contains): Copied directly from
+	camel-folder-search.c, a more robust/faster/simpler body search
+	code.
+	(mime_part_matches): Removed entirely.
+	(handle_multipart): Removed entirely.
+	(build_match_regex): Copied from camel-folder-search.  Builds a
+	set of simple strings into a regex pattern that matches any of
+	them (for faster & simpler matching).  Expanded to accept regex
+	patterns itself, so it can merge them together.
+	(body_contains): Use build match/match message to match using a
+	built regex.
+	(body_regex): Likewise, this time we tell it we're building a
+	regex though.
+	(header_full_regex): Use build_match_regex to take the drudgery
+	out of it, and expand it to handle multiple regex's at once.
+	(get_full_header): slightly cleaner (well i dunno, the sprintf
+	stuff just got to me).
+	(header_regex): Cleaned up to use build_match_Regex too, and to
+	properly check types.
+	(filter_message_search): Just allocate 'fms' on the stack.
+
+	* camel-filter-driver.c (camel_filter_driver_finalise): 
+	(camel_filter_driver_init): 
+	(camel_filter_driver_class_init): 
+	(camel_filter_driver_get_type): Changed from gtk object to camel
+	object.
+	(camel_filter_driver_add_rule): New function to add a rule to be
+	processed in sexp form.
+	(camel_filter_driver_init): Init the rules list.
+	(camel_filter_driver_finalise): Clear the rules/rules list.
+	(camel_filter_driver_filter_message): Scan rules list directly
+	rather than creating on the fly.
+
+	* Makefile.am (libcamelinclude_HEADERS): Added camel-filter-driver.h
+	(libcamel_la_SOURCES): Added camel-filter-driver.c, code taken
+	from filter-driver, which can drive, uh, filters based on sexp's.
+	(libcamelinclude_HEADERS): 
+	(libcamel_la_SOURCES): Added camel-filter-search.[ch]
+
 	* camel-folder-summary.c (camel_folder_summary_decode_string):
 	Chganged len back to be unsigned.  And do a simple range check on
 	the string value to try and detect corrupted summary files.
diff --git a/camel/Makefile.am b/camel/Makefile.am
index 24483b16f2..8564db65d5 100644
--- a/camel/Makefile.am
+++ b/camel/Makefile.am
@@ -20,6 +20,8 @@ libcamel_la_SOURCES = 				\
 	camel-address.c				\
 	camel-data-wrapper.c			\
 	camel-exception.c			\
+	camel-filter-driver.c			\
+	camel-filter-search.c			\
 	camel-folder-search.c			\
 	camel-folder-summary.c			\
 	camel-folder-thread.c			\
@@ -73,6 +75,8 @@ libcamelinclude_HEADERS =			\
 	camel-data-wrapper.h			\
 	camel-exception-list.def		\
 	camel-exception.h			\
+	camel-filter-driver.h			\
+	camel-filter-search.h			\
 	camel-folder-search.h			\
 	camel-folder-summary.h			\
 	camel-folder-thread.h			\
diff --git a/camel/camel-filter-driver.c b/camel/camel-filter-driver.c
new file mode 100644
index 0000000000..42aa442c60
--- /dev/null
+++ b/camel/camel-filter-driver.c
@@ -0,0 +1,774 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
+ *  Copyright (C) 2000 Helix Code Inc.
+ *  Copyright (C) 2001 Ximian Inc.
+ *
+ *  Authors: Michael Zucchi <notzed@helixcode.com>
+ *           Jeffrey Stedfast <fejj@helixcode.com>
+ *
+ *  This program is free software; you can redistribute it and/or 
+ *  modify it under the terms of the GNU General Public License as 
+ *  published by the Free Software Foundation; either version 2 of the
+ *  License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
+ */
+
+#include "camel-filter-driver.h"
+#include "camel-filter-search.h"
+
+#include "camel-exception.h"
+#include "camel-service.h"
+#include "camel-mime-message.h"
+
+#include <glib.h>
+
+#include <time.h>
+
+#include "e-util/e-sexp.h"
+#include "e-util/e-memory.h"
+#include "e-util/e-msgport.h"	/* for edlist */
+
+#define d(x)
+
+/* type of status for a log report */
+enum filter_log_t {
+	FILTER_LOG_NONE,
+	FILTER_LOG_START,       /* start of new log entry */
+	FILTER_LOG_ACTION,      /* an action performed */
+	FILTER_LOG_END,	        /* end of log */
+};
+
+/* list of rule nodes */
+struct _filter_rule {
+	struct _filter_rule *next;
+	struct _filter_rule *prev;
+
+	char *match;
+	char *action;
+	char *name;
+};
+
+struct _CamelFilterDriverPrivate {
+	GHashTable *globals;       /* global variables */
+
+	CamelFolder *defaultfolder;	/* defualt folder */
+	
+	FDStatusFunc *statusfunc; 	/* status callback */
+	void *statusdata;		/* status callback data */
+	
+	/* for callback */
+	FilterGetFolderFunc get_folder;
+	void *data;
+	
+	/* run-time data */
+	GHashTable *folders;       /* folders that message has been copied to */
+	GHashTable *forwards;      /* addresses that have been forwarded the message */
+	
+	gboolean terminated;       /* message processing was terminated */
+	gboolean deleted;          /* message was marked for deletion */
+	gboolean copied;           /* message was copied to some folder or another */
+	
+	CamelMimeMessage *message; /* input message */
+	CamelMessageInfo *info;    /* message summary info */
+	
+	FILE *logfile;             /* log file */
+	
+	EDList rules;		/* list of _filter_rule structs */
+
+	CamelException *ex;
+	
+	/* evaluator */
+	ESExp *eval;
+};
+
+#define _PRIVATE(o) (((CamelFilterDriver *)(o))->priv)
+
+static void camel_filter_driver_class_init (CamelFilterDriverClass *klass);
+static void camel_filter_driver_init       (CamelFilterDriver *obj);
+static void camel_filter_driver_finalise   (CamelObject *obj);
+
+static void camel_filter_driver_log (CamelFilterDriver *driver, enum filter_log_t status, const char *desc, ...);
+
+static CamelFolder *open_folder (CamelFilterDriver *d, const char *folder_url);
+static int close_folders (CamelFilterDriver *d);
+
+static ESExpResult *do_delete (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *mark_forward (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *do_copy (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *do_move (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *do_stop (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *do_colour (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *do_score (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+static ESExpResult *do_flag (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *);
+
+/* these are our filter actions - each must have a callback */
+static struct {
+	char *name;
+	ESExpFunc *func;
+	int type;		/* set to 1 if a function can perform shortcut evaluation, or
+				   doesn't execute everything, 0 otherwise */
+} symbols[] = {
+	{ "delete",          (ESExpFunc *) do_delete,    0 },
+	{ "forward-to",      (ESExpFunc *) mark_forward, 0 },
+	{ "copy-to",         (ESExpFunc *) do_copy,      0 },
+	{ "move-to",         (ESExpFunc *) do_move,      0 },
+	{ "stop",            (ESExpFunc *) do_stop,      0 },
+	{ "set-colour",      (ESExpFunc *) do_colour,    0 },
+	{ "set-score",       (ESExpFunc *) do_score,     0 },
+	{ "set-system-flag", (ESExpFunc *) do_flag,      0 }
+};
+
+static CamelObjectClass *camel_filter_driver_parent;
+
+guint
+camel_filter_driver_get_type (void)
+{
+	static CamelType type = CAMEL_INVALID_TYPE;
+
+	if (type == CAMEL_INVALID_TYPE)	{
+		type = camel_type_register(CAMEL_OBJECT_TYPE, "CamelFilterDriver",
+					   sizeof(CamelFilterDriver),
+					   sizeof(CamelFilterDriverClass),
+					   (CamelObjectClassInitFunc)camel_filter_driver_class_init,
+					   NULL,
+					   (CamelObjectInitFunc)camel_filter_driver_init,
+					   (CamelObjectFinalizeFunc)camel_filter_driver_finalise);
+	}
+	
+	return type;
+}
+
+static void
+camel_filter_driver_class_init (CamelFilterDriverClass *klass)
+{
+	/*CamelObjectClass *object_class = (CamelObjectClass *) klass;*/
+
+	camel_filter_driver_parent = camel_type_get_global_classfuncs(camel_object_get_type());
+}
+
+static void
+camel_filter_driver_init (CamelFilterDriver *obj)
+{
+	struct _CamelFilterDriverPrivate *p;
+	int i;
+	
+	p = _PRIVATE (obj) = g_malloc0 (sizeof (*p));
+
+	e_dlist_init(&p->rules);
+
+	p->eval = e_sexp_new ();
+	/* Load in builtin symbols */
+	for (i = 0; i < sizeof (symbols) / sizeof (symbols[0]); i++) {
+		if (symbols[i].type == 1) {
+			e_sexp_add_ifunction (p->eval, 0, symbols[i].name, (ESExpIFunc *)symbols[i].func, obj);
+		} else {
+			e_sexp_add_function (p->eval, 0, symbols[i].name, symbols[i].func, obj);
+		}
+	}
+	
+	p->globals = g_hash_table_new (g_str_hash, g_str_equal);
+	
+	p->folders = g_hash_table_new (g_str_hash, g_str_equal);
+}
+
+static void
+free_hash_strings (void *key, void *value, void *data)
+{
+	g_free (key);
+	g_free (value);
+}
+
+static void
+camel_filter_driver_finalise (CamelObject *obj)
+{
+	CamelFilterDriver *driver = (CamelFilterDriver *) obj;
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);	
+	struct _filter_rule *node;
+
+	/* close all folders that were opened for appending */
+	close_folders (driver);
+	g_hash_table_destroy (p->folders);
+	
+	g_hash_table_foreach (p->globals, free_hash_strings, driver);
+	g_hash_table_destroy (p->globals);
+
+	e_sexp_unref(p->eval);
+	
+	if (p->defaultfolder)
+		camel_object_unref (CAMEL_OBJECT (p->defaultfolder));
+
+	while ((node = (struct _filter_rule *)e_dlist_remhead(&p->rules))) {
+		g_free(node->match);
+		g_free(node->action);
+		g_free(node->name);
+		g_free(node);
+	}
+	
+	g_free (p);
+}
+
+/**
+ * camel_filter_driver_new:
+ * @system: path to system rules
+ * @user: path to user rules
+ * @get_folder: function to call to fetch folders
+ *
+ * Create a new CamelFilterDriver object.
+ * 
+ * Return value: A new CamelFilterDriver widget.
+ **/
+CamelFilterDriver *
+camel_filter_driver_new (FilterGetFolderFunc get_folder, void *data)
+{
+	CamelFilterDriver *new;
+	struct _CamelFilterDriverPrivate *p;
+	
+	new = CAMEL_FILTER_DRIVER (camel_object_new(camel_filter_driver_get_type ()));
+	p = _PRIVATE (new);
+	
+	p->get_folder = get_folder;
+	p->data = data;
+	
+	return new;
+}
+
+void
+camel_filter_driver_set_logfile (CamelFilterDriver *d, FILE *logfile)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (d);
+	
+	p->logfile = logfile;
+}
+
+void
+camel_filter_driver_set_status_func (CamelFilterDriver *d, FDStatusFunc *func, void *data)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (d);
+	
+	p->statusfunc = func;
+	p->statusdata = data;
+}
+
+void
+camel_filter_driver_set_default_folder (CamelFilterDriver *d, CamelFolder *def)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (d);
+	
+	if (p->defaultfolder)
+		camel_object_unref (CAMEL_OBJECT (p->defaultfolder));
+	
+	p->defaultfolder = def;
+	
+	if (p->defaultfolder)
+		camel_object_ref (CAMEL_OBJECT (p->defaultfolder));
+}
+
+void
+camel_filter_driver_add_rule(CamelFilterDriver *d, const char *name, const char *match, const char *action)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (d);
+	struct _filter_rule *node;
+
+	node = g_malloc(sizeof(*node));
+	node->match = g_strdup(match);
+	node->action = g_strdup(action);
+	node->name = g_strdup(name);
+	e_dlist_addtail(&p->rules, (EDListNode *)node);
+}
+
+static void
+report_status (CamelFilterDriver *driver, enum filter_status_t status, const char *desc, ...)
+{
+	/* call user-defined status report function */
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	va_list ap;
+	char *str;
+	
+	if (p->statusfunc) {
+		va_start (ap, desc);
+		str = g_strdup_vprintf (desc, ap);
+		p->statusfunc (driver, status, str, p->statusdata);
+		g_free (str);
+	}
+}
+
+
+#if 0
+void
+camel_filter_driver_set_global (CamelFilterDriver *d, const char *name, const char *value)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (d);
+	char *oldkey, *oldvalue;
+	
+	if (g_hash_table_lookup_extended (p->globals, name, (void *)&oldkey, (void *)&oldvalue)) {
+		g_free (oldvalue);
+		g_hash_table_insert (p->globals, oldkey, g_strdup (value));
+	} else {
+		g_hash_table_insert (p->globals, g_strdup (name), g_strdup (value));
+	}
+}
+#endif
+
+static ESExpResult *
+do_delete (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	d(fprintf (stderr, "doing delete\n"));
+	p->deleted = TRUE;
+	camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Delete");
+	
+	return NULL;
+}
+
+static ESExpResult *
+mark_forward (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	/*struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);*/
+	
+	d(fprintf (stderr, "marking message for forwarding\n"));
+	/* FIXME: do stuff here */
+	camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Forward");
+	
+	return NULL;
+}
+
+static ESExpResult *
+do_copy (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	int i;
+	
+	d(fprintf (stderr, "copying message...\n"));
+	
+	for (i = 0; i < argc; i++) {
+		if (argv[i]->type == ESEXP_RES_STRING) {
+			/* open folders we intent to copy to */
+			char *folder = argv[i]->value.string;
+			char *service_url;
+			CamelFolder *outbox;
+			
+			outbox = open_folder (driver, folder);
+			if (!outbox)
+				continue;
+			
+			p->copied = TRUE;
+			camel_folder_append_message (outbox, p->message, p->info, p->ex);
+			
+			service_url = camel_service_get_url (CAMEL_SERVICE (camel_folder_get_parent_store (outbox)));
+			camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Copy to folder %s", service_url);
+			g_free (service_url);
+		}
+	}
+	
+	return NULL;
+}
+
+static ESExpResult *
+do_move (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	int i;
+	
+	d(fprintf (stderr, "moving message...\n"));
+	
+	for (i = 0; i < argc; i++) {
+		if (argv[i]->type == ESEXP_RES_STRING) {
+			/* open folders we intent to move to */
+			char *folder = argv[i]->value.string;
+			char *service_url;
+			CamelFolder *outbox;
+			
+			outbox = open_folder (driver, folder);
+			if (!outbox)
+				continue;
+			
+			p->copied = TRUE;
+			p->deleted = TRUE;  /* a 'move' is a copy & delete */
+			
+			camel_folder_append_message (outbox, p->message, p->info, p->ex);
+			
+			service_url = camel_service_get_url (CAMEL_SERVICE (camel_folder_get_parent_store (outbox)));
+			camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Move to folder %s", service_url);
+			g_free (service_url);
+		}
+	}
+	
+	return NULL;
+}
+
+static ESExpResult *
+do_stop (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Stopped processing");
+	d(fprintf (stderr, "terminating message processing\n"));
+	p->terminated = TRUE;
+	
+	return NULL;
+}
+
+static ESExpResult *
+do_colour (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	d(fprintf (stderr, "setting colour tag\n"));
+	if (argc > 0 && argv[0]->type == ESEXP_RES_STRING) {
+		camel_tag_set (&p->info->user_tags, "colour", argv[0]->value.string);
+		camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Set colour to %s", argv[0]->value.string);
+	}
+	
+	return NULL;
+}
+
+static ESExpResult *
+do_score (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	d(fprintf (stderr, "setting score tag\n"));
+	if (argc > 0 && argv[0]->type == ESEXP_RES_INT) {
+		char *value;
+		
+		value = g_strdup_printf ("%d", argv[0]->value.number);
+		camel_tag_set (&p->info->user_tags, "score", value);
+		camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Set score to %d", argv[0]->value.number);
+		g_free (value);
+	}
+	
+	return NULL;
+}
+
+static ESExpResult *
+do_flag (struct _ESExp *f, int argc, struct _ESExpResult **argv, CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	d(fprintf (stderr, "setting flag\n"));
+	if (argc == 1 && argv[0]->type == ESEXP_RES_STRING) {
+		p->info->flags |= camel_system_flag (argv[0]->value.string) | CAMEL_MESSAGE_FOLDER_FLAGGED;
+		camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Set %s flag", argv[0]->value.string);
+	}
+	
+	return NULL;
+}
+
+static CamelFolder *
+open_folder (CamelFilterDriver *driver, const char *folder_url)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	CamelFolder *camelfolder;
+	
+	/* we have a lookup table of currently open folders */
+	camelfolder = g_hash_table_lookup (p->folders, folder_url);
+	if (camelfolder)
+		return camelfolder;
+	
+	camelfolder = p->get_folder (driver, folder_url, p->data);
+	
+	if (camelfolder) {
+		g_hash_table_insert (p->folders, g_strdup (folder_url), camelfolder);
+		camel_folder_freeze (camelfolder);
+	}
+	
+	return camelfolder;
+}
+
+static void
+close_folder (void *key, void *value, void *data)
+{	
+	CamelFolder *folder = value;
+	CamelFilterDriver *driver = data;
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	g_free (key);
+	camel_folder_sync (folder, FALSE, p->ex);
+	camel_folder_thaw (folder);
+	camel_object_unref (CAMEL_OBJECT (folder));
+}
+
+/* flush/close all folders */
+static int
+close_folders (CamelFilterDriver *driver)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	g_hash_table_foreach (p->folders, close_folder, driver);
+	g_hash_table_destroy (p->folders);
+	p->folders = g_hash_table_new (g_str_hash, g_str_equal);
+	
+	/* FIXME: status from driver */
+	return 0;
+}
+
+#if 0
+static void
+free_key (gpointer key, gpointer value, gpointer user_data)
+{
+	g_free (key);
+}
+#endif
+
+
+static void
+camel_filter_driver_log (CamelFilterDriver *driver, enum filter_log_t status, const char *desc, ...)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	
+	if (p->logfile) {
+		char *str = NULL;
+		
+		if (desc) {
+			va_list ap;
+			
+			va_start (ap, desc);
+			str = g_strdup_vprintf (desc, ap);
+		}
+		
+		switch (status) {
+		case FILTER_LOG_START: {
+			/* write log header */
+			const char *subject = NULL;
+			char *fromstr;
+			const CamelInternetAddress *from;
+			char date[50];
+			time_t t;
+
+			/* FIXME: does this need locking?  Probably */
+
+			from = camel_mime_message_get_from (p->message);
+			fromstr = camel_address_format((CamelAddress *)from);
+			subject = camel_mime_message_get_subject (p->message);
+			
+			time (&t);
+			strftime (date, 49, "%a, %d %b %Y %H:%M:%S", localtime (&t));
+			fprintf (p->logfile, "Applied filter \"%s\" to message from %s - \"%s\" at %s\n",
+				 str, fromstr ? fromstr : "unknown", subject ? subject : "", date);
+			g_free(fromstr);
+			break;
+		}
+		case FILTER_LOG_ACTION:
+			fprintf (p->logfile, "Action: %s\n", str);
+			break;
+		case FILTER_LOG_END:
+			fprintf (p->logfile, "\n");
+			break;
+		default:
+			/* nothing else is loggable */
+			break;
+		}
+		
+		g_free (str);
+	}
+}
+
+
+/* will filter only an mbox - is more efficient as it doesn't need to open the folder through camel directly */
+void
+camel_filter_driver_filter_mbox (CamelFilterDriver *driver, const char *mbox, const char *source, CamelException *ex)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	CamelMimeParser *mp = NULL;
+	char *source_url = NULL;
+	int fd = -1;
+	int i = 0;
+	struct stat st;
+	
+	fd = open (mbox, O_RDONLY);
+	if (fd == -1) {
+		camel_exception_set (ex, CAMEL_EXCEPTION_SYSTEM, "Unable to open spool folder");
+		goto fail;
+	}
+	/* to get the filesize */
+	fstat (fd, &st);
+	
+	mp = camel_mime_parser_new ();
+	camel_mime_parser_scan_from (mp, TRUE);
+	if (camel_mime_parser_init_with_fd (mp, fd) == -1) {
+		camel_exception_set (ex, CAMEL_EXCEPTION_SYSTEM, "Unable to process spool folder");
+		goto fail;
+	}
+	fd = -1;
+	
+	source_url = g_strdup_printf ("file://%s", mbox);
+	
+	while (camel_mime_parser_step (mp, 0, 0) == HSCAN_FROM) {
+		CamelMimeMessage *msg;
+		int pc = 0;
+		
+		if (st.st_size > 0)
+			pc = (int)(100.0 * ((double)camel_mime_parser_tell (mp) / (double)st.st_size));
+		
+		report_status (driver, FILTER_STATUS_START, "Getting message %d (%d%% of file)", i, pc);
+		
+		msg = camel_mime_message_new ();
+		if (camel_mime_part_construct_from_parser (CAMEL_MIME_PART (msg), mp) == -1) {
+			report_status (driver, FILTER_STATUS_END, "Failed message %d", i);
+			camel_exception_set (ex, CAMEL_EXCEPTION_SYSTEM, "Cannot open message");
+			camel_object_unref (CAMEL_OBJECT (msg));
+			goto fail;
+		}
+		
+		camel_filter_driver_filter_message (driver, msg, NULL, source_url, source, ex);
+		camel_object_unref (CAMEL_OBJECT (msg));
+		if (camel_exception_is_set (ex)) {
+			report_status (driver, FILTER_STATUS_END, "Failed message %d", i);
+			goto fail;
+		}
+		
+		report_status (driver, FILTER_STATUS_END, "Finished message %d", i);
+		i++;
+		
+		/* skip over the FROM_END state */
+		camel_mime_parser_step (mp, 0, 0);
+	}
+	
+	if (p->defaultfolder)
+		camel_folder_sync (p->defaultfolder, FALSE, ex);
+	
+fail:
+	g_free (source_url);
+	if (fd != -1)
+		close (fd);
+	if (mp)
+		camel_object_unref (CAMEL_OBJECT (mp));
+}
+
+/* will filter a folder */
+void
+camel_filter_driver_filter_folder (CamelFilterDriver *driver, CamelFolder *folder, const char *source,
+				   GPtrArray *uids, gboolean remove, CamelException *ex)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	int i;
+	int freeuids = FALSE;
+	CamelMimeMessage *message;
+	CamelMessageInfo *info;
+	char *source_url, *service_url;
+	
+	service_url = camel_service_get_url (CAMEL_SERVICE (camel_folder_get_parent_store (folder)));
+	source_url = g_strdup_printf ("%s%s", service_url, camel_folder_get_full_name (folder));
+	g_free (service_url);
+	
+	if (uids == NULL) {
+		uids = camel_folder_get_uids (folder);
+		freeuids = TRUE;
+	}
+	
+	for (i = 0; i < uids->len; i++) {
+		report_status (driver, FILTER_STATUS_START, "Getting message %d of %d", i+1, uids->len);
+		
+		message = camel_folder_get_message (folder, uids->pdata[i], ex);
+		if (camel_exception_is_set (ex)) {
+			report_status (driver, FILTER_STATUS_END, "Failed at message %d of %d", i+1, uids->len);
+			break;
+		}
+		
+		if (camel_folder_has_summary_capability (folder))
+			info = camel_folder_get_message_info (folder, uids->pdata[i]);
+		else
+			info = NULL;
+		
+		camel_filter_driver_filter_message (driver, message, info, source_url, source, ex);
+
+		if (camel_folder_has_summary_capability (folder))
+			camel_folder_free_message_info (folder, info);
+
+		if (camel_exception_is_set (ex)) {
+			report_status (driver, FILTER_STATUS_END, "Failed at message %d of %d", i+1, uids->len);
+			break;
+		}
+		
+		if (remove)
+			camel_folder_set_message_flags (folder, uids->pdata[i],
+							CAMEL_MESSAGE_DELETED, CAMEL_MESSAGE_DELETED);
+		
+		camel_object_unref (CAMEL_OBJECT (message));
+	}
+	
+	if (freeuids)
+		camel_folder_free_uids (folder, uids);
+	
+	if (p->defaultfolder)
+		camel_folder_sync (p->defaultfolder, FALSE, ex);
+	
+	g_free (source_url);
+}
+
+void
+camel_filter_driver_filter_message (CamelFilterDriver *driver, CamelMimeMessage *message, CamelMessageInfo *info,
+				    const char *source_url, const char *source, CamelException *ex)
+{
+	struct _CamelFilterDriverPrivate *p = _PRIVATE (driver);
+	ESExpResult *r;
+	struct _filter_rule *node;
+	gboolean freeinfo = FALSE;
+	gboolean filtered = FALSE;
+	
+	if (info == NULL) {
+		struct _header_raw *h = CAMEL_MIME_PART (message)->headers;
+		
+		info = camel_message_info_new_from_header (h);
+		freeinfo = TRUE;
+	} else {
+		if (info->flags & CAMEL_MESSAGE_DELETED)
+			return;
+	}
+	
+	p->ex = ex;
+	p->terminated = FALSE;
+	p->deleted = FALSE;
+	p->copied = FALSE;
+	p->message = message;
+	p->info = info;
+
+	node = (struct _filter_rule *)p->rules.head;
+	while (node->next) {
+		d(fprintf (stderr, "applying rule %s\n action %s\n", node->match, node->action));
+		
+		if (camel_filter_search_match(p->message, p->info, source_url, node->match, p->ex)) {
+			filtered = TRUE;
+			camel_filter_driver_log (driver, FILTER_LOG_START, node->name);
+#ifndef NO_WARNINGS
+#warning "Must check expression parsed and executed properly?"
+#endif			
+			/* perform necessary filtering actions */
+			e_sexp_input_text (p->eval, node->action, strlen (node->action));
+			e_sexp_parse (p->eval);
+			r = e_sexp_eval (p->eval);
+			e_sexp_result_free (r);
+			if (p->terminated)
+				break;
+		}
+		node = node->next;
+	}
+	
+	/* Logic: if !Moved and there exists a default folder... */
+	if (!(p->copied && p->deleted) && p->defaultfolder) {
+		/* copy it to the default inbox */
+		filtered = TRUE;
+		camel_filter_driver_log (driver, FILTER_LOG_ACTION, "Copy to default folder");
+		camel_folder_append_message (p->defaultfolder, p->message, p->info, p->ex);
+	}
+	
+	/* *Now* we can set the DELETED flag... */
+	if (p->deleted)
+		info->flags = info->flags | CAMEL_MESSAGE_DELETED | CAMEL_MESSAGE_FOLDER_FLAGGED;
+	
+	if (freeinfo)
+		camel_message_info_free (info);
+	
+	if (filtered)
+		camel_filter_driver_log (driver, FILTER_LOG_END, NULL);
+}
diff --git a/camel/camel-filter-driver.h b/camel/camel-filter-driver.h
new file mode 100644
index 0000000000..0d3df60fe0
--- /dev/null
+++ b/camel/camel-filter-driver.h
@@ -0,0 +1,88 @@
+/*
+ *  Copyright (C) 2000 Helix Code Inc.
+ *
+ *  Authors: Michael Zucchi <notzed@helixcode.com>
+ *           Jeffrey Stedfast <fejj@helixcode.com>
+ *
+ *  This program is free software; you can redistribute it and/or 
+ *  modify it under the terms of the GNU General Public License as 
+ *  published by the Free Software Foundation; either version 2 of the
+ *  License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ *  USA
+ */
+
+#ifndef _CAMEL_FILTER_DRIVER_H
+#define _CAMEL_FILTER_DRIVER_H
+
+#include <camel/camel-object.h>
+#include <camel/camel-session.h>
+#include <camel/camel-folder.h>
+
+#define CAMEL_FILTER_DRIVER_TYPE   (camel_filter_driver_get_type())
+#define CAMEL_FILTER_DRIVER(obj)         CAMEL_CHECK_CAST (obj, camel_filter_driver_get_type (), CamelFilterDriver)
+#define CAMEL_FILTER_DRIVER_CLASS(klass) CAMEL__CHECK_CLASS_CAST (klass, camel_filter_driver_get_type (), CamelFilterDriverClass)
+#define CAMEL_IS_FILTER_DRIVER(obj)      CAMEL_CHECK_TYPE (obj, camel_filter_driver_get_type ())
+
+typedef struct _CamelFilterDriver      CamelFilterDriver;
+typedef struct _CamelFilterDriverClass CamelFilterDriverClass;
+
+struct _CamelFilterDriver {
+	CamelObject parent;
+
+	struct _CamelFilterDriverPrivate *priv;
+};
+
+struct _CamelFilterDriverClass {
+	CamelObjectClass parent_class;
+};
+
+/* FIXME: this maybe should change... */
+/* type of status for a status report */
+enum filter_status_t {
+	FILTER_STATUS_NONE,
+	FILTER_STATUS_START,	/* start of new message processed */
+	FILTER_STATUS_ACTION,	/* an action performed */
+	FILTER_STATUS_PROGRESS,	/* (an) extra update(s), if its taking longer to process */
+	FILTER_STATUS_END,	/* end of message */
+};
+
+typedef CamelFolder * (*FilterGetFolderFunc) (CamelFilterDriver *, const char *uri, void *data);
+/* report status */
+typedef void (FDStatusFunc)(CamelFilterDriver *driver, enum filter_status_t status, const char *desc, void *data);
+
+guint         camel_filter_driver_get_type (void);
+CamelFilterDriver  *camel_filter_driver_new     (FilterGetFolderFunc fetcher, void *data);
+
+/* modifiers */
+void    camel_filter_driver_set_logfile         (CamelFilterDriver *d, FILE *logfile);
+void	camel_filter_driver_set_status_func     (CamelFilterDriver *d, FDStatusFunc *func, void *data);
+void	camel_filter_driver_set_default_folder  (CamelFilterDriver *d, CamelFolder *def);
+void 	camel_filter_driver_add_rule		(CamelFilterDriver *d, const char *name, const char *match, const char *action);
+
+/*void camel_filter_driver_set_global(CamelFilterDriver *, const char *name, const char *value);*/
+
+void    camel_filter_driver_filter_message      (CamelFilterDriver *driver, CamelMimeMessage *message, CamelMessageInfo *info,
+						 const char *source_url, const char *source, CamelException *ex);
+void    camel_filter_driver_filter_mbox         (CamelFilterDriver *driver, const char *mbox, const char *source, CamelException *ex);
+void    camel_filter_driver_filter_folder       (CamelFilterDriver *driver, CamelFolder *folder, const char *source,
+						 GPtrArray *uids, gboolean remove, CamelException *ex);
+
+#if 0
+/* generate the search query/action string for a filter option */
+void camel_filter_driver_expand_option (CamelFilterDriver *d, GString *s, GString *action, struct filter_option *op);
+
+/* get info about rules (options) */
+int camel_filter_driver_rule_count (CamelFilterDriver *d);
+struct filter_option *camel_filter_driver_rule_get (CamelFilterDriver *d, int n);
+#endif
+
+#endif /* ! _CAMEL_FILTER_DRIVER_H */
diff --git a/camel/camel-filter-search.c b/camel/camel-filter-search.c
new file mode 100644
index 0000000000..5663bf622f
--- /dev/null
+++ b/camel/camel-filter-search.c
@@ -0,0 +1,630 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
+ *  Authors: Jeffrey Stedfast <fejj@helixcode.com>
+ *	     Michael Zucchi <NotZed@Ximian.com>
+ *
+ *  Copyright 2000 Helix Code, Inc. (www.helixcode.com)
+ *  Copyright 2001 Ximian Inc. (www.ximian.com)
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
+ *
+ */
+
+/* (from glibc headers:
+   POSIX says that <sys/types.h> must be included (by the caller) before <regex.h>.  */
+#include <sys/types.h>
+#include <regex.h>
+#include <string.h>
+#include <ctype.h>
+
+#warning "Fixme: remove gal/widgets/e-unicode dependency"
+#include <gal/widgets/e-unicode.h>
+
+#include "e-util/e-sexp.h"
+
+#include "camel-mime-message.h"
+#include "camel-filter-search.h"
+#include "camel-exception.h"
+#include "camel-multipart.h"
+#include "camel-stream-mem.h"
+
+#define d(x)
+
+typedef struct {
+	CamelMimeMessage *message;
+	CamelMessageInfo *info;
+	const char *source;
+	CamelException *ex;
+} FilterMessageSearch;
+
+/* ESExp callbacks */
+static ESExpResult *header_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_matches (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_starts_with (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_ends_with (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_exists (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_soundex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_regex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *header_full_regex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *match_all (struct _ESExp *f, int argc, struct _ESExpTerm **argv, FilterMessageSearch *fms);
+static ESExpResult *body_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *body_regex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *user_flag (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *user_tag (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *system_flag (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *get_sent_date (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *get_received_date (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *get_current_date (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *get_score (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+static ESExpResult *get_source (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms);
+
+/* builtin functions */
+static struct {
+	char *name;
+	ESExpFunc *func;
+	int type;		/* set to 1 if a function can perform shortcut evaluation, or
+				   doesn't execute everything, 0 otherwise */
+} symbols[] = {
+	{ "match-all",          (ESExpFunc *) match_all,          1 },
+	{ "body-contains",      (ESExpFunc *) body_contains,      0 },
+	{ "body-regex",         (ESExpFunc *) body_regex,         0 },
+	{ "header-contains",    (ESExpFunc *) header_contains,    0 },
+	{ "header-matches",     (ESExpFunc *) header_matches,     0 },
+	{ "header-starts-with", (ESExpFunc *) header_starts_with, 0 },
+	{ "header-ends-with",   (ESExpFunc *) header_ends_with,   0 },
+	{ "header-exists",      (ESExpFunc *) header_exists,      0 },
+	{ "header-soundex",     (ESExpFunc *) header_soundex,     0 },
+	{ "header-regex",       (ESExpFunc *) header_regex,       0 },
+	{ "header-full-regex",  (ESExpFunc *) header_full_regex,  0 },
+	{ "user-tag",           (ESExpFunc *) user_tag,           0 },
+	{ "user-flag",          (ESExpFunc *) user_flag,          0 },
+	{ "system-flag",        (ESExpFunc *) system_flag,        0 },
+	{ "get-sent-date",      (ESExpFunc *) get_sent_date,      0 },
+	{ "get-received-date",  (ESExpFunc *) get_received_date,  0 },
+	{ "get-current-date",   (ESExpFunc *) get_current_date,   0 },
+	{ "get-score",          (ESExpFunc *) get_score,          0 },
+	{ "get-source",         (ESExpFunc *) get_source,         0 },
+};
+
+/* builds the regex into pattern */
+/* taken from camel-folder-search, with added isregex & exception parameter */
+/* Basically, we build a new regex, either based on subset regex's, or substrings,
+   that can be executed once over the whoel body, to match anything suitable.
+   This is more efficient than multiple searches, and probably most (naive) strstr
+   implementations, over long content.
+
+   A small issue is that case-insenstivity wont work entirely correct for utf8 strings. */
+static int
+build_match_regex(regex_t *pattern, int isregex, int argc, struct _ESExpResult **argv, CamelException *ex)
+{
+	GString *match = g_string_new("");
+	int c, i, count=0, err;
+	char *word;
+
+	/* build a regex pattern we can use to match the words, we OR them together */
+	if (argc>1)
+		g_string_append_c(match, '(');
+	for (i=0;i<argc;i++) {
+		if (argv[i]->type == ESEXP_RES_STRING) {
+			if (count > 0)
+				g_string_append_c(match, '|');
+			/* escape any special chars (not sure if this list is complete) */
+			word = argv[i]->value.string;
+			if (isregex) {
+				g_string_append(match, word);
+			} else {
+				while ((c = *word++)) {
+					if (strchr("*\\.()[]^$+", c) != NULL) {
+						g_string_append_c(match, '\\');
+					}
+					g_string_append_c(match, c);
+				}
+			}
+			count++;
+		} else {
+			g_warning("Invalid type passed to body-contains match function");
+		}
+	}
+	if (argc>1)
+		g_string_append_c(match, ')');
+	err = regcomp(pattern, match->str, REG_EXTENDED|REG_ICASE|REG_NOSUB);
+	if (err != 0) {
+		/* regerror gets called twice to get the full error string 
+		   length to do proper posix error reporting */
+		int len = regerror(err, pattern, 0, 0);
+		char *buffer = g_malloc0(len + 1);
+
+		regerror(err, pattern, buffer, len);
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
+				     _("Regular expression compilation failed: %s: %s"),
+				     match->str, buffer);
+
+		regfree(pattern);
+	}
+	d(printf("Built regex: '%s'\n", match->str));
+	g_string_free(match, TRUE);
+	return err;
+}
+
+static unsigned char soundex_table[256] = {
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0, 49, 50, 51,  0, 49, 50,  0,  0, 50, 50, 52, 53, 53,  0,
+	 49, 50, 54, 50, 51,  0, 49,  0, 50,  0, 50,  0,  0,  0,  0,  0,
+	  0,  0, 49, 50, 51,  0, 49, 50,  0,  0, 50, 50, 52, 53, 53,  0,
+	 49, 50, 54, 50, 51,  0, 49,  0, 50,  0, 50,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+	  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,
+};
+
+static void
+soundexify (const gchar *sound, gchar code[5])
+{
+	guchar *c, last = '\0';
+	gint n;
+	
+	for (c = (guchar *) sound; *c && !isalpha (*c); c++);
+	code[0] = toupper (*c);
+	memset (code + 1, '0', 3);
+	for (n = 1; *c && n < 5; c++) {
+		guchar ch = soundex_table[*c];
+		
+		if (ch && ch != last) {
+			code[n++] = ch;
+			last = ch;
+		}
+	}
+	code[4] = '\0';
+}
+
+static gint
+soundexcmp (const gchar *sound1, const gchar *sound2)
+{
+	gchar code1[5], code2[5];
+	
+	soundexify (sound1, code1);
+	soundexify (sound2, code2);
+	
+	return strcmp (code1, code2);
+}
+
+static gboolean check_match(const char *value, const char *match, int how)
+{
+	const char *p;
+
+	while (*value && isspace(*value))
+		value++;
+
+	if (strlen(value) < strlen(match))
+		return FALSE;
+
+	/* from dan the man, if we have mixed case, perform a case-sensitive match,
+	   otherwise not */
+	p = match;
+	while (*p) {
+		if (isupper(*p)) {
+			switch(how) {
+			case 0:	/* is */
+				return strcmp(value, match) == 0;
+			case 1:	/* contains */
+				return strstr(value, match) != NULL;
+			case 2:	/* starts with */
+				return strncmp(value, match, strlen(match)) == 0;
+			case 3:	/* ends with */
+				return strcmp(value+strlen(value)-strlen(match), match) == 0;
+			case 4:	/* soundex */
+				return soundexcmp(value, match) == 0;
+			}
+			return FALSE;
+		}
+		p++;
+	}
+	switch(how) {
+	case 0:	/* is */
+		return strcasecmp(value, match) == 0;
+	case 1:	/* contains */
+		return e_utf8_strstrcase(value, match) != NULL;
+	case 2:	/* starts with */
+		return strncasecmp(value, match, strlen(match)) == 0;
+	case 3:	/* ends with */
+		return strcasecmp(value+strlen(value)-strlen(match), match) == 0;
+	case 4:	/* soundex */
+		return soundexcmp(value, match) == 0;
+	}
+
+	return FALSE;
+}
+
+static ESExpResult *
+check_header(struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms, int how)
+{
+	gboolean matched = FALSE;
+	ESExpResult *r;
+	int i;
+
+	if (argc > 1 && argv[0]->type == ESEXP_RES_STRING) {
+		const char *header = camel_medium_get_header (CAMEL_MEDIUM (fms->message), argv[0]->value.string);
+
+		if (header) {
+			for (i=1;i<argc && !matched;i++) {
+				if (argv[i]->type == ESEXP_RES_STRING
+				    && check_match(header, argv[i]->value.string, how)) {
+					matched = TRUE;
+					break;
+				}
+			}
+		}
+	}
+	
+	r = e_sexp_result_new (ESEXP_RES_BOOL);
+	r->value.bool = matched;
+	
+	return r;
+}
+
+static ESExpResult *
+header_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	return check_header(f, argc, argv, fms, 1);
+}
+
+
+static ESExpResult *
+header_matches (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	return check_header(f, argc, argv, fms, 0);
+}
+
+static ESExpResult *
+header_starts_with (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	return check_header(f, argc, argv, fms, 2);
+}
+
+static ESExpResult *
+header_ends_with (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	return check_header(f, argc, argv, fms, 3);
+}
+
+static ESExpResult *
+header_soundex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	return check_header(f, argc, argv, fms, 4);
+}
+
+static ESExpResult *
+header_exists (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	gboolean matched = FALSE;
+	ESExpResult *r;
+	int i;
+
+	for (i=0;i<argc && !matched;i++) {
+		if (argv[i]->type == ESEXP_RES_STRING)
+			matched = camel_medium_get_header (CAMEL_MEDIUM (fms->message), argv[i]->value.string) != NULL;
+	}
+	
+	r = e_sexp_result_new (ESEXP_RES_BOOL);
+	r->value.bool = matched;
+	
+	return r;
+}
+
+static ESExpResult *
+header_regex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r = e_sexp_result_new (ESEXP_RES_BOOL);
+	regex_t pattern;
+	const char *contents;
+
+	if (argc>1
+	    && argv[0]->type == ESEXP_RES_STRING
+	    && (contents = camel_medium_get_header (CAMEL_MEDIUM (fms->message), argv[0]->value.string))
+	    && build_match_regex(&pattern, TRUE, argc-1, argv+1, fms->ex) == 0) {
+		r->value.bool = regexec(&pattern, contents, 0, NULL, 0) == 0;
+		regfree(&pattern);
+	} else
+		r->value.bool = FALSE;
+	
+	return r;
+}
+
+static gchar *
+get_full_header (CamelMimeMessage *message)
+{
+	CamelMimePart *mp = CAMEL_MIME_PART (message);
+	GString *str = g_string_new ("");
+	char   *ret;
+	struct _header_raw *h;
+	
+	for (h = mp->headers; h; h = h->next) {
+		if (h->value != NULL) {
+			g_string_append(str, h->name);
+			if (isspace(h->value[0]))
+				g_string_append(str, ":");
+			else
+				g_string_append(str, ": ");
+			g_string_append(str, h->value);
+		}
+	}
+	
+	ret = str->str;
+	g_string_free (str, FALSE);
+	
+	return ret;
+}
+
+static ESExpResult *
+header_full_regex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r = e_sexp_result_new (ESEXP_RES_BOOL);
+	regex_t pattern;
+	char *contents;
+
+	if (build_match_regex(&pattern, TRUE, argc, argv, fms->ex) == 0) {
+		contents = get_full_header (fms->message);
+		r->value.bool = regexec(&pattern, contents, 0, NULL, 0) == 0;
+		g_free(contents);
+		regfree(&pattern);
+	} else
+		r->value.bool = FALSE;
+	
+	return r;
+}
+
+static ESExpResult *
+match_all (struct _ESExp *f, int argc, struct _ESExpTerm **argv, FilterMessageSearch *fms)
+{
+	/* match-all: when dealing with single messages is a no-op */
+	ESExpResult *r;
+	
+	if (argc > 0)
+		return e_sexp_term_eval(f, argv[0]);
+
+	r = e_sexp_result_new (ESEXP_RES_BOOL);
+	r->value.bool = FALSE;
+
+	return r;
+}
+
+/* performs a 'slow' content-based match */
+/* taken directly from camel-folder-search.c */
+static gboolean
+message_body_contains(CamelDataWrapper *object, regex_t *pattern)
+{
+	CamelDataWrapper *containee;
+	int truth = FALSE;
+	int parts, i;
+
+	containee = camel_medium_get_content_object(CAMEL_MEDIUM(object));
+
+	if (containee == NULL)
+		return FALSE;
+
+	/* TODO: I find it odd that get_part and get_content_object do not
+	   add a reference, probably need fixing for multithreading */
+
+	/* using the object types is more accurate than using the mime/types */
+	if (CAMEL_IS_MULTIPART(containee)) {
+		parts = camel_multipart_get_number(CAMEL_MULTIPART(containee));
+		for (i=0;i<parts && truth==FALSE;i++) {
+			CamelDataWrapper *part = (CamelDataWrapper *)camel_multipart_get_part(CAMEL_MULTIPART(containee), i);
+			if (part) {
+				truth = message_body_contains(part, pattern);
+			}
+		}
+	} else if (CAMEL_IS_MIME_MESSAGE(containee)) {
+		/* for messages we only look at its contents */
+		truth = message_body_contains((CamelDataWrapper *)containee, pattern);
+	} else if (header_content_type_is(CAMEL_DATA_WRAPPER(containee)->mime_type, "text", "*")) {
+		/* for all other text parts, we look inside, otherwise we dont care */
+		CamelStreamMem *mem = (CamelStreamMem *)camel_stream_mem_new();
+
+		camel_data_wrapper_write_to_stream(containee, (CamelStream *)mem);
+		camel_stream_write((CamelStream *)mem, "", 1);
+		truth = regexec(pattern, mem->buffer->data, 0, NULL, 0) == 0;
+		camel_object_unref((CamelObject *)mem);
+	}
+	return truth;
+}
+
+static ESExpResult *
+body_contains (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r = e_sexp_result_new (ESEXP_RES_BOOL);
+	regex_t pattern;
+
+	if (build_match_regex(&pattern, FALSE, argc, argv, fms->ex) == 0) {
+		r->value.bool = message_body_contains((CamelDataWrapper *)fms->message, &pattern);
+		regfree(&pattern);
+	} else
+		r->value.bool = FALSE;
+	
+	return r;
+}
+
+static ESExpResult *
+body_regex (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r = e_sexp_result_new (ESEXP_RES_BOOL);
+	regex_t pattern;
+
+	if (build_match_regex(&pattern, TRUE, argc, argv, fms->ex) == 0) {
+		r->value.bool = message_body_contains((CamelDataWrapper *)fms->message, &pattern);
+		regfree(&pattern);
+	} else
+		r->value.bool = FALSE;
+	
+	return r;
+}
+
+static ESExpResult *
+user_flag (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	gboolean truth = FALSE;
+	int i;
+	
+	/* performs an OR of all words */
+	for (i = 0; i < argc && !truth; i++) {
+		if (argv[i]->type == ESEXP_RES_STRING
+		    && camel_flag_get (&fms->info->user_flags, argv[i]->value.string)) {
+			truth = TRUE;
+			break;
+		}
+	}
+	
+	r = e_sexp_result_new (ESEXP_RES_BOOL);
+	r->value.bool = truth;
+	
+	return r;
+}
+
+static ESExpResult *
+system_flag (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	gboolean truth = FALSE;
+	
+	if (argc == 1)
+		truth = camel_system_flag_get (fms->info->flags, argv[0]->value.string);
+	
+	r = e_sexp_result_new (ESEXP_RES_BOOL);
+	r->value.bool = truth;
+	
+	return r;
+}
+
+static ESExpResult *
+user_tag (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	const char *tag;
+	
+	tag = camel_tag_get (&fms->info->user_tags, argv[0]->value.string);
+	
+	r = e_sexp_result_new (ESEXP_RES_STRING);
+	r->value.string = g_strdup (tag ? tag : "");
+	
+	return r;
+}
+
+static ESExpResult *
+get_sent_date (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	
+	r = e_sexp_result_new(ESEXP_RES_INT);
+	r->value.number = camel_mime_message_get_date(fms->message, NULL);
+	
+	return r;
+}
+
+static ESExpResult *
+get_received_date (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	
+	r = e_sexp_result_new(ESEXP_RES_INT);
+	r->value.number = camel_mime_message_get_date_received(fms->message, NULL);
+	
+	return r;
+}
+
+static ESExpResult *
+get_current_date (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	
+	r = e_sexp_result_new (ESEXP_RES_INT);
+	r->value.number = time (NULL);
+	
+	return r;
+}
+
+static ESExpResult *
+get_score (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	const char *tag;
+	
+	tag = camel_tag_get (&fms->info->user_tags, "score");
+	
+	r = e_sexp_result_new (ESEXP_RES_INT);
+	if (tag)
+		r->value.number = atoi (tag);
+	else
+		r->value.number = 0;
+	
+	return r;
+}
+
+static ESExpResult *
+get_source (struct _ESExp *f, int argc, struct _ESExpResult **argv, FilterMessageSearch *fms)
+{
+	ESExpResult *r;
+	
+	r = e_sexp_result_new (ESEXP_RES_STRING);
+	r->value.string = g_strdup (fms->source);
+	
+	return r;
+}
+
+gboolean camel_filter_search_match(CamelMimeMessage *message, CamelMessageInfo *info,
+				   const char *source, const char *expression, CamelException *ex)
+{
+	FilterMessageSearch fms;
+	ESExp *sexp;
+	ESExpResult *result;
+	gboolean retval;
+	int i;
+	
+	fms.message = message;
+	fms.info = info;
+	fms.source = source;
+	fms.ex = ex;
+	
+	sexp = e_sexp_new ();
+	
+	for (i = 0; i < sizeof (symbols) / sizeof (symbols[0]); i++) {
+		if (symbols[i].type == 1)
+			e_sexp_add_ifunction (sexp, 0, symbols[i].name, (ESExpIFunc *)symbols[i].func, &fms);
+		else
+			e_sexp_add_function (sexp, 0, symbols[i].name, symbols[i].func, &fms);
+	}
+	
+	e_sexp_input_text (sexp, expression, strlen (expression));
+	e_sexp_parse (sexp);
+	result = e_sexp_eval (sexp);
+	
+	if (result->type == ESEXP_RES_BOOL)
+		retval = result->value.bool;
+	else
+		retval = FALSE;
+	
+	e_sexp_unref(sexp);
+	e_sexp_result_free (result);
+	
+	return retval;
+}
diff --git a/camel/camel-filter-search.h b/camel/camel-filter-search.h
new file mode 100644
index 0000000000..0dba92eb2d
--- /dev/null
+++ b/camel/camel-filter-search.h
@@ -0,0 +1,44 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
+ *  Authors: Jeffrey Stedfast <fejj@helixcode.com>
+ *	     Michael Zucchi <NotZed@Ximian.com>
+ *
+ *  Copyright 2000 Helix Code, Inc. (www.helixcode.com)
+ *  Copyright 2001 Ximian Inc. (www.ximian.com)
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
+ *
+ */
+
+#ifndef CAMEL_FILTER_SEARCH_H
+#define CAMEL_FILTER_SEARCH_H
+
+#ifdef __cplusplus
+extern "C" {
+#pragma }
+#endif /* __cplusplus */
+
+#include <glib.h>
+#include <camel/camel-mime-message.h>
+#include <camel/camel-folder-summary.h>
+
+gboolean camel_filter_search_match(CamelMimeMessage *message, CamelMessageInfo *info,
+				   const char *source, const char *expression, CamelException *ex); 
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* ! CAMEL_FILTER_SEARCH_H */
diff --git a/camel/camel-folder-search.c b/camel/camel-folder-search.c
index bc9e13cb34..c1023b5f41 100644
--- a/camel/camel-folder-search.c
+++ b/camel/camel-folder-search.c
@@ -30,6 +30,7 @@
 #include <sys/types.h>
 #include <regex.h>
 
+#warning "Fixme: remove gal/widgets/e-unicode dependency"
 #include <gal/widgets/e-unicode.h>
 #include "camel-folder-search.h"
 #include "string-utils.h"
@@ -800,6 +801,7 @@ g_lib_sux_htor(char *key, int value, struct _glib_sux_donkeys *fuckup)
 }
 
 /* performs a 'slow' content-based match */
+/* there is also an identical copy of this in camel-filter-search.c */
 static gboolean
 message_body_contains(CamelDataWrapper *object, regex_t *pattern)
 {
diff --git a/camel/camel-folder-summary.c b/camel/camel-folder-summary.c
index c948d6865f..52c9a4aa46 100644
--- a/camel/camel-folder-summary.c
+++ b/camel/camel-folder-summary.c
@@ -347,6 +347,55 @@ camel_folder_summary_index(CamelFolderSummary *s, int i)
 }
 
 /**
+ * camel_folder_summary_index:
+ * @s: 
+ * @i: 
+ * 
+ * Obtain a copy of the summary array.  This is done atomically,
+ * so cannot contain empty entries.
+ *
+ * It must be freed using camel_folder_summary_array_free().
+ **/
+GPtrArray *
+camel_folder_summary_array(CamelFolderSummary *s)
+{
+	CamelMessageInfo *info;
+	GPtrArray *res = g_ptr_array_new();
+
+	CAMEL_SUMMARY_LOCK(s, ref_lock);
+	CAMEL_SUMMARY_LOCK(s, summary_lock);
+
+	g_ptr_array_set_size(res, s->messages->len);
+	for (i=0;i<s->messages->len;i++) {
+		info = res->pdata[i] = g_ptr_array_index(s->messages, i);
+		info->refcount++;
+	}
+
+	CAMEL_SUMMARY_UNLOCK(s, summary_lock);
+	CAMEL_SUMMARY_UNLOCK(s, ref_lock);
+
+	return res;
+}
+
+/**
+ * camel_folder_summary_array_free:
+ * @s: 
+ * @array: 
+ * 
+ * Free the folder summary array.
+ **/
+void
+camel_folder_summary_array_free(CamelFolderSummary *s, GPtrArray *array)
+{
+	int i;
+
+	for (i=0;i<array->len;i++)
+		camel_folder_summary_info_free(s, array->pdata[i]);
+
+	g_ptr_array_free(array, TRUE);
+}
+
+/**
  * camel_folder_summary_uid:
  * @s: 
  * @uid: 
diff --git a/camel/camel-folder-summary.h b/camel/camel-folder-summary.h
index 4d07951d4b..860b9cb8d8 100644
--- a/camel/camel-folder-summary.h
+++ b/camel/camel-folder-summary.h
@@ -247,6 +247,8 @@ void camel_folder_summary_clear(CamelFolderSummary *s);
 int camel_folder_summary_count(CamelFolderSummary *);
 CamelMessageInfo *camel_folder_summary_index(CamelFolderSummary *, int);
 CamelMessageInfo *camel_folder_summary_uid(CamelFolderSummary *, const char *uid);
+GPtrArray *camel_folder_summary_array(CamelFolderSummary *s);
+void camel_folder_summary_array_free(CamelFolderSummary *s, GPtrArray *array);
 
 /* summary formatting utils */
 char *camel_folder_summary_format_address(struct _header_raw *h, const char *name);
diff --git a/camel/camel-folder.c b/camel/camel-folder.c
index 682a4902f0..17d65097b5 100644
--- a/camel/camel-folder.c
+++ b/camel/camel-folder.c
@@ -927,14 +927,9 @@ get_summary(CamelFolder *folder)
 	GPtrArray *res = g_ptr_array_new();
 	int i, count;
 
-	g_return_val_if_fail(folder->summary != NULL, res);
+	g_assert(folder->summary != NULL);
 
-	count = camel_folder_summary_count(folder->summary);
-	g_ptr_array_set_size(res, count);
-	for (i=0;i<count;i++)
-		res->pdata[i] = camel_folder_summary_index(folder->summary, i);
-
-	return res;
+	return camel_folder_summary_array(folder->summary);
 }
 
 /**
@@ -964,12 +959,9 @@ free_summary(CamelFolder *folder, GPtrArray *summary)
 {
 	int i;
 
-	g_return_if_fail(folder->summary != NULL);
+	g_assert(folder->summary != NULL);
 
-	for (i=0;i<summary->len;i++)
-		camel_folder_summary_info_free(folder->summary, summary->pdata[i]);
-
-	g_ptr_array_free(summary, TRUE);
+	camel_folder_summary_array_free(folder->summary, summary);
 }
 
 /**
@@ -1095,6 +1087,8 @@ copy_message_to (CamelFolder *source, const char *uid, CamelFolder *dest, CamelE
  * This copies a message from one folder to another. If the @source and
  * @dest folders have the same parent_store, this may be more efficient
  * than a camel_folder_append_message().
+ *
+ * This function is still depcreated, it is not the same as move_message_to.
  **/
 void
 camel_folder_copy_message_to (CamelFolder *source, const char *uid,
diff --git a/camel/camel-session.c b/camel/camel-session.c
index 1c3f108616..d32ec547a7 100644
--- a/camel/camel-session.c
+++ b/camel/camel-session.c
@@ -43,6 +43,8 @@
 
 #include "camel-private.h"
 
+#define d(x)
+
 static CamelObjectClass *parent_class;
 
 static void
@@ -641,7 +643,7 @@ void camel_cancel_cancel(CamelCancel *cc)
 			CAMEL_ACTIVE_UNLOCK();
 		}
 	} else if ((cc->flags & CAMEL_CANCEL_CANCELLED) == 0) {
-		printf("cancelling thread %d\n", cc->id);
+		d(printf("cancelling thread %d\n", cc->id));
 
 		CAMEL_CANCEL_LOCK(cc);
 		msg = g_malloc0(sizeof(*msg));
@@ -671,7 +673,7 @@ void camel_cancel_register(CamelCancel *cc)
 	cc->id = id;
 	g_hash_table_insert(cancel_active, (void *)id, cc);
 
-	printf("registering thread %d for cancellation\n", id);
+	d(printf("registering thread %d for cancellation\n", id));
 
 	CAMEL_ACTIVE_UNLOCK();
 
@@ -698,8 +700,7 @@ void camel_cancel_unregister(CamelCancel *cc)
 
 	CAMEL_ACTIVE_UNLOCK();
 
-	if (cc)
-		printf("unregistering thread %d for cancellation\n", cc->id);
+	d({if (cc) printf("unregistering thread %d for cancellation\n", cc->id)});
 
 	if (cc)
 		camel_cancel_unref(cc);
@@ -710,7 +711,7 @@ gboolean camel_cancel_check(CamelCancel *cc)
 {
 	CamelCancelMsg *msg;
 
-	printf("checking for cancel in thread %d\n", pthread_self());
+	d(printf("checking for cancel in thread %d\n", pthread_self()));
 
 	if (cc == NULL) {
 		if (cancel_active) {
@@ -723,18 +724,18 @@ gboolean camel_cancel_check(CamelCancel *cc)
 	}
 
 	if (cc->blocked > 0) {
-		printf("ahah!  cancellation is blocked\n");
+		d(printf("ahah!  cancellation is blocked\n"));
 		return FALSE;
 	}
 
 	if (cc->flags & CAMEL_CANCEL_CANCELLED) {
-		printf("previously cancelled\n");
+		d(printf("previously cancelled\n"));
 		return TRUE;
 	}
 
 	msg = (CamelCancelMsg *)e_msgport_get(cc->cancel_port);
 	if (msg) {
-		printf("Got cancellation message\n");
+		d(printf("Got cancellation message\n"));
 		CAMEL_CANCEL_LOCK(cc);
 		cc->flags |= CAMEL_CANCEL_CANCELLED;
 		CAMEL_CANCEL_UNLOCK(cc);
diff --git a/camel/tests/folder/Makefile.am b/camel/tests/folder/Makefile.am
index 4401d310f6..f6dd852dd8 100644
--- a/camel/tests/folder/Makefile.am
+++ b/camel/tests/folder/Makefile.am
@@ -16,12 +16,14 @@ check_PROGRAMS =  	\
 	test1	test4	test5	\
 	test2	test6	test7	\
 	test3			\
-	test8
+	test8			\
+	test9
 
 TESTS = test1 test4 test5 \
 	test2 test6 test7 \
 	test3		  \
-	test8
+	test8		  \
+	test9
 
 
 
diff --git a/camel/tests/folder/README b/camel/tests/folder/README
index e02308d2d7..4fed421413 100644
--- a/camel/tests/folder/README
+++ b/camel/tests/folder/README
@@ -8,4 +8,4 @@ test6	basic folder operations, IMAP
 test7	basic folder operations, NNTP
 
 test8	multithreaded folder torture test, local
-
+test9	filtering
diff --git a/camel/tests/folder/test8.c b/camel/tests/folder/test8.c
index 30552cf782..5665f96f2a 100644
--- a/camel/tests/folder/test8.c
+++ b/camel/tests/folder/test8.c
@@ -107,6 +107,7 @@ worker(void *d)
 		content = g_strdup_printf("Test message %d contents\n\n", id+i);
 		test_message_compare_content(camel_medium_get_content_object((CamelMedium *)msg), content, strlen(content));
 		test_free(content);
+		pull();
 
 		push("deleting message, cleanup");
 		j=(100.0*rand()/(RAND_MAX+1.0));
diff --git a/camel/tests/folder/test9.c b/camel/tests/folder/test9.c
new file mode 100644
index 0000000000..68c761c40d
--- /dev/null
+++ b/camel/tests/folder/test9.c
@@ -0,0 +1,176 @@
+/* folder/index testing */
+
+#include "camel-test.h"
+#include "messages.h"
+#include "folders.h"
+
+#include "camel/camel-exception.h"
+#include "camel/camel-service.h"
+#include "camel/camel-session.h"
+#include "camel/camel-store.h"
+
+#include "camel/camel-folder.h"
+#include "camel/camel-folder-summary.h"
+#include "camel/camel-mime-message.h"
+#include "camel/camel-filter-driver.h"
+#include "camel/camel-stream-fs.h"
+
+#define ARRAY_LEN(x) (sizeof(x)/sizeof(x[0]))
+
+
+/* god, who designed this horrid interface */
+static char *auth_callback(CamelAuthCallbackMode mode,
+			   char *data, gboolean secret,
+			   CamelService *service, char *item,
+			   CamelException *ex)
+{
+	return NULL;
+}
+
+struct {
+	char *name;
+	CamelFolder *folder;
+} mailboxes[] = {
+	{ "INBOX", NULL },
+	{ "folder1", NULL },
+	{ "folder2", NULL },
+	{ "folder3", NULL },
+	{ "folder4", NULL },
+};
+
+struct {
+	char *name, *match, *action;
+} rules[] = {
+	{ "empty1", "(match-all (header-contains \"Frobnitz\"))", "(copy-to \"folder1\")" },
+	{ "empty2", "(header-contains \"Frobnitz\")", "(copy-to \"folder2\")" },
+	{ "count11", "(and (header-contains \"subject\" \"Test1\") (header-contains \"subject\" \"subject\"))", "(move-to \"folder3\")" },
+	{ "empty3", "(and (header-contains \"subject\" \"Test1\") (header-contains \"subject\" \"subject\"))", "(move-to \"folder4\")" },
+	{ "count1", "(body-contains \"data50\")", "(copy-to \"folder1\")" },
+	{ "stop", "(body-contains \"data2\")", "(stop)" },
+	{ "notreached1", "(body-contains \"data2\")", "(move-to \"folder2\")" },
+	{ "count1", "(body-contains \"data3\")", "(move-to \"folder2\")" },
+};
+
+
+static CamelFolder *get_folder(CamelFilterDriver *d, const char *uri, void *data)
+{
+	int i;
+
+	for (i=0;i<ARRAY_LEN(mailboxes);i++)
+		if (!strcmp(mailboxes[i].name, uri)) {
+			camel_object_ref((CamelObject *)mailboxes[i].folder);
+			return mailboxes[i].folder;
+		}
+	return NULL;
+}
+
+int main(int argc, char **argv)
+{
+	CamelSession *session;
+	CamelStore *store;
+	CamelException *ex;
+	CamelFolder *folder;
+	CamelMimeMessage *msg;
+	int i, j;
+	CamelStream *mbox;
+	CamelFilterDriver *driver;
+
+	gtk_init(&argc, &argv);
+
+	camel_test_init(argc, argv);
+
+	ex = camel_exception_new();
+
+	/* clear out any camel-test data */
+	system("/bin/rm -rf /tmp/camel-test");
+
+	camel_test_start("Simple filtering of mbox");
+
+	session = camel_session_new("/tmp/camel-test", auth_callback, NULL, NULL);
+
+	/* todo: cross-check everything with folder_info checks as well */
+	/* todo: work out how to do imap/pop/nntp tests */
+
+	push("getting store");
+	store = camel_session_get_store(session, "mbox:///tmp/camel-test/mbox", ex);
+	check_msg(!camel_exception_is_set(ex), "getting store: %s", camel_exception_get_description(ex));
+	check(store != NULL);
+	pull();
+
+	push("Creating output folders");
+	for (i=0;i<ARRAY_LEN(mailboxes);i++) {
+		push("creating %s", mailboxes[i].name);
+		mailboxes[i].folder = folder = camel_store_get_folder(store, mailboxes[i].name, CAMEL_STORE_FOLDER_CREATE, ex);
+		check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+		check(folder != NULL);
+		
+		/* we need an empty folder for this to work */
+		test_folder_counts(folder, 0, 0);
+		pull();
+	}
+	pull();
+
+	/* append a bunch of messages with specific content */
+	push("creating 100 test message mbox");
+	mbox = camel_stream_fs_new_with_name("/tmp/camel-test/inbox", O_WRONLY|O_CREAT|O_EXCL, 0600);
+	for (j=0;j<100;j++) {
+		char *content, *subject;
+			
+		push("creating test message");
+		msg = test_message_create_simple();
+		content = g_strdup_printf("data%d content\n", j);
+		test_message_set_content_simple((CamelMimePart *)msg, 0, "text/plain",
+						content, strlen(content));
+		test_free(content);
+		subject = g_strdup_printf("Test%d message%d subject", j, 100-j);
+		camel_mime_message_set_subject(msg, subject);
+		
+		camel_mime_message_set_date(msg, j*60*24, 0);
+		pull();
+		
+		camel_stream_printf(mbox, "From \n");
+		check(camel_data_wrapper_write_to_stream((CamelDataWrapper *)msg, mbox) != -1);
+#if 0		
+		push("appending simple message %d", j);
+		camel_folder_append_message(folder, msg, NULL, ex);
+		check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+		pull();
+#endif				
+		test_free(subject);
+		
+		check_unref(msg, 1);
+	}
+	check(camel_stream_close(mbox) != -1);
+	check_unref(mbox, 1);
+	pull();
+
+	push("Building filters");
+	driver = camel_filter_driver_new(get_folder, NULL);
+	for (i=0;i<ARRAY_LEN(rules);i++) {
+		camel_filter_driver_add_rule(driver, rules[i].name, rules[i].match, rules[i].action);
+	}
+	pull();
+
+	push("Executing filters");
+	camel_filter_driver_set_default_folder(driver, mailboxes[0].folder);
+	camel_filter_driver_filter_mbox(driver, "/tmp/camel-test/inbox", "", ex);
+	check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+
+	/* now need to check the folder counts/etc */
+
+	check_unref(driver, 1);
+	pull();
+
+	for (i=0;i<ARRAY_LEN(mailboxes);i++) {
+		check_unref(mailboxes[i].folder, 1);
+	}
+
+	check_unref(store, 1);
+
+	check_unref(session, 1);
+	camel_exception_free(ex);
+
+	camel_test_end();
+
+	return 0;
+}
diff --git a/camel/tests/lib/camel-test.c b/camel/tests/lib/camel-test.c
index e6a7c29340..a61f949f0a 100644
--- a/camel/tests/lib/camel-test.c
+++ b/camel/tests/lib/camel-test.c
@@ -7,6 +7,7 @@
 
 #ifdef ENABLE_THREADS
 #include <pthread.h>
+#include <unistd.h>
 #endif
 
 #ifdef ENABLE_THREADS
@@ -74,6 +75,13 @@ static void die(int sig)
 		indie = 1;
 		printf("\n\nReceived fatal signal %d\n", sig);
 		g_hash_table_foreach(info_table, (GHFunc)dump_action, 0);
+
+#ifdef ENABLE_THREADS
+		if (camel_test_verbose > 2) {
+			printf("Attach debugger to pid %d to debug\n", getpid());
+			sleep(1000);
+		}
+#endif
 	}
 
 	_exit(1);
diff --git a/camel/tests/message/test3.c b/camel/tests/message/test3.c
index 78ac7b3042..0a9683d146 100644
--- a/camel/tests/message/test3.c
+++ b/camel/tests/message/test3.c
@@ -30,8 +30,8 @@ int main(int argc, char **argv)
 	mp = camel_multipart_new();
 
 	/* Hrm, this should be able to set its own boundary, no? */
-	camel_multipart_set_boundary(mp, "_=,.XYZ Kangaroo Meat is ! ABADF00D");
-	check(strcmp(camel_multipart_get_boundary(mp), "_=,.XYZ Kangaroo Meat is ! ABADF00D") == 0);
+	camel_multipart_set_boundary(mp, "_=,.XYZ_Kangaroo_Meat_is_!_ABADF00D");
+	check(strcmp(camel_multipart_get_boundary(mp), "_=,.XYZ_Kangaroo_Meat_is_!_ABADF00D") == 0);
 
 	camel_medium_set_content_object((CamelMedium *)msg, (CamelDataWrapper *)mp);
 	check(camel_multipart_get_number(mp) == 0);
@@ -110,7 +110,7 @@ int main(int argc, char **argv)
 	check(CAMEL_IS_MULTIPART(mp2));
 	check(camel_multipart_get_number(mp2) == 3);
 
-	check(strcmp(camel_multipart_get_boundary(mp2), "_=,.XYZ Kangaroo Meat is ! ABADF00D") == 0);
+	check(strcmp(camel_multipart_get_boundary(mp2), "_=,.XYZ_Kangaroo_Meat_is_!_ABADF00D") == 0);
 	check(mp2->preface == NULL || strlen(mp2->preface) == 0);
 
 	/* FIXME */
@@ -137,7 +137,7 @@ int main(int argc, char **argv)
 	check(CAMEL_IS_MULTIPART(mp2));
 	check(camel_multipart_get_number(mp2) == 3);
 
-	check(strcmp(camel_multipart_get_boundary(mp2), "_=,.XYZ Kangaroo Meat is ! ABADF00D") == 0);
+	check(strcmp(camel_multipart_get_boundary(mp2), "_=,.XYZ_Kangaroo_Meat_is_!_ABADF00D") == 0);
 	check(mp2->preface == NULL || strlen(mp2->preface) == 0);
 
 	/* FIXME */
@@ -174,7 +174,7 @@ int main(int argc, char **argv)
 	check(CAMEL_IS_MULTIPART(mp2));
 	check(camel_multipart_get_number(mp2) == 3);
 
-	check(strcmp(camel_multipart_get_boundary(mp2), "_=,.XYZ Kangaroo Meat is ! ABADF00D") == 0);
+	check(strcmp(camel_multipart_get_boundary(mp2), "_=,.XYZ_Kangaroo_Meat_is_!_ABADF00D") == 0);
 	check(strcmp(mp2->preface, "pre-text\nLines.") == 0);
 	check(strcmp(mp2->postface, "post-text, no lines.\nOne line.\n") == 0);
 	test_message_compare_content(camel_medium_get_content_object(CAMEL_MEDIUM(camel_multipart_get_part(mp2, 0))),