Added flags argument. (message_changed): Add support for the auto-update

2001-02-19  Not Zed  <NotZed@Ximian.com>

        * providers/vee/camel-vee-folder.c (camel_vee_folder_new): Added
        flags argument.
        (message_changed): Add support for the auto-update flag, in which
        case we auto-match the content as it changes.
        (camel_vee_folder_init): Remoive "DYNAMIC" optionality
        (camel_vee_folder_finalise): "
        (vee_expunge): As we expunge folders, re-sync their content by
        rerunning the query.

        * providers/vee/camel-vee-store.c (vee_get_folder): Pass flags to
        folder new.

2001-02-18  Not Zed  <NotZed@Ximian.com>

        * providers/vee/Makefile.am (noinst_HEADERS): Added
        camel-vee-private.h.

        * providers/vee/camel-vee-private.h: New file to add locking stuff
        for vee folders.

        * providers/vee/camel-vee-folder.c (camel_vee_folder_new): Add
        locking around unmatched setup.
        (camel_vee_folder_init): Setup lock.
        (camel_vee_folder_finalise): Free locks.
        (folder_changed): Add locking.
        (unmatched_finalise): "
        (message_changed): "
        (vee_folder_build): "

2001-02-17  Not Zed  <NotZed@Ximian.com>

        * providers/vee/camel-vee-folder.c (vee_folder_add): Check uid
        doesn't exist before adding it.

2001-02-16  Not Zed  <NotZed@Ximian.com>

        * providers/vee/camel-vee-folder.c (camel_vee_folder_new): Create
        a new virtual virtual folder UNMATCHED, to store all messages that
        dont match other vfolders.
        (unmatched_folder_changed): When a vfolder issues a changed event,
        use its info to update the unmatched folder info.
        (unmatched_finalise): When a vfolder is finalised, remove any uids
        it has as if we had a removed event for it.

svn path=/trunk/; revision=8276

6 files changed, 482 insertions, 120 deletions

diff --git a/camel/providers/vee/Makefile.am b/camel/providers/vee/Makefile.am
index 6c0693b02d..b5026c823e 100644
--- a/camel/providers/vee/Makefile.am
+++ b/camel/providers/vee/Makefile.am
@@ -28,6 +28,9 @@ libcamelveeinclude_HEADERS =			\
 	camel-vee-folder.h			\
 	camel-vee-store.h
 
+noinst_HEADERS =				\
+	camel-vee-private.h
+
 libcamelvee_la_LDFLAGS = -version-info 0:0:0
 
 libcamelvee_la_LIBADD = $(top_builddir)/e-util/libeutil.la $(top_builddir)/libibex/libibex.la $(UNICODE_LIBS)
diff --git a/camel/providers/vee/camel-vee-folder.c b/camel/providers/vee/camel-vee-folder.c
index 057d227a61..617b67b162 100644
--- a/camel/providers/vee/camel-vee-folder.c
+++ b/camel/providers/vee/camel-vee-folder.c
@@ -25,9 +25,10 @@
 #include "camel-store.h"
 #include "camel-folder-summary.h"
 #include "camel-mime-message.h"
-#ifdef DYNAMIC
 #include "camel-folder-search.h"
-#endif
+
+#include "camel-vee-store.h"	/* for open flags */
+#include "camel-vee-private.h"
 
 #ifdef DOESTRV
 #include "e-util/e-memory.h"
@@ -43,10 +44,6 @@ typedef struct _CamelVeeMessageInfo {
 	CamelFolder *folder;
 } CamelVeeMessageInfo;
 
-struct _CamelVeeFolderPrivate {
-	GList *folders;
-};
-
 #define _PRIVATE(o) (((CamelVeeFolder *)(o))->priv)
 
 static void vee_sync (CamelFolder *folder, gboolean expunge, CamelException *ex);
@@ -64,14 +61,29 @@ static void camel_vee_folder_class_init (CamelVeeFolderClass *klass);
 static void camel_vee_folder_init       (CamelVeeFolder *obj);
 static void camel_vee_folder_finalise   (CamelObject *obj);
 
+static void unmatched_finalise(CamelFolder *sub, gpointer type, CamelVeeFolder *vf);
+
 static void folder_changed(CamelFolder *sub, gpointer type, CamelVeeFolder *vf);
-static void message_changed(CamelFolder *f, const char *uid, CamelVeeFolder *mf);
+static void message_changed(CamelFolder *f, const char *uid, CamelVeeFolder *vf);
 
 static void vee_folder_build(CamelVeeFolder *vf, CamelException *ex);
 static void vee_folder_build_folder(CamelVeeFolder *vf, CamelFolder *source, CamelException *ex);
 
 static CamelFolderClass *camel_vee_folder_parent;
 
+/* a vfolder for unmatched messages */
+static CamelVeeFolder *folder_unmatched;
+static GHashTable *unmatched_uids;
+#ifdef ENABLE_THREADS
+#include <pthread.h>
+static pthread_mutex_t unmatched_lock = PTHREAD_MUTEX_INITIALIZER;
+#define UNMATCHED_LOCK() pthread_mutex_lock(&unmatched_lock)
+#define UNMATCHED_UNLOCK() pthread_mutex_unlock(&unmatched_lock)
+#else
+#define UNMATCHED_LOCK()
+#define UNMATCHED_UNLOCK()
+#endif
+
 CamelType
 camel_vee_folder_get_type (void)
 {
@@ -128,9 +140,13 @@ camel_vee_folder_init (CamelVeeFolder *obj)
 		CAMEL_MESSAGE_SEEN;
 
 	obj->changes = camel_folder_change_info_new();
-#ifdef DYNAMIC
 	obj->search = camel_folder_search_new();
+
+#ifdef ENABLE_THREADS
+	p->summary_lock = g_mutex_new();
+	p->subfolder_lock = g_mutex_new();
 #endif
+
 }
 
 static void
@@ -155,9 +171,13 @@ camel_vee_folder_finalise (CamelObject *obj)
 	g_free(vf->vname);
 
 	camel_folder_change_info_free(vf->changes);
-#ifdef DYNAMIC
 	camel_object_unref((CamelObject *)vf->search);
+
+#ifdef ENABLE_THREADS
+	g_mutex_free(p->summary_lock);
+	g_mutex_free(p->subfolder_lock);
 #endif
+	g_free(p);
 }
 
 /**
@@ -171,16 +191,13 @@ camel_vee_folder_finalise (CamelObject *obj)
  * Return value: A new CamelVeeFolder widget.
  **/
 CamelFolder *
-camel_vee_folder_new (CamelStore *parent_store, const char *name, CamelException *ex)
+camel_vee_folder_new(CamelStore *parent_store, const char *name, guint32 flags, CamelException *ex)
 {
 	CamelFolderInfo *fi;
 	CamelFolder *folder;
 	CamelVeeFolder *vf;
 	char *namepart, *searchpart;
 
-	folder =  CAMEL_FOLDER (camel_object_new (camel_vee_folder_get_type()));
-	vf = (CamelVeeFolder *)folder;
-
 	namepart = g_strdup(name);
 	searchpart = strchr(namepart, '?');
 	if (searchpart == NULL) {
@@ -190,6 +207,40 @@ camel_vee_folder_new (CamelStore *parent_store, const char *name, CamelException
 		*searchpart++ = 0;
 	}
 
+	UNMATCHED_LOCK();
+
+	if (folder_unmatched == NULL) {
+		printf("setting up unmatched folder\n");
+		unmatched_uids = g_hash_table_new(g_str_hash, g_str_equal);
+
+		folder = (CamelFolder *)camel_object_new(camel_vee_folder_get_type());
+		folder_unmatched = vf = (CamelVeeFolder *)folder;
+		camel_folder_construct(folder, parent_store, "UNMATCHED", "UNMATCHED");
+		folder->summary = camel_folder_summary_new();
+		folder->summary->message_info_size = sizeof(CamelVeeMessageInfo);
+
+		vf->expression = g_strdup("(header-contains \"subject\" \"--= in =-=-=+ valid , ., .l\")");
+		vf->vname = g_strdup("UNMATCHED");
+	}
+
+	UNMATCHED_UNLOCK();
+
+	printf("opening vee folder %s\n", name);
+	if (strcmp(namepart, "UNMATCHED") == 0) {
+		camel_object_ref((CamelObject *)folder_unmatched);
+		g_free(namepart);
+		printf("opened UNMATCHED folder %p %s with %d messages\n", folder_unmatched, name, camel_folder_get_message_count((CamelFolder *)folder_unmatched));
+		return (CamelFolder *)folder_unmatched;
+	}
+
+
+	folder =  CAMEL_FOLDER (camel_object_new (camel_vee_folder_get_type()));
+	vf = (CamelVeeFolder *)folder;
+	vf->flags = flags;
+
+	/* remove folders as they vanish */
+	camel_object_hook_event((CamelObject *)vf, "finalize", (CamelObjectEventHookFunc)unmatched_finalise, folder_unmatched);
+
 	camel_folder_construct (folder, parent_store, namepart, namepart);
 
 	folder->summary = camel_folder_summary_new();
@@ -199,35 +250,139 @@ camel_vee_folder_new (CamelStore *parent_store, const char *name, CamelException
 	vf->vname = namepart;
 
 	vee_folder_build(vf, ex);
-	if (camel_exception_is_set (ex)) {
-		camel_object_unref (CAMEL_OBJECT (folder));
+	if (camel_exception_is_set(ex)) {
+		printf("opening folder failed\n");
+		camel_object_unref((CamelObject *)folder);
 		return NULL;
 	}
 	
-	fi = g_new0 (CamelFolderInfo, 1);
-	fi->full_name = g_strdup (name);
-	fi->name = g_strdup (name);
-	fi->url = g_strdup_printf ("vfolder:%s?%s", vf->vname, vf->expression);
+	printf("opened normal folder folder %p %s with %d messages\n", folder, name, camel_folder_get_message_count(folder));
+
+	/* FIXME: should be moved to store */
+	fi = g_new0(CamelFolderInfo, 1);
+	fi->full_name = g_strdup(name);
+	fi->name = g_strdup(name);
+	fi->url = g_strdup_printf("vfolder:%s?%s", vf->vname, vf->expression);
 	fi->unread_message_count = -1;
 	
-	camel_object_trigger_event (CAMEL_OBJECT (parent_store),
-				    "folder_created", fi);
-	
+	camel_object_trigger_event(CAMEL_OBJECT(parent_store), "folder_created", fi);
 	camel_folder_info_free (fi);
-	
+
 	return folder;
 }
 
+static CamelVeeMessageInfo * vee_folder_add_uid(CamelVeeFolder *vf, CamelFolder *f, const char *inuid);
+
+/* must be called with summary_lock held */
+static void
+unmatched_uid_remove(const char *uidin, CamelFolder *source)
+{
+	char *oldkey, *uid;
+	int n;
+
+	uid = g_strdup_printf("%p:%s", source, uidin);
+	
+	/*printf("checking unmatched uid (remove from source) %s\n", uid);*/
+
+	UNMATCHED_LOCK();
+
+	if (g_hash_table_lookup_extended(unmatched_uids, uid, (void **)&oldkey, (void **)&n)) {
+		if (n == 1) {
+			/*printf("lost all matches, adding uid to unmatched\n");*/
+			if (vee_folder_add_uid(folder_unmatched, source, oldkey))
+				camel_folder_change_info_add_uid(folder_unmatched->changes, uid);
+			g_hash_table_remove(unmatched_uids, oldkey);
+			g_free(oldkey);
+		} else
+			g_hash_table_insert(unmatched_uids, oldkey, (void *)n-1);
+	} else {
+		/*printf("unknown uid, adding to unmatched\n");*/
+		/* FIXME: lookup to see if we already have it first, to save doing it later */
+		if (vee_folder_add_uid(folder_unmatched, source, uidin))
+			camel_folder_change_info_add_uid(folder_unmatched->changes, uid);
+	}
+
+	UNMATCHED_UNLOCK();
+
+	g_free(uid);
+}
+
+/* add a uid to the unmatched folder if it is unmatched everywhere else */
+static void
+unmatched_uid_check(const char *uidin, CamelFolder *source)
+{
+	char *oldkey, *uid;
+	int n;
+
+	uid = g_strdup_printf("%p:%s", source, uidin);
+	
+	/*printf("checking unmatched uid (remove from source) %s\n", uid);*/
+
+	UNMATCHED_LOCK();
+
+	if (!g_hash_table_lookup_extended(unmatched_uids, uid, (void **)&oldkey, (void **)&n)) {
+		/*printf("unknown uid, adding to unmatched\n");*/
+		/* FIXME: lookup to see if we already have it first, to save doing it later */
+		if (vee_folder_add_uid(folder_unmatched, source, uidin))
+			camel_folder_change_info_add_uid(folder_unmatched->changes, uid);
+	}
+
+	UNMATCHED_UNLOCK();
+
+	g_free(uid);
+}
+
+/* must be called with summary_lock held */
+static void
+unmatched_uid_add(const char *uidin, CamelFolder *source)
+{
+	char *oldkey, *uid;
+	int n;
+	CamelMessageInfo *info;
+
+	uid = g_strdup_printf("%p:%s", source, uidin);
+
+	/*printf("checking unmatched uid (added to source) %s\n", uid);*/
+
+	UNMATCHED_LOCK();
+
+	info = camel_folder_summary_uid(((CamelFolder *)folder_unmatched)->summary, uid);
+	if (info) {
+		/*printf("we have it, lets remove it\n");*/
+		camel_folder_summary_remove_uid(((CamelFolder *)folder_unmatched)->summary, uid);
+		camel_folder_change_info_remove_uid(folder_unmatched->changes, uid);
+		camel_folder_summary_info_free(((CamelFolder *)folder_unmatched)->summary, info);
+	}
+
+	if (g_hash_table_lookup_extended(unmatched_uids, uid, (void **)&oldkey, (void **)&n)) {
+		g_hash_table_insert(unmatched_uids, oldkey, (void **)n+1);
+		g_free(uid);
+	} else
+		g_hash_table_insert(unmatched_uids, uid, (void **)1);
+
+	UNMATCHED_UNLOCK();
+}
+
+/* must be called with summary_lock held */
 static CamelVeeMessageInfo *
 vee_folder_add(CamelVeeFolder *vf, CamelFolder *f, CamelMessageInfo *info)
 {
 	CamelVeeMessageInfo *mi;
 	char *uid;
 	CamelFolder *folder = (CamelFolder *)vf;
+	CamelMessageInfo *dinfo;
+
+	uid = g_strdup_printf("%p:%s", f, camel_message_info_uid(info));
+	/* FIXME: Has races */
+	dinfo = camel_folder_summary_uid(folder->summary, uid);
+	if (dinfo) {
+		g_free(uid);
+		camel_folder_summary_info_free(folder->summary, dinfo);
+		return NULL;
+	}
 
 	mi = (CamelVeeMessageInfo *)camel_folder_summary_info_new(folder->summary);
 	camel_message_info_dup_to(info, (CamelMessageInfo *)mi);
-	uid = g_strdup_printf("%p:%s", f, camel_message_info_uid(info));
 #ifdef DOESTRV
 	mi->info.strings = e_strv_set_ref_free(mi->info.strings, CAMEL_MESSAGE_INFO_UID, uid);
 	mi->info.strings = e_strv_pack(mi->info.strings);
@@ -241,6 +396,7 @@ vee_folder_add(CamelVeeFolder *vf, CamelFolder *f, CamelMessageInfo *info)
 	return mi;
 }
 
+/* must be called with summary_lock held */
 static CamelVeeMessageInfo *
 vee_folder_add_uid(CamelVeeFolder *vf, CamelFolder *f, const char *inuid)
 {
@@ -249,13 +405,16 @@ vee_folder_add_uid(CamelVeeFolder *vf, CamelFolder *f, const char *inuid)
 
 	info = camel_folder_get_message_info(f, inuid);
 	if (info) {
-		mi = vee_folder_add(vf, f, info);
+		if ((mi = vee_folder_add(vf, f, info)))
+			if (vf != folder_unmatched)
+				unmatched_uid_add(inuid, f);
+
 		camel_folder_free_message_info(f, info);
 	}
 	return mi;
 }
 
-#ifdef DYNAMIC
+/* must be called with summary_lock held */
 static void
 vfolder_remove_match(CamelVeeFolder *vf, CamelVeeMessageInfo *vinfo)
 {
@@ -263,23 +422,29 @@ vfolder_remove_match(CamelVeeFolder *vf, CamelVeeMessageInfo *vinfo)
 
 	printf("removing match %s\n", uid);
 
-	camel_folder_summary_remove(((CamelFolder *)vf)->summary, (CamelMessageInfo *)vinfo);
+	unmatched_uid_remove(strchr(uid, ':'), vinfo->folder);
+
 	camel_folder_change_info_remove_uid(vf->changes, uid);
+	camel_folder_summary_remove(((CamelFolder *)vf)->summary, (CamelMessageInfo *)vinfo);
 }
 
+/* must be called with summary_lock held */
 static CamelVeeMessageInfo *
 vee_folder_add_change(CamelVeeFolder *vf, CamelFolder *f, CamelMessageInfo *info)
 {
 	CamelVeeMessageInfo *mi = NULL;
 
 	mi = vee_folder_add(vf, f, info);
-	camel_folder_change_info_add_uid(vf->changes, camel_message_info_uid(mi));
+	if (mi) {
+		unmatched_uid_add(camel_message_info_uid(info), f);
+
+		camel_folder_change_info_add_uid(vf->changes, camel_message_info_uid(mi));
+	}
 	
 	return mi;
 }
 
-#endif
-
+/* must be called with summary_lock held */
 static void
 vfolder_change_match(CamelVeeFolder *vf, CamelVeeMessageInfo *vinfo, const CamelMessageInfo *info)
 {
@@ -304,26 +469,95 @@ vfolder_change_match(CamelVeeFolder *vf, CamelVeeMessageInfo *vinfo, const Camel
 	camel_folder_change_info_change_uid(vf->changes, camel_message_info_uid(vinfo));
 }
 
+/* track changes to the unmatched folders */
+static void
+unmatched_finalise(CamelFolder *sub, gpointer type, CamelVeeFolder *vf)
+{
+	int count, i;
+
+	CAMEL_VEE_FOLDER_LOCK(vf, summary_lock);
+	UNMATCHED_LOCK();
+
+	count = camel_folder_summary_count(((CamelFolder *)folder_unmatched)->summary);
+	for (i=0;i<count;i++) {
+		CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index(((CamelFolder *)folder_unmatched)->summary, i);
+		const char *uid;
+		char *oldkey;
+		int n;
+
+		if (mi) {
+			uid = camel_message_info_uid(mi);
+			if (mi->folder == sub) {
+				if (g_hash_table_lookup_extended(unmatched_uids, uid, (void **)&oldkey, (void **)&n)) {
+					if (n == 1)
+						g_hash_table_remove(unmatched_uids, oldkey);
+					else
+						camel_folder_change_info_remove_uid(folder_unmatched->changes, uid);
+				}
+				camel_folder_summary_remove(((CamelFolder *)folder_unmatched)->summary, (CamelMessageInfo *)mi);
+				i--;
+			}
+			camel_folder_summary_info_free(((CamelFolder *)folder_unmatched)->summary, (CamelMessageInfo *)mi);
+		}
+	}
+	
+	if (camel_folder_change_info_changed(folder_unmatched->changes)) {
+		camel_object_trigger_event((CamelObject *)folder_unmatched, "folder_changed", folder_unmatched->changes);
+		camel_folder_change_info_clear(folder_unmatched->changes);
+	}
+
+	UNMATCHED_UNLOCK();
+	CAMEL_VEE_FOLDER_UNLOCK(vf, summary_lock);
+}
+
 /* FIXME: This code is a big race, as it is never called locked ... */
 
 static void
 folder_changed(CamelFolder *sub, gpointer type, CamelVeeFolder *vf)
 {
-	CamelException *ex;
-
-#ifdef DYNAMIC
 	CamelFolderChangeInfo *changes = type;
 	CamelFolder *folder = (CamelFolder *)vf;
+	char *vuid;
+	CamelVeeMessageInfo *vinfo;
+	int i;
+	CamelMessageInfo *info;
+
+	printf("folder_changed(%p, %p) (for %s)\n", sub, vf, vf->expression);
+
+	/* if not auto-updating, only propagate changed events, not added/removed items */
+	if ((vf->flags & CAMEL_STORE_VEE_FOLDER_AUTO) == 0) {
+		CAMEL_VEE_FOLDER_LOCK(vf, summary_lock);
+
+		for (i=0;i<changes->uid_changed->len;i++) {
+			info = camel_folder_get_message_info(sub, changes->uid_changed->pdata[i]);
+			vuid = g_strdup_printf("%p:%s", sub, (char *)changes->uid_changed->pdata[i]);
+			vinfo = (CamelVeeMessageInfo *)camel_folder_summary_uid(folder->summary, vuid);
+			if (vinfo && info)
+				vfolder_change_match(vf, vinfo, info);
+			
+			g_free(vuid);
+			
+			if (info)
+				camel_folder_free_message_info(sub, info);
+			if (vinfo)
+				camel_folder_summary_info_free(folder->summary, (CamelMessageInfo *)vinfo);
+		}
+
+		if (camel_folder_change_info_changed(vf->changes)) {
+			camel_object_trigger_event((CamelObject *)vf, "folder_changed", vf->changes);
+			camel_folder_change_info_clear(vf->changes);
+		}
+
+		CAMEL_VEE_FOLDER_UNLOCK(vf, summary_lock);
+
+		return;
+	}
+
+	/* if we are autoupdating, then do the magic */
 
 	/* assume its faster to search a long list in whole, than by part */
 	if (changes && (changes->uid_added->len + changes->uid_changed->len) < 500) {
-		int i;
-		char *vuid;
-		CamelVeeMessageInfo *vinfo;
 		gboolean match;
-		CamelMessageInfo *info;
-
-		ex = camel_exception_new();
 
 		/* FIXME: We dont search body contents with this search, so, it isn't as
 		   useful as it might be.
@@ -335,10 +569,12 @@ folder_changed(CamelFolder *sub, gpointer type, CamelVeeFolder *vf)
 
 		/* see if added ones now match us */
 		for (i=0;i<changes->uid_added->len;i++) {
+			printf("checking new uid: %s\n", (char *)changes->uid_added->pdata[i]);
 			info = camel_folder_get_message_info(sub, changes->uid_added->pdata[i]);
 			if (info) {
+				printf("uid ok, subject: %s\n", camel_message_info_subject(info));
 				camel_folder_search_set_folder(vf->search, sub);
-				match = camel_folder_search_match_expression(vf->search, vf->expression, info, ex);
+				match = camel_folder_search_match_expression(vf->search, vf->expression, info, NULL);
 				if (match)
 					vinfo = vee_folder_add_change(vf, sub, info);
 				camel_folder_free_message_info(sub, info);
@@ -352,21 +588,14 @@ folder_changed(CamelFolder *sub, gpointer type, CamelVeeFolder *vf)
 			vinfo = (CamelVeeMessageInfo *)camel_folder_summary_uid(folder->summary, vuid);
 			if (info) {
 				camel_folder_search_set_folder(vf->search, sub);
-#if 0
-				match = camel_folder_search_match_expression(vf->search, vf->expression, info, ex);
-#endif
+				match = camel_folder_search_match_expression(vf->search, vf->expression, info, NULL);
 				if (vinfo) {
-#if 0
 					if (!match)
 						vfolder_remove_match(vf, vinfo);
 					else
-#endif
 						vfolder_change_match(vf, vinfo, info);
-				}
-#if 0
-				else if (match)
+				} else if (match)
 					vee_folder_add_change(vf, sub, info);
-#endif
 				camel_folder_free_message_info(sub, info);
 			} else if (vinfo)
 				vfolder_remove_match(vf, vinfo);
@@ -377,8 +606,6 @@ folder_changed(CamelFolder *sub, gpointer type, CamelVeeFolder *vf)
 			g_free(vuid);
 		}
 
-		camel_exception_free(ex);
-
 		/* mirror removes directly, if they used to match */
 		for (i=0;i<changes->uid_removed->len;i++) {
 			vuid = g_strdup_printf("%p:%s", sub, (char *)changes->uid_removed->pdata[i]);
@@ -390,35 +617,42 @@ folder_changed(CamelFolder *sub, gpointer type, CamelVeeFolder *vf)
 			g_free(vuid);
 		}
 	} else {
-#endif
-		ex = camel_exception_new();
-		vee_folder_build_folder(vf, sub, ex);
-		camel_exception_free(ex);
-#ifdef DYNAMIC
+		vee_folder_build_folder(vf, sub, NULL);
 	}
-#endif
 
 	/* cascade up, if we need to */
 	if (camel_folder_change_info_changed(vf->changes)) {
+		printf("got folder changes\n");
 		camel_object_trigger_event( CAMEL_OBJECT(vf), "folder_changed", vf->changes);
 		camel_folder_change_info_clear(vf->changes);
+	} else
+		printf("no, we didn't really get any changes\n");
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, summary_lock);
+
+	UNMATCHED_LOCK();
+
+	if (camel_folder_change_info_changed(folder_unmatched->changes)) {
+		camel_object_trigger_event((CamelObject *)folder_unmatched, "folder_changed", folder_unmatched->changes);
+		camel_folder_change_info_clear(folder_unmatched->changes);
 	}
+
+	UNMATCHED_UNLOCK();
 }
 
 /* FIXME: This code is a race, as it is never called locked */
 
 /* track flag changes in the summary */
 static void
-message_changed(CamelFolder *f, const char *uid, CamelVeeFolder *mf)
+message_changed(CamelFolder *f, const char *uid, CamelVeeFolder *vf)
 {
 	CamelMessageInfo *info;
 	CamelVeeMessageInfo *vinfo;
 	char *vuid;
-	CamelFolder *folder = (CamelFolder *)mf;
-#ifdef DYNAMIC
-	/*gboolean match;*/
-	CamelException *ex;
-#endif
+	CamelFolder *folder = (CamelFolder *)vf;
+	gboolean match;
+
+	CAMEL_VEE_FOLDER_LOCK(vf, summary_lock);
 
 	info = camel_folder_get_message_info(f, uid);
 	vuid = g_strdup_printf("%p:%s", f, uid);
@@ -429,32 +663,26 @@ message_changed(CamelFolder *f, const char *uid, CamelVeeFolder *mf)
 	/* Hmm, this might not work if the folder uses some weird search thing,
 	   and/or can be slow since it wont use any index index, hmmm. */
 
-#ifdef DYNAMIC
-	camel_folder_search_set_folder(mf->search, f);
-	ex = camel_exception_new();
-#if 0
-	match = camel_folder_search_match_expression(mf->search, mf->expression, info, ex);
-#endif
-	camel_exception_free(ex);
-	if (info) {
-		if (vinfo) {
-#if 0
-			if (!match)
-				vfolder_remove_match(mf, vinfo);
-			else
-#endif
-				vfolder_change_match(mf, vinfo, info);
-		}
-#if 0
-		else if (match)
-			vee_folder_add_change(mf, f, info);
-#endif
-	} else if (vinfo)
-		vfolder_remove_match(mf, vinfo);
-#else
-	if (info && vinfo)
-		vfolder_change_match(mf, vinfo, info);
-#endif
+	if (vf->flags & CAMEL_STORE_VEE_FOLDER_AUTO) {
+		camel_folder_search_set_folder(vf->search, f);
+		
+		match = camel_folder_search_match_expression(vf->search, vf->expression, info, NULL);
+
+		if (info) {
+			if (vinfo) {
+				if (!match)
+					vfolder_remove_match(vf, vinfo);
+				else
+					vfolder_change_match(vf, vinfo, info);
+			}
+			else if (match)
+				vee_folder_add_change(vf, f, info);
+		} else if (vinfo)
+			vfolder_remove_match(vf, vinfo);
+	} else {
+		if (info && vinfo)
+			vfolder_change_match(vf, vinfo, info);
+	}
 
 	if (info)
 		camel_folder_free_message_info(f, info);
@@ -463,41 +691,54 @@ message_changed(CamelFolder *f, const char *uid, CamelVeeFolder *mf)
 
 	/* cascade up, if required.  This could probably be delayed,
 	   but doesn't matter really, that is what freeze is for. */
-	if (camel_folder_change_info_changed(mf->changes)) {
-		camel_object_trigger_event( CAMEL_OBJECT(mf), "folder_changed", mf->changes);
-		camel_folder_change_info_clear(mf->changes);
+	if (camel_folder_change_info_changed(vf->changes)) {
+		camel_object_trigger_event( CAMEL_OBJECT(vf), "folder_changed", vf->changes);
+		camel_folder_change_info_clear(vf->changes);
+	}
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, summary_lock);
+
+	UNMATCHED_LOCK();
+
+	if (camel_folder_change_info_changed(folder_unmatched->changes)) {
+		camel_object_trigger_event((CamelObject *)folder_unmatched, "folder_changed", folder_unmatched->changes);
+		camel_folder_change_info_clear(folder_unmatched->changes);
 	}
 
+	UNMATCHED_UNLOCK();
+
 	g_free(vuid);
 }
 
 void
 camel_vee_folder_add_folder(CamelVeeFolder *vf, CamelFolder *sub)
 {
-	struct _CamelVeeFolderPrivate *p = _PRIVATE(vf);
-	CamelException *ex;
+	struct _CamelVeeFolderPrivate *p = _PRIVATE(vf), *up = _PRIVATE(folder_unmatched);
 
 	camel_object_ref((CamelObject *)sub);
+
+	CAMEL_VEE_FOLDER_LOCK(vf, subfolder_lock);
+
+	/* the reference is shared with both the real vfolder and the unmatched vfolder */
 	p->folders = g_list_append(p->folders, sub);
+	up->folders = g_list_append(up->folders, sub);
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
 
-	camel_object_hook_event ((CamelObject *)sub, "folder_changed", (CamelObjectEventHookFunc) folder_changed, vf);
-	camel_object_hook_event ((CamelObject *)sub, "message_changed", (CamelObjectEventHookFunc) message_changed, vf);
+	printf("camel_vee_folder_add_folde(%p, %p)\n", vf, sub);
 
-	ex = camel_exception_new();
-	vee_folder_build_folder(vf, sub, ex);
-	camel_exception_free(ex);
+	camel_object_hook_event((CamelObject *)sub, "folder_changed", (CamelObjectEventHookFunc)folder_changed, vf);
+	camel_object_hook_event((CamelObject *)sub, "message_changed", (CamelObjectEventHookFunc)message_changed, vf);
+
+	CAMEL_VEE_FOLDER_LOCK(vf, summary_lock);
+
+	vee_folder_build_folder(vf, sub, NULL);
 
 	/* we'll assume the caller is going to update the whole list after they do this
 	   this may or may not be the right thing to do, but it should be close enough */
-#if 0	
-	if (camel_folder_change_info_changed(vf->changes)) {
-		camel_object_trigger_event( CAMEL_OBJECT(vf), "folder_changed", vf->changes);
-		camel_folder_change_info_clear(vf->changes);
-	}
-#else
 	camel_folder_change_info_clear(vf->changes);
-#endif
-	
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, summary_lock);
 }
 
 static void
@@ -507,13 +748,20 @@ vee_sync (CamelFolder *folder, gboolean expunge, CamelException *ex)
 	struct _CamelVeeFolderPrivate *p = _PRIVATE(vf);
 	GList *node;
 
+	CAMEL_VEE_FOLDER_LOCK(vf, subfolder_lock);
+
 	node = p->folders;
 	while (node) {
 		CamelFolder *f = node->data;
 
 		camel_folder_sync(f, expunge, ex);
+		if (camel_exception_is_set(ex))
+			break;
+
 		node = node->next;
 	}
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
 }
 
 static void
@@ -523,13 +771,23 @@ vee_expunge (CamelFolder *folder, CamelException *ex)
 	struct _CamelVeeFolderPrivate *p = _PRIVATE(vf);
 	GList *node;
 
+	CAMEL_VEE_FOLDER_LOCK(vf, subfolder_lock);
+
 	node = p->folders;
 	while (node) {
 		CamelFolder *f = node->data;
 
 		camel_folder_expunge(f, ex);
+		if (camel_exception_is_set(ex))
+			break;
+		vee_folder_build_folder(vf, f, ex);
+		if (camel_exception_is_set(ex))
+			break;
+
 		node = node->next;
 	}
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
 }
 
 static CamelMimeMessage *vee_get_message(CamelFolder *folder, const gchar *uid, CamelException *ex)
@@ -558,6 +816,8 @@ vee_search_by_expression(CamelFolder *folder, const char *expression, CamelExcep
 	CamelVeeFolder *vf = (CamelVeeFolder *)folder;
 	struct _CamelVeeFolderPrivate *p = _PRIVATE(vf);
 
+	CAMEL_VEE_FOLDER_LOCK(vf, subfolder_lock);
+
 	expr = g_strdup_printf("(and %s %s)", vf->expression, expression);
 	node = p->folders;
 	while (node) {
@@ -572,6 +832,9 @@ vee_search_by_expression(CamelFolder *folder, const char *expression, CamelExcep
 		camel_folder_search_free(f, matches);
 		node = g_list_next(node);
 	}
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
+
 	return result;
 }
 
@@ -624,6 +887,7 @@ vee_move_message_to(CamelFolder *folder, const char *uid, CamelFolder *dest, Cam
 */
 
 /* this does most of the vfolder magic */
+/* must have summary_lock held when calling */
 static void
 vee_folder_build(CamelVeeFolder *vf, CamelException *ex)
 {
@@ -633,6 +897,8 @@ vee_folder_build(CamelVeeFolder *vf, CamelException *ex)
 
 	camel_folder_summary_clear(folder->summary);
 
+	CAMEL_VEE_FOLDER_LOCK(vf, subfolder_lock);
+
 	node = p->folders;
 	while (node) {
 		GPtrArray *matches;
@@ -646,28 +912,42 @@ vee_folder_build(CamelVeeFolder *vf, CamelException *ex)
 		camel_folder_search_free(f, matches);
 		node = g_list_next(node);
 	}
+
+	CAMEL_VEE_FOLDER_UNLOCK(vf, subfolder_lock);
 }
 
+static void
+removed_uid(void *key, void *value, void *data)
+{
+	unmatched_uid_check(key, data);
+}
 
 /* build query contents for a single folder */
+/* must have summary_lock held when calling */
 static void
 vee_folder_build_folder(CamelVeeFolder *vf, CamelFolder *source, CamelException *ex)
 {
-	GPtrArray *matches;
+	GPtrArray *matches, *all;
+	GHashTable *left;
 	CamelFolder *f = source;
 	CamelVeeMessageInfo *mi;
 	CamelFolder *folder = (CamelFolder *)vf;
 	int i;
 	int count;
 
+	left = g_hash_table_new(g_str_hash, g_str_equal);
+	all = camel_folder_get_uids(f);
+	for (i=0;i<all->len;i++)
+		g_hash_table_insert(left, all->pdata[i], (void *)1);
+
 	count = camel_folder_summary_count(folder->summary);
 	for (i=0;i<count;i++) {
 		CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index(folder->summary, i);
 		if (mi) {
 			if (mi->folder == source) {
-				const char *uid = camel_message_info_uid(mi);
-				camel_folder_change_info_add_source(vf->changes, uid);
+				camel_folder_change_info_add_source(vf->changes, camel_message_info_uid(mi));
 				camel_folder_summary_remove(folder->summary, (CamelMessageInfo *)mi);
+				unmatched_uid_remove(camel_message_info_uid(mi), source);
 				i--;
 			}
 			camel_folder_summary_info_free(folder->summary, (CamelMessageInfo *)mi);
@@ -676,15 +956,26 @@ vee_folder_build_folder(CamelVeeFolder *vf, CamelFolder *source, CamelException
 
 	matches = camel_folder_search_by_expression(f, vf->expression, ex);
 	for (i = 0; i < matches->len; i++) {
+		g_hash_table_remove(left, matches->pdata[i]);
 		mi = vee_folder_add_uid(vf, f, matches->pdata[i]);
 		if (mi)
 			camel_folder_change_info_add_update(vf->changes, camel_message_info_uid(mi));
 	}
+
+	/* check if we have a match for these in another vfolder, else add them to the UNMATCHED folder */
+	g_hash_table_foreach(left, removed_uid, source);
+	g_hash_table_destroy(left);
 	camel_folder_search_free(f, matches);
+	camel_folder_free_uids(f, all);
 
 	camel_folder_change_info_build_diff(vf->changes);
-}
+	camel_folder_change_info_build_diff(folder_unmatched->changes);
 
+	if (camel_folder_change_info_changed(folder_unmatched->changes)) {
+		camel_object_trigger_event((CamelObject *)folder_unmatched, "folder_changed", folder_unmatched->changes);
+		camel_folder_change_info_clear(folder_unmatched->changes);
+	}
+}
 
 /*
 
diff --git a/camel/providers/vee/camel-vee-folder.h b/camel/providers/vee/camel-vee-folder.h
index 65c718a8d6..6ae685a8fe 100644
--- a/camel/providers/vee/camel-vee-folder.h
+++ b/camel/providers/vee/camel-vee-folder.h
@@ -24,9 +24,6 @@
 
 #include <camel/camel-folder.h>
 
-/* try the dynamic update version */
-#define DYNAMIC
-
 #define CAMEL_VEE_FOLDER(obj)         CAMEL_CHECK_CAST (obj, camel_vee_folder_get_type (), CamelVeeFolder)
 #define CAMEL_VEE_FOLDER_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_vee_folder_get_type (), CamelVeeFolderClass)
 #define CAMEL_IS_VEE_FOLDER(obj)      CAMEL_CHECK_TYPE (obj, camel_vee_folder_get_type ())
@@ -43,10 +40,10 @@ struct _CamelVeeFolder {
 	char *vname;		/* local name */
 	CamelFolder *local;	/* local storage for folder */
 
+	guint32 flags;		/* folder open flags */
+
 	CamelFolderChangeInfo *changes;
-#ifdef DYNAMIC
 	CamelFolderSearch *search;
-#endif
 };
 
 struct _CamelVeeFolderClass {
@@ -54,9 +51,7 @@ struct _CamelVeeFolderClass {
 };
 
 guint		camel_vee_folder_get_type	(void);
-CamelFolder    *camel_vee_folder_new		(CamelStore *parent_store,
-						 const char *name,
-						 CamelException *ex);
+CamelFolder    *camel_vee_folder_new		(CamelStore *parent_store, const char *name, guint32 flags, CamelException *ex);
 
 void camel_vee_folder_add_folder(CamelVeeFolder *vf, CamelFolder *sub);
 
diff --git a/camel/providers/vee/camel-vee-private.h b/camel/providers/vee/camel-vee-private.h
new file mode 100644
index 0000000000..6633647bfc
--- /dev/null
+++ b/camel/providers/vee/camel-vee-private.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *  camel-vee-private.h: Private info for vee.
+ *
+ * Authors: Michael Zucchi <notzed@helixcode.com>
+ *
+ * Copyright 1999, 2000 Helix Code, Inc. (http://www.helixcode.com)
+ *
+ * This program is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU General Public License as 
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+#ifndef CAMEL_VEE_PRIVATE_H
+#define CAMEL_VEE_PRIVATE_H 1
+
+#ifdef __cplusplus
+extern "C" {
+#pragma }
+#endif /* __cplusplus }*/
+
+/* need a way to configure and save this data, if this header is to
+   be installed.  For now, dont install it */
+
+#include "config.h"
+
+struct _CamelVeeStorePrivate {
+};
+
+#ifdef ENABLE_THREADS
+#define CAMEL_VEE_STORE_LOCK(f, l) (e_mutex_lock(((CamelVeeStore *)f)->priv->l))
+#define CAMEL_VEE_STORE_UNLOCK(f, l) (e_mutex_unlock(((CamelVeeStore *)f)->priv->l))
+#else
+#define CAMEL_VEE_STORE_LOCK(f, l)
+#define CAMEL_VEE_STORE_UNLOCK(f, l)
+#endif
+
+struct _CamelVeeFolderPrivate {
+	GList *folders;		/* lock using subfolder_lock before changing/accessing */
+
+#ifdef ENABLE_THREADS
+	GMutex *summary_lock;		/* for locking vfolder summary */
+	GMutex *subfolder_lock;		/* for locking the subfolder list */
+#endif
+};
+
+#ifdef ENABLE_THREADS
+#define CAMEL_VEE_FOLDER_LOCK(f, l) (g_mutex_lock(((CamelVeeFolder *)f)->priv->l))
+#define CAMEL_VEE_FOLDER_UNLOCK(f, l) (g_mutex_unlock(((CamelVeeFolder *)f)->priv->l))
+#else
+#define CAMEL_VEE_FOLDER_LOCK(f, l)
+#define CAMEL_VEE_FOLDER_UNLOCK(f, l)
+#endif
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* CAMEL_VEE_PRIVATE_H */
+
diff --git a/camel/providers/vee/camel-vee-store.c b/camel/providers/vee/camel-vee-store.c
index b39707e1a0..4920e04a83 100644
--- a/camel/providers/vee/camel-vee-store.c
+++ b/camel/providers/vee/camel-vee-store.c
@@ -89,5 +89,5 @@ camel_vee_store_new (void)
 static CamelFolder *
 vee_get_folder (CamelStore *store, const char *folder_name, guint32 flags, CamelException *ex)
 {
-	return camel_vee_folder_new (store, folder_name, ex);
+	return camel_vee_folder_new(store, folder_name, flags, ex);
 }
diff --git a/camel/providers/vee/camel-vee-store.h b/camel/providers/vee/camel-vee-store.h
index 2b7f17c749..ff3d548180 100644
--- a/camel/providers/vee/camel-vee-store.h
+++ b/camel/providers/vee/camel-vee-store.h
@@ -31,6 +31,9 @@
 typedef struct _CamelVeeStore      CamelVeeStore;
 typedef struct _CamelVeeStoreClass CamelVeeStoreClass;
 
+/* open mode for folder, vee folder auto-update */
+#define CAMEL_STORE_VEE_FOLDER_AUTO (1<<16)
+
 struct _CamelVeeStore {
 	CamelStore parent;