** Merged in camel-incremental-branch.

2000-11-02  Not Zed  <NotZed@HelixCode.com>

        ** Merged in camel-incremental-branch.

        * mail-format.c (mail_get_message_body): Jeff!  Sigh.
        We should definetly not be strduping the
        content, it has already been copied and duplicated.  Look at
        get_data_wrapper_text.

svn path=/trunk/; revision=6337

11 files changed, 956 insertions, 218 deletions

diff --git a/mail/ChangeLog b/mail/ChangeLog
index e5c98f2368..677c6ab7db 100644
--- a/mail/ChangeLog
+++ b/mail/ChangeLog
@@ -1,3 +1,12 @@
+2000-11-02  Not Zed  <NotZed@HelixCode.com>
+
+        ** Merged in camel-incremental-branch.
+
+	* mail-format.c (mail_get_message_body): Jeff!  Sigh.
+	We should definetly not be strduping the
+	content, it has already been copied and duplicated.  Look at
+	get_data_wrapper_text.
+
 2000-11-01  Chris Toshok  <toshok@helixcode.com>
 
 	* subscribe-dialog.h: add fields search_entry and search_top.
@@ -276,6 +285,210 @@
 
 	* message-list.c: Fixed some column widths.
 
+2000-11-02  Not Zed  <NotZed@HelixCode.com>
+
+	* message-list.c (get_message_info): Call get_message_uid to get
+	the uid, save some duplicated code.
+	(folder_changed): Handle the case of a NULL changes input.
+
+	* message-thread.c (thread_messages): Removed pointless
+	variable/assignment 'container'.
+	(thread_messages): Try and cope with duplicate message id's.
+
+2000-11-01  Not Zed  <NotZed@HelixCode.com>
+
+	* mail-callbacks.c (main_select_first_unread): Changed to use 0 as
+	the first row to select a message.
+
+	* mail-ops.h (mail_do_regenerate_messagelist): Removed from
+	header.  This function is no longer public since it is really an
+	internal message-list function.
+
+	* folder-browser.c (search_full_clicked): Call the set_search()
+	function, rather than messagelist_rebuild, which is going private.
+	(search_set): Same here.
+	(folder_browser_clear_search): And here.
+	(etable_key): Call message_list_select() instead of
+	message_list_home and message_list_end.  Removing some odd code
+	duplication.
+
+	* message-thread.c (do_thread_messages): Moved the mail lock to
+	here, rather than locking for each message lookup (which is
+	useless anyway).  This is still not correct either, as the tree
+	references folder data ... but a bit better than it was.
+	(thread_messages): Removed the mail tool lock stuff, lock in
+	higher functions.
+
+	* message-list.h: Added a threaded indicator to the message list
+	itself.
+	(threaded_view): removed a mystery variable.
+
+	* message-list.c (do_regenerate_messagelist): Made the code a
+	little more readable.
+	(build_tree): Fixed argument to be a thread_messages struct, not a
+	container.
+	(cleanup_regenerate_messagelist): Free changeinfo.
+	(mail_do_regenerate_messagelist): If we are adding changes to a
+	flat view, we dont need to goto the other thread at all, so
+	process immediately.
+	(message_list_toggle_threads): Clear the tree if we're changing
+	the view mode.
+	(message_list_toggle_threads): And reset the rowmap, since it is no
+	longer valid.
+	(build_tree): If we are building into an already empty tree, just
+	build into that (probably irrelevant optimisation).
+	(build_subtree): Build hte subtree in the same order as we got it,
+	not inverted order.
+	(message_list_set_threaded): New function to select the threaded
+	view/flat view.
+	(mail_do_regenerate_messagelist): Removed references to
+	mail_config, get it from the ml->threaded var instead.
+	(message_list_destroy): No longer free the key data for the
+	uid_rowmap.
+	(new_id_from_uid): Convert a uid string into an id string.
+	(new_id_from_subject): Likewise for subject strings.
+	 'id' strings replace the 'uid:' and 'subject:' stuff with
+	 accessors and macros and use less memory and is more readable.
+	(id_is_uid): macro to check if an id string is a uid.
+	(id_uid): Returns the uid part of a uid id string.
+	(id_subject): Returns the uid part of a subject id string.
+	(build_subtree): Use the new id functions, and dont duplicate the
+	uid in the uid rowmap, but just reference it from the tree node.
+	(node_equal): Use new id functions.
+	(add_node_diff): And here too.
+	(remove_node_diff): And here.  Also remove the uid from the
+	rowmap, and dont free it anymore.
+	(get_message_info): And here.
+	(get_message_uid): And here.
+	(subtree_unread): And here.
+	(ml_tree_value_at):  "
+	(ml_tree_set_value_at): Noted a memory leak.  do_flag_messages()
+	doesn't free the contents of the uid array, just the uid array
+	(well that i can tell, teh code has more problems anyway).
+	(ml_tree_set_value_at): And fix the id accessors.
+	(save_node_state):  " 
+	(build_flat): Use id macros/functions.  Dont alloc memory for hash
+	key.
+	(build_flat_diff): Use id macros.
+	(build_flat_diff): Remove the hash table entry before freeing its
+	key data (in the node).
+	(free_key): Removed.  Keys are no longer alloc'd.
+	(clear_tree): When we clear the tree, also clear the uid_rowmap,
+	as it is no longer valid (or contains allocated keys!).
+	(free_tree_ids): Renamed from nuke_uids.
+	(free_ids_cb): Renamed from nuke_uids_cb.
+	(free_tree_ids): Changed arg to be a ETreeModel directly.
+	(ml_tree_value_at): Map id to subject using the right macro.
+	(free_tree_ids): Check we have any nodes to traverse first.
+	(build_flat): Insert to row -1 to append the nodes (faster).
+	(remove_node_diff): Only remove the uid rowmap entry if it is
+	referencing this node (i.e. the key string is the same key string,
+	not just a matching key string).
+	(add_node_diff): Remove the uid rowmap entry before inserting a
+	new one to force the key to be replaced.  This is required as the
+	tree may temporarily contain duplicate messages during the
+	rebuilding phase.
+	(message_list_set_search): New function, set the search string.
+	Only redo the search if it has changed, etc.
+	(mail_do_regenerate_messagelist): Made static.  There is no need
+	for external code to call this.
+	(message_list_set_folder): NOP if the new folder is the same.
+	(message_list_set_folder): Clear the tree before rebuilding it.
+	(message_list_select): Ok, this wins the award for 'most bizarre
+	interface'.  Changed the start row to mean the end of the list if
+	we supply -1, rather than the start of the list.  Also fixed the
+	endpoints (it would never select message 0 if searching
+	backwards).
+	(idle_select_row): Changed start row to 0 from -1.
+	(message_list_end): Removed.
+	(message_list_home): Removed.
+	(go_to_message): Removed.  message_list_select can do this.
+	(message_list_select): Check that direction is one of the valid
+	ones, otherwise we could be thrown for loops.
+
+2000-10-31  Not Zed  <NotZed@HelixCode.com>
+
+	* message-list.c (node_equal): Compares an etree node with a
+	message-thread node to see if they point to the same object.
+	(add_node_diff): Adds a new thread node to the etree.
+	(remove_node_diff): Removed an etree node, freeing any additional
+	data.
+	(build_subtree_diff): Takes an existing etree definition, and a
+	new thread definition and makes the etree match, using as few
+	operations as possible.
+	(do_regenerate_messagelist): No longer free/clear the uid/rowmap
+	here.
+	(regenerate_messagelist_input_t): Added a tree field - are we
+	building a tree view?
+	(regnerate_messagelist_data_t): Added a tree field, if we built a
+	tree result.  Added a changes parameter, for building diff's after
+	search/etc.
+	(mail_do_regenerate_messagelist): Setup the tree indicator.
+	(build_flat_diff): Apply a changeset to a message list.
+	(build_flat): Added a changes argument, if present, use
+	build_flat_diff() to build the list.
+	(do_regenerate_messagelist): If we are generating a threaded view,
+	build the threaded list here, rather in another separate
+	invocation.
+	(cleanup_regenerate_messagelist): Call build_tree directly on the
+	threaded list.
+	(message_list_init): Init the uid_rowmap hash table here instead
+	of somewhere odd.
+	(message_list_destroy): Assume uid_rowmap exists.
+	(do_regenerate_messagelist): Remove the code here that is messing
+	with the message list data (search/uid_rowmap).  We're in a
+	different thread boys ...
+
+2000-10-26  Not Zed  <NotZed@HelixCode.com>
+
+	* message-list.c (cleanup_regenerate_messagelist): Fixed some
+	logic to make more sense (gboolean)!pointer replaced with
+	(pointer != NULL).
+	(build_tree): Put the tree pre/post change stuff in here, where it
+	should be.
+	(build_flat): Same here.
+	(cleanup_regenerate_messagelist): Remove model_changed stuff here.
+	(setup_regenerate_messagelist): Remove pre_change stuff here.
+
+2000-10-20  Not Zed  <NotZed@HelixCode.com>
+
+	* message-list.c (main_folder_changed): Perform incremental update
+	of the display for flat view.
+	(ml_tree_value_at): Spit out a mroe meaningful warning when we
+	can't find the uid in our tree, in the folder.
+
+	* message-thread.c (thread_messages): Made public.
+	(thread_messages_free): Made public.
+	(thread_messages): Now we also return a struct _thread_messages,
+	which is passed to other functions.
+	(container_free): Renamed from thread_messages_free.
+	(thread_messages_free): Take a thread_messages argument.
+	(thread_messages_add): New function to add a list of uid's to the
+	thread list.
+	(thread_messages_remove): Likewise, for removing them.
+	(cleanup_thread_messages): Change for struct changes.
+	(do_thread_messages): Likewise.
+
+2000-10-19  Not Zed  <NotZed@HelixCode.com>
+
+	* mail-tools.c (mail_tool_do_movemail): removed unused var
+
+	* folder-browser.c (search_full_clicked): Fix for api changes,
+	such as it can be called an api, its mroe an utter mess infact.
+	(search_set): Same.
+	(search_set): And here.
+	(folder_browser_clear_search): And here.
+
+	* message-list.c (folder_changed): Copy and forward the changeinfo
+	list to the mian thread.
+	(main_folder_changed): Free the changeinfo. Todo: something smart
+	with this information.
+	(struct regenerate_messagelist_input_s): Added a changes field.
+	(mail_do_regenerate_messagelist): Added a change list argument.
+	(message_list_set_folder): Fix for mail_do_regenreate_messagelist
+	api.
+	(message_list_toggle_threads): Same.
+
 2000-10-18  Iain Holmes  <iain@helixcode.com>
 
 	* mail-config-gui.c (mail_config): Make all the CLists have passive
diff --git a/mail/folder-browser.c b/mail/folder-browser.c
index a88b5db0ac..6051c26aff 100644
--- a/mail/folder-browser.c
+++ b/mail/folder-browser.c
@@ -165,7 +165,7 @@ search_full_clicked(MailSearchDialogue *msd, guint button, FolderBrowser *fb)
 	case 0:			/* 'ok' */
 	case 1:			/* 'search' */
 		query = mail_search_dialogue_get_query(msd);
-		mail_do_regenerate_messagelist(fb->message_list, query);
+		message_list_set_search(fb->message_list, query);
 		g_free(query);
 		/* save the search as well */
 		if (fb->search_full)
@@ -178,7 +178,7 @@ search_full_clicked(MailSearchDialogue *msd, guint button, FolderBrowser *fb)
 	case 2:			/* 'cancel' */
 		gnome_dialog_close((GnomeDialog *)msd);
 	case -1:		/* dialogue closed */
-		mail_do_regenerate_messagelist(fb->message_list, 0);
+		message_list_set_search(fb->message_list, 0);
 		/* reset the search buttons state */
 		gtk_menu_set_active(GTK_MENU(GTK_OPTION_MENU(fb->search_menu)->menu), 0);
 		gtk_widget_set_sensitive(fb->search_entry, TRUE);
@@ -223,7 +223,7 @@ search_set(FolderBrowser *fb)
 	if (text == NULL || text[0] == 0) {
 		if (text) 
 			g_free(text);
-		mail_do_regenerate_messagelist (fb->message_list, NULL);
+		message_list_set_search(fb->message_list, NULL);
 		return;
 	}
 
@@ -241,7 +241,7 @@ search_set(FolderBrowser *fb)
 			str++;
 		}
 	}
-	mail_do_regenerate_messagelist (fb->message_list, out->str);
+	message_list_set_search(fb->message_list, out->str);
 	g_string_free(out, TRUE);
 
 	g_free (text);
@@ -387,7 +387,7 @@ folder_browser_clear_search (FolderBrowser *fb)
 {
 	gtk_entry_set_text (GTK_ENTRY (fb->search_entry), "");
 	gtk_option_menu_set_history (GTK_OPTION_MENU (fb->search_menu), 0);
-	mail_do_regenerate_messagelist (fb->message_list, NULL);
+	message_list_set_search(fb->message_list, NULL);
 }
 
 static int
@@ -432,12 +432,12 @@ etable_key (ETable *table, int row, int col, GdkEvent *ev, FolderBrowser *fb)
 
 	case GDK_Home:
 	case GDK_KP_Home:
-		message_list_home (fb->message_list);
+		message_list_select(fb->message_list, 0, MESSAGE_LIST_SELECT_NEXT, 0, 0);
 		return TRUE;
 
 	case GDK_End:
 	case GDK_KP_End:
-		message_list_end (fb->message_list);
+		message_list_select(fb->message_list, -1, MESSAGE_LIST_SELECT_PREVIOUS, 0, 0);
 		return TRUE;
 
 	case 'n':
diff --git a/mail/mail-callbacks.c b/mail/mail-callbacks.c
index efe5f8d1f5..db13ea39b0 100644
--- a/mail/mail-callbacks.c
+++ b/mail/mail-callbacks.c
@@ -141,7 +141,7 @@ main_select_first_unread (CamelObject *object, gpointer event_data, gpointer dat
 	FolderBrowser *fb = FOLDER_BROWSER (data);
 	/*ETable *table = E_TABLE_SCROLLED (fb->message_list->etable)->table;*/
 	
-	message_list_select (fb->message_list, -1, MESSAGE_LIST_SELECT_NEXT,
+	message_list_select (fb->message_list, 0, MESSAGE_LIST_SELECT_NEXT,
   			     0, CAMEL_MESSAGE_SEEN);
 }
 
diff --git a/mail/mail-format.c b/mail/mail-format.c
index edf64be621..a7e46f272f 100644
--- a/mail/mail-format.c
+++ b/mail/mail-format.c
@@ -1550,13 +1550,12 @@ mail_get_message_body (CamelDataWrapper *data, gboolean want_plain, gboolean *is
 	 */
 	if (g_strcasecmp (mime_type->type, "message") == 0) {
 		*is_html = FALSE;
-		return g_strdup (get_data_wrapper_text (data));
+		return get_data_wrapper_text (data);
 	}
 	
 	if (g_strcasecmp (mime_type->type, "text") == 0) {
-		fprintf (stderr, "here we are...\n");
 		*is_html = !g_strcasecmp (mime_type->subtype, "html");
-		return g_strdup (get_data_wrapper_text (data));
+		return get_data_wrapper_text (data);
 	}
 	
 	/* If it's not message and it's not text, and it's not
diff --git a/mail/mail-ops.c b/mail/mail-ops.c
index a6fd33639a..a13205fc38 100644
--- a/mail/mail-ops.c
+++ b/mail/mail-ops.c
@@ -102,7 +102,7 @@ static void
 mail_op_report_status (FilterDriver *driver, enum filter_status_t status, const char *desc,
 		       CamelMimeMessage *msg, void *data)
 {
-	printf("reporting status: %s\n", desc);
+	/*printf("reporting status: %s\n", desc);*/
 
 	/* FIXME: make it work */
 	switch(status) {
diff --git a/mail/mail-ops.h b/mail/mail-ops.h
index 1be4dc68bc..e61f5f87ce 100644
--- a/mail/mail-ops.h
+++ b/mail/mail-ops.h
@@ -72,6 +72,3 @@ void mail_do_setup_folder (const char *name, CamelFolder **folder);
 void mail_do_view_messages (CamelFolder *folder, GPtrArray *uids,
 			    FolderBrowser *fb);
 
-/* This actually lives in message-list.c */
-void mail_do_regenerate_messagelist (MessageList *list,
-				     const gchar *search);
diff --git a/mail/main.c b/mail/main.c
index d81b456a98..d75d9d3e70 100644
--- a/mail/main.c
+++ b/mail/main.c
@@ -34,6 +34,7 @@ static int blowup(int status)
 {
 	printf("memory blew up, status %d\n", status);
 	/*abort();*/
+	return status;
 }
 
 int
diff --git a/mail/message-list.c b/mail/message-list.c
index 9c44d0a8df..832e7807d1 100644
--- a/mail/message-list.c
+++ b/mail/message-list.c
@@ -6,6 +6,7 @@
  * Author:
  *     Miguel de Icaza (miguel@helixcode.com)
  *     Bertrand Guiheneuf (bg@aful.org)
+ *     And just about everyone else in evolution ...
  *
  * (C) 2000 Helix Code, Inc.
  */
@@ -53,6 +54,16 @@
 #include "art/score-higher.xpm"
 #include "art/score-highest.xpm"
 
+#define TIMEIT
+
+#ifdef TIMEIT
+#include <sys/time.h>
+#include <unistd.h>
+#endif
+
+#define d(x)
+#define t(x) x
+
 /*
  * Default sizes for the ETable display
  *
@@ -87,10 +98,19 @@ static gint on_click (ETableScrolled *table, gint row, gint col, GdkEvent *event
 static void on_double_click (ETableScrolled *table, gint row, MessageList *list);
 static void select_msg (MessageList *message_list, gint row);
 static char *filter_date (const void *data);
-static void nuke_uids (GtkObject *o);
+static void free_tree_ids (ETreeModel *etm);
 
 static void save_tree_state(MessageList *ml);
 
+/* note: @changes is owned/freed by the caller */
+static void mail_do_regenerate_messagelist (MessageList *list, const gchar *search, CamelFolderChangeInfo *changes);
+
+/* macros for working with id's (stored in the tree nodes) */
+#define id_is_uid(id) (id[0] == 'u')/* is this a uid id? */
+#define id_is_subject(id) (id[0] == 's') /* is this a subject id? */
+#define id_uid(id) (&id[1])	/* get the uid part of the id */
+#define id_subject(id) (&id[1])	/* get the subject part of the id */
+
 static struct {
 	char **image_base;
 	GdkPixbuf  *pixbuf;
@@ -228,30 +248,6 @@ subject_compare (gconstpointer subject1, gconstpointer subject2)
 	return g_strcasecmp (sub1, sub2);
 }
 
-/* Gets the CamelMessageInfo for the message displayed at the given
- * view row.
- */
-static const CamelMessageInfo *
-get_message_info (MessageList *message_list, int row)
-{
-	ETreeModel *model = (ETreeModel *)message_list->table_model;
-	ETreePath *node;
-	char *uid;
-	
-	if (row >= e_table_model_row_count (message_list->table_model))
-		return NULL;
-	
-	node = e_tree_model_node_at_row (model, row);
-	g_return_val_if_fail (node != NULL, NULL);
-	uid = e_tree_model_node_get_data (model, node);
-	
-	if (strncmp (uid, "uid:", 4) != 0)
-		return NULL;
-	uid += 4;
-	
-	return camel_folder_get_message_info (message_list->folder, uid);
-}
-
 /* Gets the uid of the message displayed at a given view row */
 static const char *
 get_message_uid (MessageList *message_list, int row)
@@ -267,13 +263,28 @@ get_message_uid (MessageList *message_list, int row)
 	g_return_val_if_fail (node != NULL, NULL);
 	uid = e_tree_model_node_get_data (model, node);
 
-	if (strncmp (uid, "uid:", 4) != 0)
+	if (!id_is_uid(uid))
 		return NULL;
-	uid += 4;
 
-	return uid;
+	return id_uid(uid);
 }
 
+/* Gets the CamelMessageInfo for the message displayed at the given
+ * view row.
+ */
+static const CamelMessageInfo *
+get_message_info (MessageList *message_list, int row)
+{
+	const char *uid;
+	
+	uid = get_message_uid(message_list, row);
+	if (uid)
+		return camel_folder_get_message_info(message_list->folder, uid);
+
+	return NULL;
+}
+
+
 static gint 
 mark_msg_seen (gpointer data)
 {
@@ -300,7 +311,7 @@ mark_msg_seen (gpointer data)
  *
  * This moves the message list selection to a suitable row. @base_row
  * lists the first (model) row to try, but as a special case, model
- * row -1 is mapped to view row 0. @flags and @mask combine to specify
+ * row -1 is mapped to the last row. @flags and @mask combine to specify
  * what constitutes a suitable row. @direction is
  * %MESSAGE_LIST_SELECT_NEXT if it should find the next matching
  * message, or %MESSAGE_LIST_SELECT_PREVIOUS if it should find the
@@ -316,22 +327,30 @@ message_list_select (MessageList *message_list, int base_row,
 	int vrow, mrow, last;
 	ETableScrolled *ets = E_TABLE_SCROLLED (message_list->etable);
 
-	if (direction == MESSAGE_LIST_SELECT_PREVIOUS)
-		last = 0;
-	else
+	switch (direction) {
+	case MESSAGE_LIST_SELECT_PREVIOUS:
+		last = -1;
+		break;
+	case MESSAGE_LIST_SELECT_NEXT:
 		last = e_table_model_row_count (message_list->table_model);
+		break;
+	default:
+		g_warning("Invalid argument to message_list_select");
+		return;
+	}
 
 	if (base_row == -1)
-		vrow = 0;
-	else
-		vrow = e_table_model_to_view_row (ets->table, base_row);
+		base_row = e_table_model_row_count(message_list->table_model) - 1;
+
+	/* model_to_view_row etc simply dont work for sorted views.  Sigh. */
+	vrow = e_table_model_to_view_row (ets->table, base_row);
 
 	/* We don't know whether to use < or > due to "direction" */
 	while (vrow != last) {
 		mrow = e_table_view_to_model_row (ets->table, vrow);
 		info = get_message_info (message_list, mrow);
 		if (info && (info->flags & mask) == flags) {
-			e_table_scrolled_set_cursor_row (ets, mrow);
+			e_table_scrolled_set_cursor_row (ets, vrow);
 			mail_do_display_message (message_list, info->uid, mark_msg_seen);
 			return;
 		}
@@ -593,8 +612,8 @@ subtree_unread(MessageList *ml, ETreePath *node)
 	while (node) {
 		ETreePath *child;
 		uid = e_tree_model_node_get_data((ETreeModel *)ml->table_model, node);
-		if (strncmp (uid, "uid:", 4) == 0) {
-			info = camel_folder_get_message_info(ml->folder, uid+4);
+		if (id_is_uid(uid)) {
+			info = camel_folder_get_message_info(ml->folder, id_uid(uid));
 			if (!(info->flags & CAMEL_MESSAGE_SEEN))
 				return TRUE;
 		}
@@ -645,12 +664,15 @@ ml_tree_value_at (ETreeModel *etm, ETreePath *path, int col, void *model_data)
 
 	/* retrieve the message information array */
 	uid = e_tree_model_node_get_data (etm, path);
-	if (strncmp (uid, "uid:", 4) != 0)
+	if (!id_is_uid(uid))
 		goto fake;
-	uid += 4;
+	uid = id_uid(uid);
 
 	msg_info = camel_folder_get_message_info (message_list->folder, uid);
-	g_return_val_if_fail (msg_info != NULL, NULL);
+	if (msg_info == NULL) {
+		g_warning("UID for message-list not found in folder: %s", uid);
+		return NULL;
+	}
 	
 	switch (col){
 	case COL_MESSAGE_STATUS:
@@ -759,7 +781,7 @@ ml_tree_value_at (ETreeModel *etm, ETreePath *path, int col, void *model_data)
 		return (void *) 0;
 		
 	case COL_SUBJECT:
-		return strchr (uid, ':') + 1;
+		return id_subject(uid);
 		
 	case COL_FROM:
 	case COL_TO:
@@ -986,7 +1008,7 @@ message_list_init (GtkObject *object)
 				   message_list);
 	e_tree_model_root_node_set_visible ((ETreeModel *)message_list->table_model, FALSE);
 	gtk_signal_connect (GTK_OBJECT (message_list->table_model), "destroy", 
-			    (GtkSignalFunc) nuke_uids, NULL);
+			    (GtkSignalFunc) free_tree_ids, NULL);
 
 	/*
 	 * The etable
@@ -1040,12 +1062,8 @@ message_list_init (GtkObject *object)
 	 */
 	gtk_object_ref (GTK_OBJECT (message_list->etable));
 	gtk_object_sink (GTK_OBJECT (message_list->etable));
-}
 
-static void
-free_key (gpointer key, gpointer value, gpointer data)
-{
-	g_free (key);
+	message_list->uid_rowmap = g_hash_table_new (g_str_hash, g_str_equal);
 }
 
 static void
@@ -1061,12 +1079,8 @@ message_list_destroy (GtkObject *object)
 	gtk_object_unref (GTK_OBJECT (message_list->table_model));
 	gtk_object_unref (GTK_OBJECT (message_list->etable));
 	
-	if (message_list->uid_rowmap) {
-		g_hash_table_foreach (message_list->uid_rowmap,
-				      free_key, NULL);
-		g_hash_table_destroy (message_list->uid_rowmap);
-	}
-	
+	g_hash_table_destroy (message_list->uid_rowmap);
+
 	if (message_list->idle_id != 0)
 		g_source_remove(message_list->idle_id);
 	
@@ -1193,10 +1207,15 @@ clear_tree (MessageList *ml)
 {
 	ETreeModel *etm = E_TREE_MODEL (ml->table_model);
 
+	/* we also reset the uid_rowmap since it is no longer useful/valid anyway */
+	g_hash_table_destroy (ml->uid_rowmap);
+	ml->uid_rowmap = g_hash_table_new(g_str_hash, g_str_equal);
+	free_tree_ids(etm);
+	
 	if (ml->tree_root)
 		e_tree_model_node_remove (etm, ml->tree_root);
-	ml->tree_root =
-		e_tree_model_node_insert (etm, NULL, 0, NULL);
+
+	ml->tree_root = e_tree_model_node_insert (etm, NULL, 0, NULL);
 	e_tree_model_node_set_expanded (etm, ml->tree_root, TRUE);
 }
 
@@ -1214,8 +1233,8 @@ save_node_state(MessageList *ml, FILE *out, ETreePath *node)
 		    && !e_tree_model_node_is_expanded((ETreeModel *)ml->table_model, node)) {
 			data = e_tree_model_node_get_data((ETreeModel *)ml->table_model, node);
 			if (data) {
-				if (!strncmp(data, "uid:", 4)) {
-					info = camel_folder_get_message_info(ml->folder, data+4);
+				if (id_is_uid(data)) {
+					info = camel_folder_get_message_info(ml->folder, id_uid(data));
 					if (info) {
 						fprintf(out, "%s\n", info->message_id);
 					}
@@ -1295,30 +1314,102 @@ free_tree_state(GHashTable *expanded_nodes)
 /* builds the tree structure */
 static void build_subtree (MessageList *ml, ETreePath *parent, struct _container *c, int *row, GHashTable *);
 
+static void build_subtree_diff (MessageList *ml, ETreePath *parent, ETreePath *path, struct _container *c, int *row, GHashTable *expanded_nodes);
+static void remove_node_diff(MessageList *ml, ETreePath *node, int depth, int *row);
+
 static void
-build_tree (MessageList *ml, struct _container *c)
+build_tree (MessageList *ml, struct _thread_messages *thread)
 {
 	int row = 0;
 	GHashTable *expanded_nodes;
+	ETreeModel *etm = (ETreeModel *)ml->table_model;
+	ETreePath *top;
+
+#ifdef TIMEIT
+	struct timeval start, end;
+	unsigned long diff;
+
+	printf("Building tree\n");
+	gettimeofday(&start, NULL);
+#endif
+	e_table_model_pre_change(ml->table_model);
 
-	clear_tree (ml);
 	expanded_nodes = load_tree_state(ml);
-	build_subtree (ml, ml->tree_root, c, &row, expanded_nodes);
+
+#ifdef TIMEIT
+	gettimeofday(&end, NULL);
+	diff = end.tv_sec * 1000 + end.tv_usec/1000;
+	diff -= start.tv_sec * 1000 + start.tv_usec/1000;
+	printf("Loading tree state took %ld.%03ld seconds\n", diff / 1000, diff % 1000);
+#endif
+
+	if (ml->tree_root == NULL) {
+		ml->tree_root =	e_tree_model_node_insert(etm, NULL, 0, NULL);
+		e_tree_model_node_set_expanded(etm, ml->tree_root, TRUE);
+	}
+
+	top = e_tree_model_node_get_first_child(etm, ml->tree_root);
+	if (top == NULL) {
+		build_subtree(ml, ml->tree_root, thread->tree, &row, expanded_nodes);
+	} else {
+		build_subtree_diff(ml, ml->tree_root, top,  thread->tree, &row, expanded_nodes);
+	}
+
 	free_tree_state(expanded_nodes);
+	
+	e_table_model_changed(ml->table_model);
+
+#ifdef TIMEIT
+	gettimeofday(&end, NULL);
+	diff = end.tv_sec * 1000 + end.tv_usec/1000;
+	diff -= start.tv_sec * 1000 + start.tv_usec/1000;
+	printf("Building tree took %ld.%03ld seconds\n", diff / 1000, diff % 1000);
+#endif
+
+}
+
+static char *
+new_id_from_uid(const char *uid)
+{
+	char *res;
+	int len;
+
+	len = strlen(uid)+2;
+	res = g_malloc(len);
+	res[0] = 'u';
+	strcpy(res+1, uid);
+	return res;
 }
 
+static char *
+new_id_from_subject(const char *subject)
+{
+	char *res;
+	int len;
+
+	len = strlen(subject)+2;
+	res = g_malloc(len);
+	res[0] = 's';
+	strcpy(res+1, subject);
+	return res;
+}
+
+/* this is about 20% faster than build_subtree_diff,
+   entirely because e_tree_model_node_insert(xx, -1 xx)
+   is faster than inserting to the right row :( */
+/* Otherwise, this code would probably go as it does the same thing essentially */
 static void
 build_subtree (MessageList *ml, ETreePath *parent, struct _container *c, int *row, GHashTable *expanded_nodes)
 {
 	ETreeModel *tree = E_TREE_MODEL (ml->table_model);
 	ETreePath *node;
 	char *id;
-	int expanded = FALSE;	/* just removes a silly warning */
+	int expanded = FALSE;
 
 	while (c) {
 		if (c->message) {
-			id = g_strdup_printf("uid:%s", c->message->uid);
-			g_hash_table_insert(ml->uid_rowmap, g_strdup (c->message->uid), GINT_TO_POINTER (*row));
+			id = new_id_from_uid(c->message->uid);
+			g_hash_table_insert(ml->uid_rowmap, id_uid(id), GINT_TO_POINTER ((*row)++));
 			if (c->child) {
 				if (c->message && c->message->message_id)
 					expanded = !g_hash_table_lookup(expanded_nodes, c->message->message_id) != 0;
@@ -1326,13 +1417,12 @@ build_subtree (MessageList *ml, ETreePath *parent, struct _container *c, int *ro
 					expanded = TRUE;
 			}
 		} else {
-			id = g_strdup_printf("subject:%s", c->root_subject);
+			id = new_id_from_subject(c->root_subject);
 			if (c->child) {
 				expanded = !g_hash_table_lookup(expanded_nodes, id) != 0;
 			}
 		}
-		node = e_tree_model_node_insert(tree, parent, 0, id);
-		(*row) ++;
+		node = e_tree_model_node_insert(tree, parent, -1, id);
 		if (c->child) {
 			/* by default, open all trees */
 			if (expanded)
@@ -1343,53 +1433,419 @@ build_subtree (MessageList *ml, ETreePath *parent, struct _container *c, int *ro
 	}
 }
 
+/* compares a thread tree node with the etable tree node to see if they point to
+   the same object */
+static int
+node_equal(ETreeModel *etm, ETreePath *ap, struct _container *bp)
+{
+	char *uid;
+
+	uid = e_tree_model_node_get_data(etm, ap);
+
+	if (id_is_uid(uid)) {
+		if (bp->message && strcmp(id_uid(uid), bp->message->uid)==0)
+			return 1;
+	} else if (id_is_subject(uid)) {
+		if (bp->message == NULL && strcmp(id_subject(uid), bp->root_subject) == 0)
+			return 1;
+	}
+	return 0;
+}
+
+/* adds a single node, retains save state, and handles adding children if required */
+static void
+add_node_diff(MessageList *ml, ETreePath *parent, ETreePath *path, struct _container *c, int *row, int myrow, GHashTable *expanded_nodes)
+{
+	ETreeModel *etm = E_TREE_MODEL (ml->table_model);
+	ETreePath *node;
+	char *id;
+	int expanded = FALSE;
+
+	if (c->message) {
+		id = new_id_from_uid(c->message->uid);
+		/* need to remove the id first, as GHashTable' wont replace the key pointer for us */
+		g_hash_table_remove(ml->uid_rowmap, id_uid(id));
+		g_hash_table_insert(ml->uid_rowmap, id_uid(id), GINT_TO_POINTER (*row));
+		if (c->child) {
+			if (c->message && c->message->message_id)
+				expanded = !g_hash_table_lookup(expanded_nodes, c->message->message_id) != 0;
+			else
+				expanded = TRUE;
+		}
+	} else {
+		id = new_id_from_subject(c->root_subject);
+		if (c->child) {
+			expanded = !g_hash_table_lookup(expanded_nodes, id) != 0;
+		}
+	}
+
+	t(printf("Adding node: %s\n", id));
+
+	node = e_tree_model_node_insert(etm, parent, myrow, id);
+	(*row)++;
+	if (c->child) {
+		e_tree_model_node_set_expanded(etm, node, expanded);
+		t(printf("Building subtree ...\n"));
+		build_subtree_diff(ml, node, NULL, c->child, row, expanded_nodes);
+	}
+}
+
+/* removes node, children recursively and all associated data */
+static void
+remove_node_diff(MessageList *ml, ETreePath *node, int depth, int *row)
+{
+	ETreeModel *etm = E_TREE_MODEL (ml->table_model);
+	ETreePath *cp, *cn;
+	char *uid, *olduid;
+	int oldrow;
+
+	t(printf("Removing node: %s\n", (char *)e_tree_model_node_get_data(etm, node)));
+
+	/* we depth-first remove all node data's ... */
+	cp = e_tree_model_node_get_first_child(etm, node);
+	while (cp) {
+		cn = e_tree_model_node_get_next(etm, cp);
+		remove_node_diff(ml, cp, depth+1, row);
+		cp = cn;
+	}
+
+	/* and the rowid entry - if and only if it is referencing this node */
+	uid = e_tree_model_node_get_data(etm, node);
+	if (id_is_uid(uid)
+	    && g_hash_table_lookup_extended(ml->uid_rowmap, id_uid(uid), (void *)&olduid, (void *)&oldrow)
+	    && olduid == id_uid(uid)) {
+		printf("removing rowid map entry: %s\n", id_uid(uid));
+		g_hash_table_remove(ml->uid_rowmap, id_uid(uid));
+	}
+	g_free(uid);
+	e_tree_model_node_set_data(etm, node, NULL);
+
+	/**row = *row - 1;*/
+
+	/* and only at the toplevel, remove the node (etree should optimise this remove somewhat) */
+	if (depth == 0)
+		e_tree_model_node_remove(etm, node);
+}
+
+/* applies a new tree structure to an existing tree, but only by changing things
+   that have changed */
+static void
+build_subtree_diff(MessageList *ml, ETreePath *parent, ETreePath *path, struct _container *c, int *row, GHashTable *expanded_nodes)
+{
+	ETreeModel *etm = E_TREE_MODEL (ml->table_model);
+	ETreePath *ap, *ai, *at, *tmp;
+	struct _container *bp, *bi, *bt;
+	int i, j, myrow = 0;
+
+	ap = path;
+	bp = c;
+
+	while (ap || bp) {
+		/*t(printf("Processing row: %d (subtree row %d)\n", *row, myrow));*/
+		if (ap == NULL) {
+			t(printf("out of old nodes\n"));
+			/* ran out of old nodes - remaining nodes are added */
+			add_node_diff(ml, parent, ap, bp, row, myrow, expanded_nodes);
+			myrow++;
+			bp = bp->next;
+		} else if (bp == NULL) {
+			t(printf("out of new nodes\n"));
+			/* ran out of new nodes - remaining nodes are removed */
+			tmp = e_tree_model_node_get_next(etm, ap);
+			remove_node_diff(ml, ap, 0, row);
+			ap = tmp;
+		} else if (node_equal(etm, ap, bp)) {
+			/*t(printf("nodes match, verify\n"));*/
+			/* matching nodes, verify details/children */
+			if (bp->message) {
+				char *olduid;
+				int oldrow;
+
+				/* if this is a message row, check/update the row id map */
+				if (g_hash_table_lookup_extended(ml->uid_rowmap, bp->message->uid, (void *)&olduid, (void *)&oldrow)) {
+					if (oldrow != (*row)) {
+						g_hash_table_insert(ml->uid_rowmap, olduid, (void *)(*row));
+					}
+				} else {
+					g_warning("Cannot find uid %s in table?", bp->message->uid);
+					/*g_assert_not_reached();*/
+				}
+			}
+			*row = (*row)+1;
+			myrow++;
+			tmp = e_tree_model_node_get_first_child(etm, ap);
+			/* make child lists match (if either has one) */
+			if (bp->child || tmp) {
+				build_subtree_diff(ml, ap, tmp, bp->child, row, expanded_nodes);
+			}
+			ap = e_tree_model_node_get_next(etm, ap);
+			bp = bp->next;
+		} else {
+			t(printf("searching for matches\n"));
+			/* we have to scan each side for a match */
+			bi = bp->next;
+			ai = e_tree_model_node_get_next(etm, ap);
+			for (i=1;bi!=NULL;i++,bi=bi->next) {
+				if (node_equal(etm, ap, bi))
+					break;
+			}
+			for (j=1;ai!=NULL;j++,ai=e_tree_model_node_get_next(etm, ai)) {
+				if (node_equal(etm, ai, bp))
+					break;
+			}
+			if (i<j) {
+				/* smaller run of new nodes - must be nodes to add */
+				if (bi) {
+					bt = bp;
+					while (bt != bi) {
+						printf("adding new node 0\n");
+						add_node_diff(ml, parent, NULL, bt, row, myrow, expanded_nodes);
+						myrow++;
+						bt = bt->next;
+					}
+					bp = bi;
+				} else {
+					printf("adding new node 1\n");
+					/* no match in new nodes, add one, try next */
+					add_node_diff(ml, parent, NULL, bp, row, myrow, expanded_nodes);
+					myrow++;
+					bp = bp->next;
+				}
+			} else {
+				/* bigger run of old nodes - must be nodes to remove */
+				if (ai) {
+					at = ap;
+					while (at != ai) {
+						printf("removing old node 0\n");
+						tmp = e_tree_model_node_get_next(etm, at);
+						remove_node_diff(ml, at, 0, &myrow);
+						at = tmp;
+					}
+					ap = ai;
+				} else {
+					printf("adding new node 2\n");
+					/* didn't find match in old nodes, must be new node? */
+					add_node_diff(ml, parent, NULL, bp, row, myrow, expanded_nodes);
+					myrow++;
+					bp = bp->next;
+#if 0
+					tmp = e_tree_model_node_get_next(etm, ap);
+					remove_node_diff(etm, ap, 0, &myrow);
+					ap = tmp;
+#endif
+				}
+			}
+		}
+	}
+}
+
 static gboolean
-nuke_uids_cb (ETreeModel *model, ETreePath *node, gpointer data)
+free_ids_cb (ETreeModel *model, ETreePath *node, gpointer data)
 {
 	g_free (e_tree_model_node_get_data (model, node));
 	return FALSE;
 }
 
 static void
-nuke_uids (GtkObject *o)
+free_tree_ids (ETreeModel *etm)
 {
-	ETreeModel *etm = E_TREE_MODEL (o);
 	ETreePath *root = e_tree_model_get_root (etm);
 
-	if (root)
-		e_tree_model_node_traverse (etm, root,
-					    nuke_uids_cb, NULL);
+	if (e_tree_model_get_root(etm))
+		e_tree_model_node_traverse (etm, e_tree_model_get_root(etm), free_ids_cb, NULL);
 }
 
+static void build_flat_diff(MessageList *ml, CamelFolderChangeInfo *changes);
+
 static void
-build_flat (MessageList *ml, GPtrArray *uids)
+build_flat (MessageList *ml, GPtrArray *uids, CamelFolderChangeInfo *changes)
 {
 	ETreeModel *tree = E_TREE_MODEL (ml->table_model);
 	ETreePath *node;
 	char *uid;
 	int i;
 
-	clear_tree (ml);
-	for (i = 0; i < uids->len; i++) {
-		uid = g_strdup_printf ("uid:%s", (char *)uids->pdata[i]);
-		node = e_tree_model_node_insert (tree, ml->tree_root, i, uid);
-		g_hash_table_insert (ml->uid_rowmap, g_strdup (uids->pdata[i]),
-				     GINT_TO_POINTER (i));
+#ifdef TIMEIT
+	struct timeval start, end;
+	unsigned long diff;
+
+	printf("Building flat\n");
+	gettimeofday(&start, NULL);
+#endif
+
+	e_table_model_pre_change(ml->table_model);
+
+	if (changes) {
+		build_flat_diff(ml, changes);
+	} else {
+		clear_tree (ml);
+		for (i = 0; i < uids->len; i++) {
+			uid = new_id_from_uid(uids->pdata[i]);
+			node = e_tree_model_node_insert (tree, ml->tree_root, -1, uid);
+			g_hash_table_insert (ml->uid_rowmap, id_uid(uid), GINT_TO_POINTER (i));
+		}
+	}
+
+	e_table_model_changed(ml->table_model);
+
+#ifdef TIMEIT
+	gettimeofday(&end, NULL);
+	diff = end.tv_sec * 1000 + end.tv_usec/1000;
+	diff -= start.tv_sec * 1000 + start.tv_usec/1000;
+	printf("Building flat took %ld.%03ld seconds\n", diff / 1000, diff % 1000);
+#endif
+
+}
+
+/* used to sort the rows to match list order */
+struct _uidsort {
+	int row;
+	char *uid;
+};
+
+static int
+sort_uid_cmp(const void *ap, const void *bp)
+{
+	const struct _uidsort *a = (struct _uidsort *)ap;
+	const struct _uidsort *b = (struct _uidsort *)bp;
+
+	if (a->row < b->row)
+		return -1;
+	else if (a->row > b->row)
+		return 1;
+	return 0;
+}
+
+static void
+sort_uid_to_rows(MessageList *ml, GPtrArray *uids)
+{
+	struct _uidsort *uidlist;
+	int i;
+
+	uidlist = g_malloc(sizeof(struct _uidsort) * uids->len);
+	for (i=0;i<uids->len;i++) {
+		uidlist[i].row = (int)g_hash_table_lookup(ml->uid_rowmap, uids->pdata[i]);
+		uidlist[i].uid = uids->pdata[i];
+	}
+	qsort(uidlist, uids->len, sizeof(struct _uidsort), sort_uid_cmp);
+	for (i=0;i<uids->len;i++) {
+		uids->pdata[i] = uidlist[i].uid;
 	}
+	g_free(uidlist);
+}
+
+static void
+build_flat_diff(MessageList *ml, CamelFolderChangeInfo *changes)
+{
+	int row, i;
+	ETreePath *node;
+	char *uid;
+	int oldrow;
+	char *olduid;
+
+#ifdef TIMEIT
+	struct timeval start, end;
+	unsigned long diff;
+		
+	gettimeofday(&start, NULL);
+#endif
+
+	printf("updating changes to display\n");
+
+	/* remove individual nodes? */
+	if (changes->uid_removed->len > 0) {
+		/* first, we need to sort the row id's to match the summary order */
+		sort_uid_to_rows(ml, changes->uid_removed);
+
+		/* we remove from the end, so that the rowmap remains valid as we go */
+		d(printf("Removing messages from view:\n"));
+		for (i=changes->uid_removed->len-1;i>=0;i--) {
+			d(printf(" %s\n", (char *)changes->uid_removed->pdata[i]));
+			if (g_hash_table_lookup_extended(ml->uid_rowmap, changes->uid_removed->pdata[i], (void *)&olduid, (void *)&row)) {
+				node = e_tree_model_node_at_row((ETreeModel *)ml->table_model, row);
+				uid = e_tree_model_node_get_data((ETreeModel *)ml->table_model, node);
+				if (uid && id_is_uid(uid) && !strcmp(id_uid(uid), changes->uid_removed->pdata[i])) {
+					g_hash_table_remove(ml->uid_rowmap, olduid);
+					e_tree_model_node_remove((ETreeModel *)ml->table_model, node);
+					g_free(uid);
+					d(printf("  - removed\n"));
+				} else {
+					d(printf("  - is this the right uid, it doesn't match my map?\n"));
+				}
+			}
+		}
+	}
+
+	/* add new nodes? - just append to the end */
+	if (changes->uid_added->len > 0) {
+		node = e_tree_model_node_get_last_child((ETreeModel *)ml->table_model, ml->tree_root);
+		row = e_tree_model_row_of_node((ETreeModel *)ml->table_model, node) + 1;
+		d(printf("Adding messages to view:\n"));
+		for (i=0;i<changes->uid_added->len;i++) {
+			d(printf(" %s\n", (char *)changes->uid_added->pdata[i]));
+			uid = new_id_from_uid(changes->uid_added->pdata[i]);
+			node = e_tree_model_node_insert((ETreeModel *)ml->table_model, ml->tree_root, row, uid);
+			g_hash_table_insert(ml->uid_rowmap, id_uid(uid), GINT_TO_POINTER (row));
+			row++;
+		}
+	}
+
+	/* now, check the rowmap, some rows might've changed (with removes) */
+	if (changes->uid_removed->len) {
+		d(printf("checking uid mappings\n"));
+		row = 0;
+		node = e_tree_model_node_get_first_child ((ETreeModel *)ml->table_model, ml->tree_root);
+		while (node) {
+			uid = e_tree_model_node_get_data((ETreeModel *)ml->table_model, node);
+			if (id_is_uid(uid)) {
+				if (g_hash_table_lookup_extended(ml->uid_rowmap, id_uid(uid), (void *)&olduid, (void *)&oldrow)) {
+					if (oldrow != row) {
+						d(printf("row %d moved to new row %d\n", oldrow, row));
+						g_hash_table_insert(ml->uid_rowmap, olduid, (void *)row);
+					}
+				} else { /* missing?  shouldn't happen */
+					g_warning("Uid vanished from rowmap?: %s\n", uid);
+				}
+			}
+			row++;
+			node = e_tree_model_node_get_next((ETreeModel *)ml->table_model, node);
+		}
+	}
+
+#ifdef TIMEIT
+	gettimeofday(&end, NULL);
+	diff = end.tv_sec * 1000 + end.tv_usec/1000;
+	diff -= start.tv_sec * 1000 + start.tv_usec/1000;
+	printf("Inserting changes took %ld.%03ld seconds\n", diff / 1000, diff % 1000);
+#endif
+
 }
 
 static void
 main_folder_changed (CamelObject *o, gpointer event_data, gpointer user_data)
 {
-	MessageList *message_list = MESSAGE_LIST (user_data);
+	MessageList *ml = MESSAGE_LIST (user_data);
+	CamelFolderChangeInfo *changes = (CamelFolderChangeInfo *)event_data;
 
-	mail_do_regenerate_messagelist (message_list, message_list->search);
+	printf("folder changed event, changes = %p\n", changes);
+
+	mail_do_regenerate_messagelist(ml, ml->search, changes);
 }
 
 static void
 folder_changed (CamelObject *o, gpointer event_data, gpointer user_data)
 {
-	mail_op_forward_event (main_folder_changed, o, event_data, user_data);
+	/* similarly to message_changed, copy the change list and propagate it to
+	   the main thread and free it */
+	CamelFolderChangeInfo *changes;
+
+	if (event_data) {
+		changes = camel_folder_change_info_new();
+		camel_folder_change_info_cat(changes, (CamelFolderChangeInfo *)event_data);
+	} else {
+		changes = NULL;
+	}
+	mail_op_forward_event (main_folder_changed, o, changes, user_data);
 }
 
 static void
@@ -1427,8 +1883,11 @@ message_list_set_folder (MessageList *message_list, CamelFolder *camel_folder)
 	g_return_if_fail (CAMEL_IS_FOLDER (camel_folder));
 	g_return_if_fail (camel_folder_has_summary_capability (camel_folder));
 
+	if (message_list->folder == camel_folder)
+		return;
+
 	camel_exception_init (&ex);
-	
+
 	if (message_list->folder)
 		camel_object_unref (CAMEL_OBJECT (message_list->folder));
 
@@ -1444,9 +1903,8 @@ message_list_set_folder (MessageList *message_list, CamelFolder *camel_folder)
 
 	camel_object_ref (CAMEL_OBJECT (camel_folder));
 
-	/*gtk_idle_add (regen_message_list, message_list);*/
-	/*folder_changed (CAMEL_OBJECT (camel_folder), 0, message_list);*/
-	mail_do_regenerate_messagelist (message_list, message_list->search);
+	clear_tree(message_list);
+	mail_do_regenerate_messagelist (message_list, message_list->search, NULL);
 }
 
 GtkWidget *
@@ -1493,7 +1951,7 @@ idle_select_row (gpointer user_data)
 {
 	MessageList *ml = MESSAGE_LIST (user_data);
 
-	message_list_select (ml, -1, MESSAGE_LIST_SELECT_NEXT,
+	message_list_select (ml, 0, MESSAGE_LIST_SELECT_NEXT,
 			     0, CAMEL_MESSAGE_SEEN);
 	return FALSE;
 }
@@ -1562,6 +2020,7 @@ filter_mlist (GtkWidget *w, FolderBrowser *fb)
 	g_free (header_value);
 }
 
+/* TODO: Remove this stuff to external functions, provide the api to allow it */
 static gint
 on_right_click (ETableScrolled *table, gint row, gint col, GdkEvent *event, MessageList *list)
 {
@@ -1696,6 +2155,7 @@ message_list_foreach (MessageList *message_list,
 					       mlfe_callback, &mlfe_data);
 }
 
+/* FIXME: this should not be part of the message list api */
 void
 message_list_toggle_threads (BonoboUIComponent           *component,
 			     const char                  *path,
@@ -1708,19 +2168,50 @@ message_list_toggle_threads (BonoboUIComponent           *component,
 	if (type != Bonobo_UIComponent_STATE_CHANGED)
 		return;
 	
-	mail_config_set_thread_list (atoi (state));
-	mail_do_regenerate_messagelist (ml, ml->search);
+	mail_config_set_thread_list(atoi(state));
+	message_list_set_threaded(ml, atoi(state));
+}
+
+/* set whether we are in threaded view or flat view */
+void
+message_list_set_threaded(MessageList *ml, gboolean threaded)
+{
+	if (ml->threaded ^ threaded) {
+		ml->threaded = threaded;
+
+		clear_tree(ml);
+		mail_do_regenerate_messagelist(ml, ml->search, NULL);
+	}
+}
+
+void
+message_list_set_search(MessageList *ml, const char *search)
+{
+	if (search == NULL || search[0] == '\0')
+		if (ml->search == NULL || ml->search[0]=='\0')
+			return;
+
+	if (search != NULL && ml->search !=NULL && !strcmp(search, ml->search))
+		return;
+
+	clear_tree(ml);
+	mail_do_regenerate_messagelist(ml, search, NULL);
 }
 
 /* ** REGENERATE MESSAGELIST ********************************************** */
 
 typedef struct regenerate_messagelist_input_s {
 	MessageList *ml;
+	CamelFolder *folder;
 	char *search;
+	CamelFolderChangeInfo *changes;
+	gboolean dotree;	/* we are building a tree */
 } regenerate_messagelist_input_t;
 
 typedef struct regenerate_messagelist_data_s {
 	GPtrArray *uids;
+	struct _thread_messages *tree;
+	CamelFolderChangeInfo *changes;
 } regenerate_messagelist_data_t;
 
 static gchar *describe_regenerate_messagelist (gpointer in_data, gboolean gerund);
@@ -1740,8 +2231,13 @@ static void setup_regenerate_messagelist (gpointer in_data, gpointer op_data, Ca
 {
 	regenerate_messagelist_input_t *input = (regenerate_messagelist_input_t *) in_data;
 
+	if (!IS_MESSAGE_LIST (input->ml)) {
+		camel_exception_set (ex, CAMEL_EXCEPTION_INVALID_PARAM,
+				     "No messagelist specified to regenerate");
+		return;
+	}
+
 	gtk_object_ref (GTK_OBJECT (input->ml));
-	e_table_model_pre_change (input->ml->table_model);
 }
 
 static void do_regenerate_messagelist (gpointer in_data, gpointer op_data, CamelException *ex)
@@ -1749,31 +2245,23 @@ static void do_regenerate_messagelist (gpointer in_data, gpointer op_data, Camel
 	regenerate_messagelist_input_t *input = (regenerate_messagelist_input_t *) in_data;
 	regenerate_messagelist_data_t *data = (regenerate_messagelist_data_t *) op_data;
 
-        if (input->ml->search) {
-                g_free (input->ml->search);
-                input->ml->search = NULL;
-        }
-
-        if (input->ml->uid_rowmap) {
-                g_hash_table_foreach (input->ml->uid_rowmap,
-                                      free_key, NULL);
-                g_hash_table_destroy (input->ml->uid_rowmap);
-        }
-        input->ml->uid_rowmap = g_hash_table_new (g_str_hash, g_str_equal);
-
 	mail_tool_camel_lock_up();
 
 	if (input->search) {
-		data->uids = camel_folder_search_by_expression (input->ml->folder,
-								input->search, ex);
-		if (camel_exception_is_set (ex)) {
-			mail_tool_camel_lock_down();
-			return;
-		}
-
-		input->ml->search = g_strdup (input->search);
-	} else
+		data->uids = camel_folder_search_by_expression(input->ml->folder, input->search, ex);
+	} else {
 		data->uids = camel_folder_get_uids (input->ml->folder);
+	}
+
+	if (camel_exception_is_set (ex)) {
+		mail_tool_camel_lock_down();
+		return;
+	}
+
+	if (input->dotree && data->uids)
+		data->tree = thread_messages(input->ml->folder, data->uids);
+	else
+		data->tree = NULL;
 
 	mail_tool_camel_lock_down();
 }
@@ -1787,30 +2275,32 @@ static void cleanup_regenerate_messagelist (gpointer in_data, gpointer op_data,
 
 	etm = E_TREE_MODEL (input->ml->table_model);
 
-	/* FIXME: free the old tree data */
-
 	if (data->uids == NULL) { /*exception*/
 		gtk_object_unref (GTK_OBJECT (input->ml));
 		return;
 	}
 
-	if (mail_config_thread_list()) {
-		mail_do_thread_messages (input->ml, data->uids, 
-					 (gboolean) !(input->search),
-					 build_tree);
-	} else {
-		build_flat (input->ml, data->uids);
+	if (input->dotree)
+		build_tree(input->ml, data->tree);
+	else
+		build_flat(input->ml, data->uids, input->changes);
 
-		if (input->search) {
-			camel_folder_search_free (input->ml->folder, data->uids);
-		} else {
-			camel_folder_free_uids (input->ml->folder, data->uids);
-		}
-	}
+	if (input->search)
+		camel_folder_search_free (input->ml->folder, data->uids);
+	else
+		camel_folder_free_uids (input->ml->folder, data->uids);
+
+	/* update what we have as our search string */
+        if (input->ml->search)
+                g_free(input->ml->search);
+	input->ml->search = input->search;
+
+	if (data->tree)
+		thread_messages_free(data->tree);
+
+	if (input->changes)
+		camel_folder_change_info_free(input->changes);
 
-	e_table_model_changed (input->ml->table_model);
-	select_row (NULL, input->ml);
-	g_free (input->search);
 	gtk_object_unref (GTK_OBJECT (input->ml));
 }
 
@@ -1823,7 +2313,9 @@ static const mail_operation_spec op_regenerate_messagelist =
 	cleanup_regenerate_messagelist
 };
 
-void mail_do_regenerate_messagelist (MessageList *list, const gchar *search)
+/* if changes == NULL, then update the whole list, otherwise just update the changes */
+static void
+mail_do_regenerate_messagelist (MessageList *list, const gchar *search, CamelFolderChangeInfo *changes)
 {
 	regenerate_messagelist_input_t *input;
 
@@ -1833,55 +2325,19 @@ void mail_do_regenerate_messagelist (MessageList *list, const gchar *search)
 	if (!list->folder)
 		return;
 
+	/* see if we need to goto the child thread at all anyway */
+	/* currently the only case is the flat view with updates and no search */
+	if (search == NULL && changes != NULL && !list->threaded) {
+		build_flat_diff(list, changes);
+		camel_folder_change_info_free(changes);
+		return;
+	}
+
 	input = g_new (regenerate_messagelist_input_t, 1);
 	input->ml = list;
 	input->search = g_strdup (search);
+	input->changes = changes;
+	input->dotree = list->threaded;
 
 	mail_operation_queue (&op_regenerate_messagelist, input, TRUE);
 }
-
-static void
-go_to_message (MessageList *message_list,
-	       int model_row)
-{
-	ETableScrolled *table_scrolled;
-	const CamelMessageInfo *info;
-	int view_row;
-
-	table_scrolled = E_TABLE_SCROLLED (message_list->etable);
-
-	view_row = e_table_model_to_view_row (table_scrolled->table, model_row);
-	info     = get_message_info (message_list, model_row);
-
-	if (info != NULL) {
-		e_table_scrolled_set_cursor_row (table_scrolled, view_row);
-		mail_do_display_message (message_list, info->uid, mark_msg_seen);
-	}
-}
-
-void
-message_list_home (MessageList *message_list)
-{
-	g_return_if_fail (message_list != NULL);
-
-	go_to_message (message_list, 0);
-}
-
-void
-message_list_end (MessageList *message_list)
-{
-	ETableScrolled *table_scrolled;
-	ETable *table;
-	int num_rows;
-
-	g_return_if_fail (message_list != NULL);
-
-	table_scrolled = E_TABLE_SCROLLED (message_list->etable);
-	table = table_scrolled->table;
-
-	num_rows = e_table_model_row_count (table->model);
-	if (num_rows == 0)
-		return;
-
-	go_to_message (message_list, num_rows - 1);
-}
diff --git a/mail/message-list.h b/mail/message-list.h
index e66dc74a55..45a902e856 100644
--- a/mail/message-list.h
+++ b/mail/message-list.h
@@ -51,20 +51,24 @@ struct _MessageList {
 	
 	/* the folder browser that contains the 
 	 * this message list */
+	/* FIXME: This MUST BE REMOVED from this structure.  If we need access to the
+	   mail-display, we should store that instead ... */
 	FolderBrowser *parent_folder_browser;
 	
 	ETableModel  *table_model;
 	
-	ETreePath    *tree_root; /* for tree view */
+	ETreePath    *tree_root;
 	
 	GtkWidget    *etable;
 
 	CamelFolder  *folder;
+
+	GHashTable	*uid_rowmap; /* key is the uid, value is the row number.
+					Note: The key string is owned by table_model */
 	
-	GHashTable *uid_rowmap;
-	
-	char *search;		/* search string */
-	
+	char *search;		/* current search string */
+
+	gboolean threaded;	/* are we displaying threaded view? */
 	int cursor_row;
 	const char *cursor_uid;
 	
@@ -86,6 +90,7 @@ typedef enum {
 } MessageListSelectDirection;
 
 GtkType        message_list_get_type   (void);
+/* FIXME: We should be passing the MailDisplay to the list, or maybe raise signals instead */
 BonoboObject   *message_list_new        (FolderBrowser *parent_folder_browser);
 void           message_list_set_folder (MessageList *message_list,
 					CamelFolder *camel_folder);
@@ -100,10 +105,10 @@ void           message_list_select     (MessageList *message_list,
 					MessageListSelectDirection direction,
 					guint32 flags, guint32 mask);
 
-void           message_list_home       (MessageList *message_list);
-void           message_list_end        (MessageList *message_list);
+void	       message_list_set_threaded(MessageList *ml, gboolean threaded);
+void	       message_list_set_search(MessageList *ml, const char *search);
 
-extern gboolean threaded_view;
+/* FIXME: This should be an external callback that calls set_threaded() */
 void           message_list_toggle_threads (BonoboUIComponent           *component,
 					    const char                  *path,
 					    Bonobo_UIComponent_EventType type,
diff --git a/mail/message-thread.c b/mail/message-thread.c
index 4114ae0f33..0733a8b3ff 100644
--- a/mail/message-thread.c
+++ b/mail/message-thread.c
@@ -36,8 +36,12 @@
 
 #define d(x)
 
-static struct _container *thread_messages(CamelFolder *folder, GPtrArray *uids);
-static void thread_messages_free(struct _container *);
+#define TIMEIT
+
+#ifdef TIMEIT
+#include <sys/time.h>
+#include <unistd.h>
+#endif
 
 /* for debug only */
 int dump_tree(struct _container *c);
@@ -339,19 +343,27 @@ dump_tree(struct _container *c)
 	return count;
 }
 
-static void thread_messages_free(struct _container *c)
+static void
+container_free(struct _container *c)
 {
 	struct _container *n;
 
 	while (c) {
 		n = c->next;
 		if (c->child)
-			thread_messages_free(c->child); /* free's children first */
+			container_free(c->child); /* free's children first */
 		g_free(c);
 		c = n;
 	}
 }
 
+void
+thread_messages_free(struct _thread_messages *thread)
+{
+	container_free(thread->tree);
+	g_free(thread);
+}
+
 static int
 sort_node(const void *a, const void *b)
 {
@@ -411,21 +423,29 @@ sort_thread(struct _container **cp)
 	*cp = head;
 }
 
-static struct _container *
+/* NOTE: This function assumes you have obtained the relevant locks for
+   the folder, BEFORE calling it */
+struct _thread_messages *
 thread_messages(CamelFolder *folder, GPtrArray *uids)
 {
 	GHashTable *id_table, *no_id_table;
 	int i;
-	struct _container *c, *p, *child, *head, *container;
+	struct _container *c, *p, *child, *head;
 	struct _header_references *ref;
+	struct _thread_messages *thread;
+
+#ifdef TIMEIT
+	struct timeval start, end;
+	unsigned long diff;
+
+	gettimeofday(&start, NULL);
+#endif
 
 	id_table = g_hash_table_new(g_str_hash, g_str_equal);
 	no_id_table = g_hash_table_new(NULL, NULL);
 	for (i=0;i<uids->len;i++) {
 		const CamelMessageInfo *mi;
-		mail_tool_camel_lock_up ();
 		mi = camel_folder_get_message_info (folder, uids->pdata[i]);
-		mail_tool_camel_lock_down ();
 
 		if (mi == NULL) {
 			g_warning("Folder doesn't contain uid %s", (char *)uids->pdata[i]);
@@ -433,9 +453,16 @@ thread_messages(CamelFolder *folder, GPtrArray *uids)
 		}
 
 		if (mi->message_id) {
-			d(printf("doing : %s\n", mi->message_id));
 			c = g_hash_table_lookup(id_table, mi->message_id);
-			if (!c) {
+			/* check for duplicate messages */
+			if (c) {
+				/* if duplicate, just make out it is a no-id message,  but try and insert it
+				   into the right spot in the tree */
+				d(printf("doing: (duplicate message id)\n"));
+				c = g_malloc0(sizeof(*c));
+				g_hash_table_insert(no_id_table, (void *)mi, c);
+			} else {
+				d(printf("doing : %s\n", mi->message_id));
 				c = g_malloc0(sizeof(*c));
 				g_hash_table_insert(id_table, mi->message_id, c);
 			}
@@ -447,15 +474,16 @@ thread_messages(CamelFolder *folder, GPtrArray *uids)
 
 		c->message = mi;
 		c->order = i;
-		container = c;
+		child = c;
 		ref = mi->references;
 		p = NULL;
-		child = container;
 		head = NULL;
 		d(printf("references:\n"));
 		while (ref) {
 			if (ref->id == NULL) {
-				printf("ref missing id!?\n");
+				/* this shouldn't actually happen, and indicates
+				   some problems in camel */
+				d(printf("ref missing id!?\n"));
 				ref = ref->next;
 				continue;
 			}
@@ -498,7 +526,36 @@ thread_messages(CamelFolder *folder, GPtrArray *uids)
 #endif
 
 	sort_thread(&head);
-	return head;
+
+	thread = g_malloc(sizeof(*thread));
+	thread->tree = head;
+
+#ifdef TIMEIT
+	gettimeofday(&end, NULL);
+	diff = end.tv_sec * 1000 + end.tv_usec/1000;
+	diff -= start.tv_sec * 1000 + start.tv_usec/1000;
+	printf("Message threading %d messages took %d.%03d seconds\n",
+	       uids->len, diff / 1000, diff % 1000);
+#endif
+	return thread;
+}
+
+/* intended for incremental update.  Not implemented yet as, well, its probbaly
+   not worth it (memory overhead vs speed, may as well just rethread the whole
+   lot?)
+
+   But it might be implemented at a later date.
+*/
+void
+thread_messages_add(struct _thread_messages *thread, CamelFolder *folder, GPtrArray *uids)
+{
+	
+}
+
+void
+thread_messages_remove(struct _thread_messages *thread, CamelFolder *folder, GPtrArray *uids)
+{
+	
 }
 
 /* ** THREAD MESSAGES ***************************************************** */
@@ -511,7 +568,7 @@ typedef struct thread_messages_input_s {
 } thread_messages_input_t;
 
 typedef struct thread_messages_data_s {
-	struct _container *container;
+	struct _thread_messages *thread;
 } thread_messages_data_t;
 
 static gchar *describe_thread_messages (gpointer in_data, gboolean gerund);
@@ -539,7 +596,9 @@ static void do_thread_messages (gpointer in_data, gpointer op_data, CamelExcepti
 	thread_messages_input_t *input = (thread_messages_input_t *) in_data;
 	thread_messages_data_t *data = (thread_messages_data_t *) op_data;
 
-	data->container = thread_messages (input->ml->folder, input->uids);
+	mail_tool_camel_lock_up ();
+	data->thread = thread_messages (input->ml->folder, input->uids);
+	mail_tool_camel_lock_down ();
 }
 
 static void cleanup_thread_messages (gpointer in_data, gpointer op_data, CamelException *ex)
@@ -547,8 +606,8 @@ static void cleanup_thread_messages (gpointer in_data, gpointer op_data, CamelEx
 	thread_messages_input_t *input = (thread_messages_input_t *) in_data;
 	thread_messages_data_t *data = (thread_messages_data_t *) op_data;
 
-	(input->build) (input->ml, data->container);
-	thread_messages_free (data->container);
+	(input->build) (input->ml, data->thread->tree);
+	thread_messages_free (data->thread);
 
 	if (input->use_camel_uidfree) {
 		mail_tool_camel_lock_up ();
diff --git a/mail/message-thread.h b/mail/message-thread.h
index cf4665663f..d5153cbfc0 100644
--- a/mail/message-thread.h
+++ b/mail/message-thread.h
@@ -5,7 +5,6 @@
 #include "message-list.h"
 
 struct _container {
-	/* Next must be the first member */
 	struct _container *next,
 		*parent,
 		*child;
@@ -15,6 +14,15 @@ struct _container {
 	int order;
 };
 
+struct _thread_messages {
+	struct _container *tree;
+};
+
+struct _thread_messages *thread_messages(CamelFolder *folder, GPtrArray *uids);
+void thread_messages_add(struct _thread_messages *thread, CamelFolder *folder, GPtrArray *uids);
+void thread_messages_remove(struct _thread_messages *thread, CamelFolder *folder, GPtrArray *uids);
+void thread_messages_free(struct _thread_messages *c);
+
 void mail_do_thread_messages (MessageList *ml, GPtrArray *uids, 
 			      gboolean use_camel_uidfree,
 			      void (*build) (MessageList *,