When we add a new name, up all of the cache limits, because we're probably

2002-03-25  Not Zed  <NotZed@Ximian.com>

        * camel-text-index.c (text_index_add_name): When we add a new
        name, up all of the cache limits, because we're probably going to
        be adding more.
        (text_index_sync): Drop the cache limits back down again, we dont
        need them when looking words up.

        ** MERGE camel_index branch.

        * camel-text-index.[ch]: Added files i forgot to add (eep nearly
        lost all this work!)

        * camel-block-file.c (sync_nolock): Fix an infinite loop in syncing.

svn path=/trunk/; revision=16242

1 files changed, 70 insertions, 114 deletions

diff --git a/camel/camel-folder-summary.c b/camel/camel-folder-summary.c
index a424aef8de..c681ff6123 100644
--- a/camel/camel-folder-summary.c
+++ b/camel/camel-folder-summary.c
@@ -22,6 +22,10 @@
 #include <config.h>
 #endif
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
 #include <unistd.h>
 #include <ctype.h>
 #include <string.h>
@@ -42,6 +46,9 @@
 #include <camel/camel-multipart.h>
 #include <camel/camel-stream-mem.h>
 
+#include <camel/camel-stream-null.h>
+#include <camel/camel-stream-filter.h>
+
 #include "hash-table-utils.h"
 #include "e-util/md5-utils.h"
 #include "e-util/e-memory.h"
@@ -213,6 +220,11 @@ camel_folder_summary_finalize (CamelObject *obj)
 	if (p->filter_html)
 		camel_object_unref((CamelObject *)p->filter_html);
 
+	if (p->filter_stream)
+		camel_object_unref((CamelObject *)p->filter_stream);
+	if (p->index)
+		camel_object_unref((CamelObject *)p->index);
+
 #ifdef ENABLE_THREADS
 	g_mutex_free(p->summary_lock);
 	g_mutex_free(p->io_lock);
@@ -283,11 +295,16 @@ void camel_folder_summary_set_filename(CamelFolderSummary *s, const char *name)
  *
  * Unlike earlier behaviour, build_content need not be set to perform indexing.
  **/
-void camel_folder_summary_set_index(CamelFolderSummary *s, ibex *index)
+void camel_folder_summary_set_index(CamelFolderSummary *s, CamelIndex *index)
 {
 	struct _CamelFolderSummaryPrivate *p = _PRIVATE(s);
 
+	if (p->index)
+		camel_object_unref((CamelObject *)p->index);
+
 	p->index = index;
+	if (index)
+		camel_object_ref((CamelObject *)index);
 }
 
 /**
@@ -853,6 +870,7 @@ CamelMessageInfo *camel_folder_summary_info_new_from_parser(CamelFolderSummary *
 	int len;
 	struct _CamelFolderSummaryPrivate *p = _PRIVATE(s);
 	off_t start;
+	CamelIndexName *name = NULL;
 
 	/* should this check the parser is in the right state, or assume it is?? */
 
@@ -871,15 +889,21 @@ CamelMessageInfo *camel_folder_summary_info_new_from_parser(CamelFolderSummary *
 
 		if (p->index) {
 			if (p->filter_index == NULL)
-				p->filter_index = camel_mime_filter_index_new_ibex(p->index);
-			camel_mime_filter_index_set_name(p->filter_index, (char *)camel_message_info_uid(info));
-			ibex_unindex(p->index, (char *)camel_message_info_uid(info));
-			ibex_index_buffer(p->index, (char *)camel_message_info_uid(info), "ibexindexed", strlen("ibexindexed"), NULL);
+				p->filter_index = camel_mime_filter_index_new_index(p->index);
+			camel_index_delete_name(p->index, camel_message_info_uid(info));
+			name = camel_index_add_name(p->index, camel_message_info_uid(info));
+			camel_mime_filter_index_set_name(p->filter_index, name);
 		}
 
 		/* always scan the content info, even if we dont save it */
 		info->content = summary_build_content_info(s, info, mp);
 
+		if (name) {
+			camel_index_write_name(p->index, name);
+			camel_object_unref((CamelObject *)name);
+			camel_mime_filter_index_set_name(p->filter_index, NULL);
+		}
+
 		CAMEL_SUMMARY_UNLOCK(s, filter_lock);
 
 		info->size = camel_mime_parser_tell(mp) - start;
@@ -900,6 +924,7 @@ CamelMessageInfo *camel_folder_summary_info_new_from_message(CamelFolderSummary
 {
 	CamelMessageInfo *info;
 	struct _CamelFolderSummaryPrivate *p = _PRIVATE(s);
+	CamelIndexName *name = NULL;
 
 	info = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_new_from_message(s, msg);
 
@@ -907,13 +932,33 @@ CamelMessageInfo *camel_folder_summary_info_new_from_message(CamelFolderSummary
 	 * know if we are going to store this in the summary, but no matter */
 	summary_assign_uid(s, info);
 
+	CAMEL_SUMMARY_LOCK(s, filter_lock);
+
 	if (p->index) {
-		ibex_unindex(p->index, (char *)camel_message_info_uid(info));
-		ibex_index_buffer(p->index, (char *)camel_message_info_uid(info), "ibexindexed", strlen("ibexindexed"), NULL);
+		if (p->filter_index == NULL)
+			p->filter_index = camel_mime_filter_index_new_index(p->index);
+		camel_index_delete_name(p->index, camel_message_info_uid(info));
+		name = camel_index_add_name(p->index, camel_message_info_uid(info));
+		camel_mime_filter_index_set_name(p->filter_index, name);
+
+		if (p->filter_stream == NULL) {
+			CamelStream *null = camel_stream_null_new();
+
+			p->filter_stream = camel_stream_filter_new_with_stream(null);
+			camel_object_unref((CamelObject *)null);
+		}
 	}
 
 	info->content = summary_build_content_info_message(s, info, (CamelMimePart *)msg);
 
+	if (name) {
+		camel_index_write_name(p->index, name);
+		camel_object_unref((CamelObject *)name);
+		camel_mime_filter_index_set_name(p->filter_index, NULL);
+	}
+
+	CAMEL_SUMMARY_UNLOCK(s, filter_lock);
+
 	return info;
 }
 
@@ -2000,7 +2045,7 @@ summary_build_content_info(CamelFolderSummary *s, CamelMessageInfo *msginfo, Cam
 }
 
 /* build the content-info, from a message */
-/* this needs no lock, as we copy all data, and ibex is threadsafe */
+/* this needs the filter lock since it uses filters to perform indexing */
 static CamelMessageContentInfo *
 summary_build_content_info_message(CamelFolderSummary *s, CamelMessageInfo *msginfo, CamelMimePart *object)
 {
@@ -2046,14 +2091,25 @@ summary_build_content_info_message(CamelFolderSummary *s, CamelMessageInfo *msgi
 			child->parent = info;
 			my_list_append((struct _node **)&info->childs, (struct _node *)child);
 		}
-	} else if (p->index
+	} else if (p->filter_stream
 		   && header_content_type_is(CAMEL_DATA_WRAPPER(containee)->mime_type, "text", "*")) {
-		/* index all text parts if we're indexing */
-		CamelStreamMem *mem = (CamelStreamMem *)camel_stream_mem_new();
+		int html_id = -1, idx_id = -1;
+
+		/* pre-attach html filter if required, otherwise just index filter */
+		if (header_content_type_is(CAMEL_DATA_WRAPPER(containee)->mime_type, "text", "html")) {
+			if (p->filter_html == NULL)
+				p->filter_html = camel_mime_filter_html_new();
+			else
+				camel_mime_filter_reset((CamelMimeFilter *)p->filter_html);
+			html_id = camel_stream_filter_add(p->filter_stream, (CamelMimeFilter *)p->filter_html);
+		}
+		idx_id = camel_stream_filter_add(p->filter_stream, (CamelMimeFilter *)p->filter_index);
 
-		camel_data_wrapper_write_to_stream(containee, (CamelStream *)mem);
-		ibex_index_buffer(p->index, (char *)camel_message_info_uid(msginfo), mem->buffer->data, mem->buffer->len, NULL);
-		camel_object_unref((CamelObject *)mem);
+		camel_data_wrapper_write_to_stream(containee, (CamelStream *)p->filter_stream);
+		camel_stream_flush((CamelStream *)p->filter_stream);
+
+		camel_stream_filter_remove(p->filter_stream, idx_id);
+		camel_stream_filter_remove(p->filter_stream, html_id);
 	}
 
 	return info;
@@ -2680,104 +2736,4 @@ message_info_dump(CamelMessageInfo *mi)
 	printf("Flags: %04x\n", mi->flags & 0xffff);
 	content_info_dump(mi->content, 0);
 }
-
-int main(int argc, char **argv)
-{
-	CamelMimeParser *mp;
-	int fd;
-	CamelFolderSummary *s;
-	char *buffer;
-	int len;
-	int i;
-	ibex *index;
-
-	/*g_tk_init(&argc, &argv);*/
-
-#if 0
-	{
-		int i;
-		char *s;
-		char buf[1024];
-
-		for (i=0;i<434712;i++) {
-			memcpy(buf, "                                                         ", 50);
-			buf[50] = 0;
-#if 0
-			s = g_strdup(buf);
-			g_free(s);
-#endif
-		}
-		return 0;
-	}
-#endif
-
-	if (argc < 2 ) {
-		printf("usage: %s mbox\n", argv[0]);
-		return 1;
-	}
-
-	fd = open(argv[1], O_RDONLY);
-
-	index = ibex_open("index.ibex", O_CREAT|O_RDWR, 0600);
-
-	mp = camel_mime_parser_new();
-	camel_mime_parser_scan_from(mp, TRUE);
-/*	camel_mime_parser_set_header_regex(mp, "^(content-[^:]*|subject|from|to|date):");*/
-	camel_mime_parser_init_with_fd(mp, fd);
-
-	s = camel_folder_summary_new();
-	camel_folder_summary_set_build_content(s, TRUE);
-/*	camel_folder_summary_set_index(s, index);*/
-
-	while (camel_mime_parser_step(mp, &buffer, &len) == HSCAN_FROM) {
-		/*printf("Parsing message ...\n");*/
-		camel_folder_summary_add_from_parser(s, mp);
-		if (camel_mime_parser_step(mp, &buffer, &len) != HSCAN_FROM_END) {
-			g_warning("Uknown state encountered, excpecting %d, got %d\n", HSCAN_FROM_END, camel_mime_parser_state(mp));
-			break;
-		}
-	}
-
-	printf("Printing summary\n");
-	for (i=0;i<camel_folder_summary_count(s);i++) {
-		CamelMessageInfo *info = camel_folder_summary_index(s, i);
-		message_info_dump(info);
-		camel_folder_summary_info_free(info);
-	}
-
-	printf("Saivng summary\n");
-	camel_folder_summary_set_filename(s, "index.summary");
-	camel_folder_summary_save(s);
-
-	{
-		CamelFolderSummary *n;
-
-		printf("\nLoading summary\n");
-		n = camel_folder_summary_new();
-		camel_folder_summary_set_build_content(n, TRUE);
-		camel_folder_summary_set_filename(n, "index.summary");
-		camel_folder_summary_load(n);
-
-		printf("Printing summary\n");
-		for (i=0;i<camel_folder_summary_count(n);i++) {
-			CamelMessageInfo *info = camel_folder_summary_index(s, i);
-			message_info_dump(info);
-			camel_folder_summary_info_free(info);
-		}
-		camel_object_unref(n);		
-	}
-
-
-	camel_object_unref(mp);
-	camel_object_unref(s);
-
-	printf("summarised %d messages\n", camel_folder_summary_count(s));
-#if 0
-	printf("g_strdup count = %d\n", strdup_count);
-	printf("g_malloc count = %d\n", malloc_count);
-	printf("g_free count = %d\n", free_count);
-#endif
-	return 0;
-}
-
 #endif