Lock the command channel while searching. (imap_body_contains): If

2000-12-24  Not Zed  <NotZed@HelixCode.com>

        * providers/imap/camel-imap-search.c (imap_body_contains): Lock
        the command channel while searching.
        (imap_body_contains): If performing a whole uid search, then add
        references to our own summary items, dont look it up in the
        folder.  This way they can't vanish unexpectedly.

        * providers/imap/camel-imap-folder.h (CamelImapFolder): Added a
        private field.

        * providers/imap/camel-imap-private.h: Added lock for imap
        searches.

        * Merge from camel-mt-branch.

        * providers/imap/camel-imap-folder.c (imap_update_summary): Merge
        fix, use the folder->summary.
        (imap_get_message_flags, imap_set_message_flags,
        imap_get_message_user_flag, imap_set_message_user_flag): Removed
        again.
        (camel_imap_folder_init): Setup private data/lock.
        (imap_finalize): Free private data/search lock.
        (imap_search_free): Lock the search_lock.
        (imap_search_by_expression): Lock the search lock when using the
        search object.  Also copy/ref hte summary, rather than getting it
        directly.
        (imap_refresh_info): Free any info lookups.  Use folder->summary
        not imap_folder->summary.  And lock around commands.

svn path=/trunk/; revision=7150

5 files changed, 380 insertions, 57 deletions

diff --git a/camel/tests/folder/Makefile.am b/camel/tests/folder/Makefile.am
index 45f2e9895e..4401d310f6 100644
--- a/camel/tests/folder/Makefile.am
+++ b/camel/tests/folder/Makefile.am
@@ -15,11 +15,13 @@ LDADD = \
 check_PROGRAMS =  	\
 	test1	test4	test5	\
 	test2	test6	test7	\
-	test3
+	test3			\
+	test8
 
 TESTS = test1 test4 test5 \
 	test2 test6 test7 \
-	test3
+	test3		  \
+	test8
 
 
 
diff --git a/camel/tests/folder/README b/camel/tests/folder/README
index fab1358198..e02308d2d7 100644
--- a/camel/tests/folder/README
+++ b/camel/tests/folder/README
@@ -7,3 +7,5 @@ test5	camel store folder operations, NNTP
 test6	basic folder operations, IMAP
 test7	basic folder operations, NNTP
 
+test8	multithreaded folder torture test, local
+
diff --git a/camel/tests/folder/test8.c b/camel/tests/folder/test8.c
new file mode 100644
index 0000000000..30552cf782
--- /dev/null
+++ b/camel/tests/folder/test8.c
@@ -0,0 +1,222 @@
+/* threaded folder testing */
+
+#include "camel-test.h"
+#include "folders.h"
+#include "messages.h"
+
+#include <camel/camel-exception.h>
+#include <camel/camel-service.h>
+#include <camel/camel-session.h>
+#include <camel/camel-store.h>
+
+#define MAX_MESSAGES (100)
+#define MAX_THREADS (10)
+
+#define d(x) 
+
+#ifndef ENABLE_THREADS
+int main(int argc, char **argv)
+{
+	printf("Test %s is only compiled with threads enabled\n", argv[0]);
+	return 77;
+}
+#else
+
+#include <pthread.h>
+
+/* god, who designed this horrid interface */
+static char *auth_callback(CamelAuthCallbackMode mode,
+			   char *data, gboolean secret,
+			   CamelService *service, char *item,
+			   CamelException *ex)
+{
+	return NULL;
+}
+
+#define ARRAY_LEN(x) (sizeof(x)/sizeof(x[0]))
+
+static char *local_providers[] = {
+	"mbox",
+	"mh",
+	"maildir"
+};
+
+static void
+test_add_message(CamelFolder *folder, int j)
+{
+	CamelMimeMessage *msg;
+	char *content;
+	char *subject;
+	CamelException ex;
+
+	camel_exception_init(&ex);
+
+	push("creating message %d\n", j);
+	msg = test_message_create_simple();
+	content = g_strdup_printf("Test message %d contents\n\n", j);
+	test_message_set_content_simple((CamelMimePart *)msg, 0, "text/plain",
+							content, strlen(content));
+	test_free(content);
+	subject = g_strdup_printf("Test message %d subject", j);
+	camel_mime_message_set_subject(msg, subject);
+	pull();
+
+	push("appending simple message %d", j);
+	camel_folder_append_message(folder, msg, NULL, &ex);
+	check_msg(!camel_exception_is_set(&ex), "%s", camel_exception_get_description(&ex));
+	pull();
+
+	check_unref(msg, 1);
+}
+
+struct _threadinfo {
+	int id;
+	CamelFolder *folder;
+};
+
+static void *
+worker(void *d)
+{
+	struct _threadinfo *info = d;
+	int i, j, id = info->id;
+	char *sub, *content;
+	GPtrArray *res;
+	CamelException *ex = camel_exception_new();
+	CamelMimeMessage *msg;
+
+	/* we add a message, search for it, twiddle some flags, delete it */
+	/* and flat out */
+	for (i=0;i<MAX_MESSAGES;i++) {
+		d(printf("Thread %ld message %i\n", pthread_self(), i));
+		test_add_message(info->folder, id+i);
+
+		sub = g_strdup_printf("(match-all (header-contains \"subject\" \"message %d subject\"))", id+i);
+
+		push("searching for message %d\n\tusing: %s", id+i, sub);
+		res = camel_folder_search_by_expression(info->folder, sub, ex);
+		check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+		check_msg(res->len == 1, "res->len = %d", res->len);
+		pull();
+
+		push("getting message '%s'", res->pdata[0]);
+		msg = camel_folder_get_message(info->folder, (char *)res->pdata[0], ex);
+		check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+		pull();
+
+		push("comparing content");
+		content = g_strdup_printf("Test message %d contents\n\n", id+i);
+		test_message_compare_content(camel_medium_get_content_object((CamelMedium *)msg), content, strlen(content));
+		test_free(content);
+
+		push("deleting message, cleanup");
+		j=(100.0*rand()/(RAND_MAX+1.0));
+		if (j<=70) {
+			camel_folder_delete_message(info->folder, res->pdata[0]);
+		}
+
+		camel_folder_search_free(info->folder, res);
+		res = NULL;
+		test_free(sub);
+
+		check_unref(msg, 1);
+		pull();
+
+		/* about 1-in 100 calls will expunge */
+		j=(200.0*rand()/(RAND_MAX+1.0));
+		if (j<=2) {
+			d(printf("Forcing an expuge\n"));
+			push("expunging folder");
+			camel_folder_expunge(info->folder, ex);
+			check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+			pull();
+		}
+	}
+
+	camel_exception_free(ex);
+
+	return info;
+}
+
+int main(int argc, char **argv)
+{
+	CamelSession *session;
+	CamelException *ex;
+	int i, j, index;
+	char *path;
+	CamelStore *store;
+	pthread_t threads[MAX_THREADS];
+	struct _threadinfo *info;
+	CamelFolder *folder;
+	GPtrArray *uids;
+
+	camel_test_init(argc, argv);
+
+	ex = camel_exception_new();
+
+	/* clear out any camel-test data */
+	system("/bin/rm -rf /tmp/camel-test");
+
+	session = camel_session_new("/tmp/camel-test", auth_callback, NULL, NULL);
+
+	for (j=0;j<ARRAY_LEN(local_providers);j++) {
+		for (index=0;index<2;index++) {
+			path = g_strdup_printf("method %s %s", local_providers[j], index?"indexed":"nonindexed");
+			camel_test_start(path);
+			test_free(path);
+
+			push("trying %s index %d", local_providers[j], index);
+			path = g_strdup_printf("%s:///tmp/camel-test/%s", local_providers[j], local_providers[j]);
+			store = camel_session_get_store(session, path, ex);
+			check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+			test_free(path);
+			
+			if (index == 0)
+				folder = camel_store_get_folder(store, "testbox", CAMEL_STORE_FOLDER_CREATE, ex);
+			else
+				folder = camel_store_get_folder(store, "testbox",
+								CAMEL_STORE_FOLDER_CREATE|CAMEL_STORE_FOLDER_BODY_INDEX, ex);
+			check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+
+			for (i=0;i<MAX_THREADS;i++) {
+				info = g_malloc(sizeof(*info));
+				info->id = i*MAX_MESSAGES;
+				info->folder = folder;
+				pthread_create(&threads[i], 0, worker, info);
+			}
+
+			for (i=0;i<MAX_THREADS;i++) {
+				pthread_join(threads[i], (void **)&info);
+				g_free(info);
+			}
+			pull();
+
+			push("deleting remaining messages");
+			uids = camel_folder_get_uids(folder);
+			for (i=0;i<uids->len;i++) {
+				camel_folder_delete_message(folder, uids->pdata[i]);
+			}
+			camel_folder_free_uids(folder, uids);
+
+			camel_folder_expunge(folder, ex);
+			check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+
+			check_unref(folder, 1);
+
+			camel_store_delete_folder(store, "testbox", ex);
+			check_msg(!camel_exception_is_set(ex), "%s", camel_exception_get_description(ex));
+
+			check_unref(store, 1);
+
+			pull();
+
+			camel_test_end();
+		}
+	}
+
+	camel_object_unref((CamelObject *)session);
+	camel_exception_free(ex);
+
+	return 0;
+}
+
+#endif /* ENABLE_THREADS */
diff --git a/camel/tests/lib/camel-test.c b/camel/tests/lib/camel-test.c
index 6a1e00addb..09014e4b07 100644
--- a/camel/tests/lib/camel-test.c
+++ b/camel/tests/lib/camel-test.c
@@ -4,39 +4,97 @@
 #include <stdio.h>
 #include <signal.h>
 
+#ifdef ENABLE_THREADS
+#include <pthread.h>
+#endif
+
+#ifdef ENABLE_THREADS
+/* well i dunno, doesn't seem to be in the headers but hte manpage mentions it */
+/* a nonportable checking mutex for glibc, not really needed, just validates
+   the test harness really */
+# ifdef PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
+static pthread_mutex_t lock = PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP;
+# else
+static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
+# endif
+#define CAMEL_TEST_LOCK pthread_mutex_lock(&lock)
+#define CAMEL_TEST_UNLOCK pthread_mutex_unlock(&lock)
+#define CAMEL_TEST_ID (pthread_self())
+#else
+#define CAMEL_TEST_LOCK
+#define CAMEL_TEST_UNLOCK
+#define CAMEL_TEST_ID (0)
+#endif
+
+static int setup;
+static int ok;
+
 struct _stack {
 	struct _stack *next;
+	int fatal;
 	char *what;
 };
 
-static int setup;
-static struct _stack *state;
-static struct _stack *nonfatal;
-static int ok;
+/* per-thread state */
+struct _state {
+	char *test;
+	int nonfatal;
+	struct _stack *state;
+};
+
+static GHashTable *info_table;
 
 int camel_test_verbose;
 
+static void
+dump_action(int id, struct _state *s, void *d)
+{
+	struct _stack *node;
+
+#ifdef ENABLE_THREADS
+	printf("\nThread %d:\n", id);
+#endif
+	node = s->state;
+	if (node) {
+		printf("Current action:\n");
+		while (node) {
+			printf("\t%s%s\n", node->fatal?"":"[nonfatal]", node->what);
+			node = node->next;
+		}
+	}
+	printf("\tTest: %s\n", s->test);
+}
+
 static void die(int sig)
 {
 	static int indie = 0;
-	struct _stack *node;
 
 	if (!indie) {
 		indie = 1;
 		printf("\n\nReceived fatal signal %d\n", sig);
-		node = state;
-		if (node) {
-			printf("Current action:\n");
-			while (node) {
-				printf("\t%s\n", node->what);
-				node = node->next;
-			}
-		}
+		g_hash_table_foreach(info_table, (GHFunc)dump_action, 0);
 	}
 
 	_exit(1);
 }
 
+static struct _state *
+current_state(void)
+{
+	struct _state *info;
+
+	if (info_table == NULL)
+		info_table = g_hash_table_new(0, 0);
+
+	info = g_hash_table_lookup(info_table, (void *)CAMEL_TEST_ID);
+	if (info == NULL) {
+		info = g_malloc0(sizeof(*info));
+		g_hash_table_insert(info_table, (void *)CAMEL_TEST_ID, info);
+	}
+	return info;
+}
+	
+
 void camel_test_init(int argc, char **argv)
 {
 	void camel_init(void);
@@ -44,7 +102,11 @@ void camel_test_init(int argc, char **argv)
 
 	setup = 1;
 
+#ifndef ENABLE_THREADS
 	camel_init();
+#endif
+
+	info_table = g_hash_table_new(0, 0);
 
 	/* yeah, we do need ot thread init, even though camel isn't compiled with enable threads */
 	g_thread_init(NULL);
@@ -71,15 +133,25 @@ void camel_test_init(int argc, char **argv)
 
 void camel_test_start(const char *what)
 {
+	struct _state *s;
+
+	CAMEL_TEST_LOCK;
+
+	s = current_state();
+
 	if (!setup)
 		camel_test_init(0, 0);
 
 	ok = 1;
 
+	s->test = g_strdup(what);
+
 	if (camel_test_verbose > 0) {
 		printf("Test: %s ... ", what);
 		fflush(stdout);
 	}
+
+	CAMEL_TEST_UNLOCK;
 }
 
 void camel_test_push(const char *what, ...)
@@ -87,6 +159,11 @@ void camel_test_push(const char *what, ...)
 	struct _stack *node;
 	va_list ap;
 	char *text;
+	struct _state *s;
+
+	CAMEL_TEST_LOCK;
+
+	s = current_state();
 
 	va_start(ap, what);
 	text = g_strdup_vprintf(what, ap);
@@ -97,26 +174,40 @@ void camel_test_push(const char *what, ...)
 
 	node = g_malloc(sizeof(*node));
 	node->what = text;
-	node->next = state;
-	state = node;
+	node->next = s->state;
+	node->fatal = 1;
+	s->state = node;
+
+	CAMEL_TEST_UNLOCK;
 }
 
 void camel_test_pull(void)
 {
 	struct _stack *node;
+	struct _state *s;
 
-	g_assert(state);
+	CAMEL_TEST_LOCK;
+
+	s = current_state();
+
+	g_assert(s->state);
 
 	if (camel_test_verbose > 3)
-		printf("Finish step: %s\n", state->what);
+		printf("Finish step: %s\n", s->state->what);
 
-	node = state;
-	state = node->next;
+	node = s->state;
+	s->state = node->next;
+	if (!node->fatal)
+		s->nonfatal--;
 	g_free(node->what);
 	g_free(node);
+
+	CAMEL_TEST_UNLOCK;
 }
 
 /* where to set breakpoints */
+void camel_test_break(void);
+
 void camel_test_break(void)
 {
 }
@@ -130,75 +221,75 @@ void camel_test_fail(const char *why, ...)
 	va_end(ap);
 }
 
+
 void camel_test_failv(const char *why, va_list ap)
 {
-	struct _stack *node;
 	char *text;
+	struct _state *s;
+
+	CAMEL_TEST_LOCK;
+
+	s = current_state();
 
 	text = g_strdup_vprintf(why, ap);
 
-	if ((nonfatal == NULL && camel_test_verbose > 0)
-	    || (nonfatal && camel_test_verbose > 1)) {
+	if ((s->nonfatal == 0 && camel_test_verbose > 0)
+	    || (s->nonfatal && camel_test_verbose > 1)) {
 		printf("Failed.\n%s\n", text);
 		camel_test_break();
 	}
 
 	g_free(text);
 
-	if ((nonfatal == NULL && camel_test_verbose > 0)
-	    || (nonfatal && camel_test_verbose > 2)) {
-		node = state;
-		if (node) {
-			printf("Current action:\n");
-			while (node) {
-				printf("\t%s\n", node->what);
-				node = node->next;
-			}
-		}
+	if ((s->nonfatal == 0 && camel_test_verbose > 0)
+	    || (s->nonfatal && camel_test_verbose > 2)) {
+		g_hash_table_foreach(info_table, (GHFunc)dump_action, 0);
 	}
 
-	if (nonfatal == NULL) {
+	if (s->nonfatal == 0) {
 		exit(1);
 	} else {
 		ok=0;
 		if (camel_test_verbose > 1) {
-			printf("Known problem (ignored): %s\n", nonfatal->what);
+			printf("Known problem (ignored):\n");
+			dump_action(CAMEL_TEST_ID, s, 0);
 		}
 	}
+
+	CAMEL_TEST_UNLOCK;
 }
 
-void camel_test_nonfatal(const char *why, ...)
+void camel_test_nonfatal(const char *what, ...)
 {
 	struct _stack *node;
 	va_list ap;
 	char *text;
+	struct _state *s;
 
-	va_start(ap, why);
-	text = g_strdup_vprintf(why, ap);
+	CAMEL_TEST_LOCK;
+
+	s = current_state();
+
+	va_start(ap, what);
+	text = g_strdup_vprintf(what, ap);
 	va_end(ap);
 
-	if (camel_test_verbose>3)
+	if (camel_test_verbose > 3)
 		printf("Start nonfatal: %s\n", text);
 
 	node = g_malloc(sizeof(*node));
 	node->what = text;
-	node->next = nonfatal;
-	nonfatal = node;
+	node->next = s->state;
+	node->fatal = 0;
+	s->nonfatal++;
+	s->state = node;
+
+	CAMEL_TEST_UNLOCK;
 }
 
 void camel_test_fatal(void)
 {
-	struct _stack *node;
-
-	g_assert(nonfatal);
-
-	if (camel_test_verbose>3)
-		printf("Finish nonfatal: %s\n", nonfatal->what);
-
-	node = nonfatal;
-	nonfatal = node->next;
-	g_free(node->what);
-	g_free(node);
+	camel_test_pull();
 }
 
 void camel_test_end(void)
diff --git a/camel/tests/lib/folders.c b/camel/tests/lib/folders.c
index 6f24cd483a..12136b0cc7 100644
--- a/camel/tests/lib/folders.c
+++ b/camel/tests/lib/folders.c
@@ -11,7 +11,7 @@ test_folder_counts(CamelFolder *folder, int total, int unread)
 {
 	GPtrArray *s;
 	int i, myunread;
-	const CamelMessageInfo *info;
+	CamelMessageInfo *info;
 
 	push("test folder counts %d total %d unread", total, unread);
 
@@ -41,6 +41,7 @@ test_folder_counts(CamelFolder *folder, int total, int unread)
 		info = camel_folder_get_message_info(folder, s->pdata[i]);
 		if (info->flags & CAMEL_MESSAGE_SEEN)
 			myunread--;
+		camel_folder_free_message_info(folder, info);
 	}
 	check(unread == myunread);
 	camel_folder_free_uids(folder, s);
@@ -81,7 +82,7 @@ void
 test_folder_message(CamelFolder *folder, const char *uid)
 {
 	CamelMimeMessage *msg;
-	const CamelMessageInfo *info;
+	CamelMessageInfo *info;
 	GPtrArray *s;
 	int i;
 	CamelException *ex = camel_exception_new();
@@ -93,6 +94,7 @@ test_folder_message(CamelFolder *folder, const char *uid)
 	info = camel_folder_get_message_info(folder, uid);
 	check(info != NULL);
 	check(strcmp(camel_message_info_uid(info), uid) == 0);
+	camel_folder_free_message_info(folder, info);
 
 	/* then, getting message */
 	msg = camel_folder_get_message(folder, uid, ex);
@@ -137,7 +139,7 @@ void
 test_folder_not_message(CamelFolder *folder, const char *uid)
 {
 	CamelMimeMessage *msg;
-	const CamelMessageInfo *info;
+	CamelMessageInfo *info;
 	GPtrArray *s;
 	int i;
 	CamelException *ex = camel_exception_new();
@@ -314,7 +316,7 @@ test_folder_message_ops(CamelSession *session, const char *name, int local)
 	int j;
 	int indexed, max;
 	GPtrArray *uids;
-	const CamelMessageInfo *info;
+	CamelMessageInfo *info;
 
 	max=local?2:1;
 
@@ -390,6 +392,7 @@ test_folder_message_ops(CamelSession *session, const char *name, int local)
 			check_msg(strcmp(camel_message_info_subject(info), subject)==0,
 				  "info->subject %s", camel_message_info_subject(info));
 			camel_folder_free_uids(folder, uids);
+			camel_folder_free_message_info(folder, info);
 			pull();
 
 			test_free(subject);
@@ -429,6 +432,7 @@ test_folder_message_ops(CamelSession *session, const char *name, int local)
 			check_msg(strcmp(camel_message_info_subject(info), subject)==0,
 				  "info->subject %s", camel_message_info_subject(info));
 			test_free(subject);
+			camel_folder_free_message_info(folder, info);
 			pull();
 		}
 
@@ -455,6 +459,7 @@ test_folder_message_ops(CamelSession *session, const char *name, int local)
 			check_msg(strcmp(camel_message_info_subject(info), subject)==0,
 				  "info->subject %s", camel_message_info_subject(info));
 			test_free(subject);
+			camel_folder_free_message_info(folder, info);
 			pull();
 		}
 		pull();
@@ -483,6 +488,7 @@ test_folder_message_ops(CamelSession *session, const char *name, int local)
 			check_msg(strcmp(camel_message_info_subject(info), subject)==0,
 				  "info->subject %s", camel_message_info_subject(info));
 			test_free(subject);
+			camel_folder_free_message_info(folder, info);
 			pull();
 		}
 		pull();