Turn indexing back on, fingers crossed ...

2002-04-03  Not Zed  <NotZed@Ximian.com>

        * providers/local/camel-local-folder.c
        (camel_local_folder_construct): Turn indexing back on, fingers
        crossed ...

        * camel-block-file.c (sync_nolock): #!@$@$#@~#$
        DF@#$!Q@$#!@$#!#%.  Well it helps if we're iterating a list to
        iterate the node pointer ...

        * camel-text-index.c (text_index_sync): Sync the key tables
        explcitly.
        (text_index_sync): Debug out frag info.
        (camel_text_index_dump): Added a (rather large, but optional) raw
        dumping mode for debugging purposes.

        * camel-partition-table.c (camel_key_table_finalise): Sync root
        block when done.

svn path=/trunk/; revision=16329

7 files changed, 246 insertions, 11 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index 96b013de75..bb5ce59581 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -18,12 +18,25 @@
 
 2002-04-03  Not Zed  <NotZed@Ximian.com>
 
+	* providers/local/camel-local-folder.c
+	(camel_local_folder_construct): Turn indexing back on, fingers
+	crossed ...
+
+	* camel-block-file.c (sync_nolock): #!@$@$#@~#$
+	DF@#$!Q@$#!@$#!#%.  Well it helps if we're iterating a list to
+	iterate the node pointer ...
+
 	* camel-text-index.c (text_index_sync): Sync the key tables
 	explcitly.
+	(text_index_sync): Debug out frag info.
+	(camel_text_index_dump): Added a (rather large, but optional) raw
+	dumping mode for debugging purposes.
 
 	* camel-partition-table.c (camel_key_table_finalise): Sync root
 	block when done.
 	(camel_key_table_sync): New function, sync key table (root) explicitly.
+	(camel_partition_table_sync): Method to explicitly sync the
+	partition table.
 
 2002-04-02  Not Zed  <NotZed@Ximian.com>
 
diff --git a/camel/camel-block-file.c b/camel/camel-block-file.c
index e73b266026..6b889f9da1 100644
--- a/camel/camel-block-file.c
+++ b/camel/camel-block-file.c
@@ -98,7 +98,7 @@ block_file_validate_root(CamelBlockFile *bs)
 
 	fstat(bs->fd, &st);
 
-	(printf("Validate root:\n"));
+	(printf("Validate root: '%s'\n", bs->path));
 	(printf("version: %.8s (%.8s)\n", bs->root->version, bs->version));
 	(printf("block size: %d (%d)%s\n", br->block_size, bs->block_size,
 		br->block_size != bs->block_size ? " BAD":" OK"));
@@ -693,8 +693,8 @@ sync_nolock(CamelBlockFile *bs)
 			work = TRUE;
 			if (sync_block_nolock(bs, bl) == -1)
 				return -1;
-			bl = bn;
 		}
+		bl = bn;
 		bn = bn->next;
 	}
 
diff --git a/camel/camel-index-control.c b/camel/camel-index-control.c
index 9968278446..f1f4dd234d 100644
--- a/camel/camel-index-control.c
+++ b/camel/camel-index-control.c
@@ -209,4 +209,3 @@ do_perf(int argc, char **argv)
 
 	return 0;
 }
-
diff --git a/camel/camel-partition-table.c b/camel/camel-partition-table.c
index eb44d9c197..68e0e257b5 100644
--- a/camel/camel-partition-table.c
+++ b/camel/camel-partition-table.c
@@ -90,11 +90,11 @@ camel_partition_table_finalise(CamelPartitionTable *cpi)
 	p = cpi->priv;
 
 	if (cpi->blocks) {
-		camel_block_file_sync(cpi->blocks);
 		while ((bl = (CamelBlock *)e_dlist_remhead(&cpi->partition))) {
 			camel_block_file_sync_block(cpi->blocks, bl);
 			camel_block_file_unref_block(cpi->blocks, bl);
 		}
+		camel_block_file_sync(cpi->blocks);
 
 		camel_object_unref((CamelObject *)cpi->blocks);
 	}
@@ -256,6 +256,35 @@ fail:
 	return NULL;
 }
 
+/* sync our blocks, the caller must still sync the blockfile itself */
+int
+camel_partition_table_sync(CamelPartitionTable *cpi)
+{
+	CamelBlock *bl, *bn;
+	struct _CamelPartitionTablePrivate *p;
+	int ret = 0;
+
+	CAMEL_PARTITION_TABLE_LOCK(cpi, lock);
+
+	p = cpi->priv;
+
+	if (cpi->blocks) {
+		bl = (CamelBlock *)cpi->partition.head;
+		bn = bl->next;
+		while (bn) {
+			ret = camel_block_file_sync_block(cpi->blocks, bl);
+			if (ret == -1)
+				goto fail;
+			bl = bn;
+			bn = bn->next;
+		}
+	}
+fail:
+	CAMEL_PARTITION_TABLE_UNLOCK(cpi, lock);
+
+	return ret;
+}
+
 camel_key_t camel_partition_table_lookup(CamelPartitionTable *cpi, const char *key)
 {
 	CamelPartitionKeyBlock *pkb;
@@ -721,8 +750,8 @@ camel_key_table_add(CamelKeyTable *ki, const char *key, camel_block_t data, unsi
 	if (kblast->used > 0) {
 		/*left = &kblast->u.keydata[kblast->u.keys[kblast->used-1].offset] - (char *)(&kblast->u.keys[kblast->used+1]);*/
 		left = kblast->u.keys[kblast->used-1].offset - sizeof(kblast->u.keys[0])*(kblast->used+1);
-		d(printf("used = %d (%d), filled = %d, left = %d  len = %d?\n",
-			 kblast->used, kblast->used * sizeof(kblast->u.keys[0]),
+		d(printf("key '%s' used = %d (%d), filled = %d, left = %d  len = %d?\n",
+			 key, kblast->used, kblast->used * sizeof(kblast->u.keys[0]),
 			 sizeof(kblast->u.keydata) - kblast->u.keys[kblast->used-1].offset,
 			 left, len));
 		if (left < len) {
@@ -734,7 +763,7 @@ camel_key_table_add(CamelKeyTable *ki, const char *key, camel_block_t data, unsi
 			kbnext = (CamelKeyBlock *)&next->data;
 			kblast->next = next->id;
 			ki->root->last = next->id;
-			k(printf("adding new block, first = %u, last = %u\n", ki->root->first, ki->root->last));
+			d(printf("adding new block, first = %u, last = %u\n", ki->root->first, ki->root->last));
 			camel_block_file_touch_block(ki->blocks, ki->root_block);
 			camel_block_file_touch_block(ki->blocks, last);
 			camel_block_file_unref_block(ki->blocks, last);
diff --git a/camel/camel-partition-table.h b/camel/camel-partition-table.h
index 04e2b6cd88..0b87390835 100644
--- a/camel/camel-partition-table.h
+++ b/camel/camel-partition-table.h
@@ -84,6 +84,7 @@ struct _CamelPartitionTableClass {
 CamelType camel_partition_table_get_type(void);
 
 CamelPartitionTable *camel_partition_table_new(struct _CamelBlockFile *bs, camel_block_t root);
+int camel_partition_table_sync(CamelPartitionTable *cpi);
 int camel_partition_table_add(CamelPartitionTable *cpi, const char *key, camel_key_t keyid);
 camel_key_t camel_partition_table_lookup(CamelPartitionTable *cpi, const char *key);
 void camel_partition_table_remove(CamelPartitionTable *cpi, const char *key);
diff --git a/camel/camel-text-index.c b/camel/camel-text-index.c
index dadc4de083..9638ae6269 100644
--- a/camel/camel-text-index.c
+++ b/camel/camel-text-index.c
@@ -302,15 +302,17 @@ text_index_sync(CamelIndex *idx)
 	}	
 
 	if (camel_key_table_sync(p->word_index) == -1
-	    || camel_key_table_sync(p->name_index) == -1)
+	    || camel_key_table_sync(p->name_index) == -1
+	    || camel_partition_table_sync(p->word_hash) == -1
+	    || camel_partition_table_sync(p->name_hash) == -1)
 		ret = -1;
 
 	/* only do the frag/compress check if we did some new writes on this index */
 	if (ret == 0 && work) {
 		wfrag = rb->words ? (((rb->keys - rb->words) * 100)/ rb->words) : 0;
 		nfrag = rb->names ? ((rb->deleted * 100) / rb->names) : 0;
-		printf("wfrag = %d, nfrag = %d\n", wfrag, nfrag);
-		printf("  words = %d, keys = %d\n", rb->words, rb->keys);
+		d(printf("wfrag = %d, nfrag = %d\n", wfrag, nfrag));
+		d(printf("  words = %d, keys = %d\n", rb->words, rb->keys));
 		if (wfrag > 30 || nfrag > 20)
 			ret = text_index_compress_nosync(idx);
 	}
@@ -973,11 +975,186 @@ camel_text_index_info(CamelTextIndex *idx)
 	}
 }
 
+/* #define DUMP_RAW */
+
+#ifdef DUMP_RAW
+enum { KEY_ROOT = 1, KEY_DATA = 2, PARTITION_MAP = 4, PARTITION_DATA = 8 };
+
+static void
+add_type(GHashTable *map, camel_block_t id, int type)
+{
+	camel_block_t old;
+
+	old = g_hash_table_lookup(map, id);
+	if (old == type)
+		return;
+
+	if (old != 0 && old != type)
+		g_warning("block %x redefined as type %d, already type %d\n", id, type, old);
+	g_hash_table_insert(map, id, type|old);
+}
+
+static void
+add_partition(GHashTable *map, CamelBlockFile *blocks, camel_block_t id)
+{
+	CamelBlock *bl;
+	CamelPartitionMapBlock *pm;
+	int i;
+
+	while (id) {
+		add_type(map, id, PARTITION_MAP);
+		bl = camel_block_file_get_block(blocks, id);
+		if (bl == NULL) {
+			g_warning("couldn't get parition: %x\n", id);
+			return;
+		}
+
+		pm = (CamelPartitionMapBlock *)&bl->data;
+		if (pm->used > sizeof(pm->partition)/sizeof(pm->partition[0])) {
+			g_warning("Partition block %x invalid\n", id);
+			camel_block_file_unref_block(blocks, bl);
+			return;
+		}
+
+		for (i=0;i<pm->used;i++)
+			add_type(map, pm->partition[i].blockid, PARTITION_DATA);
+
+		id = pm->next;
+		camel_block_file_unref_block(blocks, bl);
+	}
+}
+
+static void
+add_keys(GHashTable *map, CamelBlockFile *blocks, camel_block_t id)
+{
+	CamelBlock *rbl, *bl;
+	CamelKeyRootBlock *root;
+	CamelKeyBlock *kb;
+
+	add_type(map, id, KEY_ROOT);
+	rbl = camel_block_file_get_block(blocks, id);
+	if (rbl == NULL) {
+		g_warning("couldn't get key root: %x\n", id);
+		return;
+	}
+	root = (CamelKeyRootBlock *)&rbl->data;
+	id = root->first;
+
+	while (id) {
+		add_type(map, id, KEY_DATA);
+		bl = camel_block_file_get_block(blocks, id);
+		if (bl == NULL) {
+			g_warning("couldn't get key: %x\n", id);
+			break;
+		}
+
+		kb = (CamelKeyBlock *)&bl->data;
+		id = kb->next;
+		camel_block_file_unref_block(blocks, bl);
+	}
+
+	camel_block_file_unref_block(blocks, rbl);
+}
+
+static void
+dump_raw(GHashTable *map, char *path)
+{
+	char buf[1024];
+	char line[256];
+	char *p, c, *e, *a, *o;
+	int v, n, len, i, type;
+	char hex[16] = "0123456789ABCDEF";
+	int fd;
+	camel_block_t id, total;
+
+	fd = open(path, O_RDONLY);
+	if (fd == -1)
+		return;
+
+	total = 0;
+	while ((len = read(fd, buf, 1024)) == 1024) {
+		id = total;
+
+
+
+		type = g_hash_table_lookup(map, id);
+		switch(type) {
+		case 0:
+			printf(" - unknown -\n");
+			break;
+		default:
+			printf(" - invalid -\n");
+			break;
+		case KEY_ROOT: {
+			CamelKeyRootBlock *r = (CamelKeyRootBlock *)buf;
+			printf("Key root:\n");
+			printf("First: %08x     Last: %08x     Free: %08x\n", r->first, r->last, r->free);
+		} break;
+		case KEY_DATA: {
+			CamelKeyBlock *k = (CamelKeyBlock *)buf;
+			printf("Key data:\n");
+			printf("Next: %08x      Used: %u\n", k->next, k->used);
+			for (i=0;i<k->used;i++) {
+				if (i == 0)
+					len = sizeof(k->u.keydata);
+				else
+					len = k->u.keys[i-1].offset;
+				len -= k->u.keys[i].offset;
+				printf("[%03d]: %08x %5d %06x %3d '%.*s'\n", i,
+				       k->u.keys[i].data, k->u.keys[i].offset, k->u.keys[i].flags,
+				       len, len, k->u.keydata+k->u.keys[i].offset);
+			}
+		} break;
+		case PARTITION_MAP: {
+			CamelPartitionMapBlock *m = (CamelPartitionMapBlock *)buf;
+			printf("Partition map\n");
+			printf("Next: %08x      Used: %u\n", m->next, m->used);
+			for (i=0;i<m->used;i++) {
+				printf("[%03d]: %08x -> %08x\n", i, m->partition[i].hashid, m->partition[i].blockid);
+			}
+		} break;
+		case PARTITION_DATA: {
+			CamelPartitionKeyBlock *k = (CamelPartitionKeyBlock *)buf;
+			printf("Partition data\n");
+			printf("Used: %u\n", k->used);
+		} break;
+		}
+
+
+		printf("--raw--\n");
+
+		len = 1024;
+		p = buf;
+		do {
+			sprintf(line, "%08x:                                                                      ", total);
+			total += 16;
+			o = line+10;
+			a = o+16*2+2;
+			i = 0;
+			while (len && i<16) {
+				c = *p++;
+				*a++ = isprint(c)?c:'.';
+				*o++ = hex[(c>>4)&0x0f];
+				*o++ = hex[c&0x0f];
+				i++;
+				if (i==8)
+					*o++ = ' ';
+				len--;
+			}
+			*a = 0;
+			printf("%s\n", line);
+		} while (len);
+		printf("\n");
+	}
+}
+#endif
+
 /* Debug */
 void
 camel_text_index_dump(CamelTextIndex *idx)
 {
 	struct _CamelTextIndexPrivate *p = CTI_PRIVATE(idx);
+#ifndef DUMP_RAW
 	camel_key_t keyid;
 	char *word;
 	const char *name;
@@ -1013,6 +1190,22 @@ camel_text_index_dump(CamelTextIndex *idx)
 		camel_object_unref((CamelObject *)idc);
 		g_free(word);
 	}
+#else
+	/* a more low-level dump routine */
+	GHashTable *block_type = g_hash_table_new(NULL, NULL);
+	camel_block_t id;
+	struct stat st;
+	int type;
+
+	add_keys(block_type, p->blocks, p->word_index->rootid);
+	add_keys(block_type, p->blocks, p->name_index->rootid);
+
+	add_partition(block_type, p->blocks, p->word_hash->rootid);
+	add_partition(block_type, p->blocks, p->name_hash->rootid);
+
+	dump_raw(block_type, p->blocks->path);
+	g_hash_table_destroy(block_type);
+#endif
 }
 
 /* more debug stuff */
diff --git a/camel/providers/local/camel-local-folder.c b/camel/providers/local/camel-local-folder.c
index a9cb4649aa..3ae6c74873 100644
--- a/camel/providers/local/camel-local-folder.c
+++ b/camel/providers/local/camel-local-folder.c
@@ -216,7 +216,7 @@ camel_local_folder_construct(CamelLocalFolder *lf, CamelStore *parent_store, con
 	   the old-format 'ibex' files that might be lying around */
 	unlink(lf->index_path);
 
-#if 1
+#if 0
 	forceindex = FALSE;
 #else
 	/* if we have no/invalid index file, force it */