Completely new implementation of NNTP.

2001-11-30  Not Zed  <NotZed@Ximian.com>

        * providers/nntp/camel-nntp-*.c:

        Completely new implementation of NNTP.

        Doesn't support subscriptions yet (lists all folders), but should
        be more reliable (faster?), and has an integrated cache.

        * camel-exception.c (camel_exception_new): Use e_memchunks for
        exception blocks.
        (camel_exception_free): Same.

        * camel-data-cache.[ch]: New object for managing on-disk caches of
        anything that can be stored in a camel-stream.

        * camel-file-utils.c (camel_file_util_mkdir): New function, just a
        nicer place to put this (than camel-store), should be removed from
        camel-store.
        (camel_file_util_safe_filename): New function to url-encode a
        filename.

        * camel-mime-parser.c (drop_states): New func to drop the parser
        state to initial state.
        (folder_scan_init_with_fd):
        (folder_scan_init_with_stream): Call above func to reset state if
        the stream is changed on us so we can change streams to reuse a
        parser object.

svn path=/trunk/; revision=14822

18 files changed, 2438 insertions, 877 deletions

diff --git a/camel/ChangeLog b/camel/ChangeLog
index fbe1c03181..b6f1dd7e95 100644
--- a/camel/ChangeLog
+++ b/camel/ChangeLog
@@ -1,3 +1,31 @@
+2001-11-30  Not Zed  <NotZed@Ximian.com>
+
+	* providers/nntp/camel-nntp-*.c:
+	Completely new implementation of NNTP.
+	
+	Doesn't support subscriptions yet (lists all folders), but should
+	be more reliable (faster?), and has an integrated cache.
+
+	* camel-exception.c (camel_exception_new): Use e_memchunks for
+	exception blocks.
+	(camel_exception_free): Same.
+
+	* camel-data-cache.[ch]: New object for managing on-disk caches of
+	anything that can be stored in a camel-stream.
+
+	* camel-file-utils.c (camel_file_util_mkdir): New function, just a
+	nicer place to put this (than camel-store), should be removed from
+	camel-store.
+	(camel_file_util_safe_filename): New function to url-encode a
+	filename.
+
+	* camel-mime-parser.c (drop_states): New func to drop the parser
+	state to initial state.
+	(folder_scan_init_with_fd): 
+	(folder_scan_init_with_stream): Call above func to reset state if
+	the stream is changed on us so we can change streams to reuse a
+	parser object.
+
 2001-11-25  Not Zed  <NotZed@Ximian.com>
 
 	* providers/nntp/camel-nntp-folder.c (nntp_folder_get_message): If
diff --git a/camel/Makefile.am b/camel/Makefile.am
index dc6ae18406..0fad241d6a 100644
--- a/camel/Makefile.am
+++ b/camel/Makefile.am
@@ -28,6 +28,7 @@ libcamel_la_SOURCES = 				\
 	camel-address.c				\
 	camel-cipher-context.c			\
 	camel-cms-context.c			\
+	camel-data-cache.c			\
 	camel-data-wrapper.c			\
 	camel-digest-folder.c			\
 	camel-disco-diary.c			\
@@ -114,6 +115,7 @@ libcamelinclude_HEADERS =			\
 	camel-charset-map.h			\
 	camel-cipher-context.h			\
 	camel-cms-context.h			\
+	camel-data-cache.h			\
 	camel-data-wrapper.h			\
 	camel-digest-folder.h			\
 	camel-disco-diary.h			\
diff --git a/camel/camel-data-cache.c b/camel/camel-data-cache.c
new file mode 100644
index 0000000000..93db52cca7
--- /dev/null
+++ b/camel/camel-data-cache.c
@@ -0,0 +1,476 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/* camel-message-cache.c: Class for a Camel cache.
+ *
+ * Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * Copyright (C) 2001 Ximian, Inc. (www.ximian.com)
+ *
+ * This program is free software; you can redistribute it and/or 
+ * modify it under the terms of version 2 of the GNU General Public 
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <string.h>
+#include <alloca.h>
+
+#include "camel-data-cache.h"
+#include "camel-exception.h"
+#include "camel-stream-fs.h"
+#include "camel-stream-mem.h"
+#include "camel-file-utils.h"
+
+extern int camel_verbose_debug;
+#define dd(x) (camel_verbose_debug?(x):0)
+
+/* how many 'bits' of hash are used to key the toplevel directory */
+#define CAMEL_DATA_CACHE_BITS (6)
+#define CAMEL_DATA_CACHE_MASK ((1<<CAMEL_DATA_CACHE_BITS)-1)
+
+/* timeout before a cache dir is checked again for expired entries,
+   once an hour should be enough */
+#define CAMEL_DATA_CACHE_CYCLE_TIME (60*60)
+
+struct _CamelDataCachePrivate {
+	GHashTable *busy_stream;
+	GHashTable *busy_path;
+
+	int expire_inc;
+	time_t expire_last[1<<CAMEL_DATA_CACHE_BITS];
+
+#ifdef ENABLE_THREADS
+	GMutex *lock;
+#define CDC_LOCK(c, l) g_mutex_lock(((CamelDataCache *)(c))->priv->l)
+#define CDC_UNLOCK(c, l) g_mutex_unlock(((CamelDataCache *)(c))->priv->l)
+#else
+#define CDC_LOCK(c, l)
+#define CDC_UNLOCK(c, l)
+#endif
+};
+
+static CamelObject *camel_data_cache_parent;
+
+static void data_cache_class_init(CamelDataCacheClass *klass)
+{
+	camel_data_cache_parent = (CamelObject *)camel_type_get_global_classfuncs (camel_object_get_type ());
+
+#if 0
+	klass->add = data_cache_add;
+	klass->get = data_cache_get;
+	klass->close = data_cache_close;
+	klass->remove = data_cache_remove;
+	klass->clear = data_cache_clear;
+#endif
+}
+
+static void data_cache_init(CamelDataCache *cdc, CamelDataCacheClass *klass)
+{
+	struct _CamelDataCachePrivate *p;
+
+	p = cdc->priv = g_malloc0(sizeof(*cdc->priv));
+
+	p->busy_stream = g_hash_table_new(NULL, NULL);
+	p->busy_path = g_hash_table_new(g_str_hash, g_str_equal);
+
+#ifdef ENABLE_THREADS
+	p->lock = g_mutex_new();
+#endif
+}
+
+static void
+free_busy(CamelStream *stream, char *path, void *data)
+{
+	camel_object_unref((CamelObject *)stream);
+	g_free(path);
+}
+
+static void data_cache_finalise(CamelDataCache *cdc)
+{
+	struct _CamelDataCachePrivate *p;
+
+	p = cdc->priv;
+
+	g_hash_table_foreach(p->busy_stream, (GHFunc)free_busy, NULL);
+	g_hash_table_destroy(p->busy_path);
+	g_hash_table_destroy(p->busy_stream);
+
+#ifdef ENABLE_THREADS
+	g_mutex_free(p->lock);
+#endif
+	g_free(p);
+}
+
+CamelType
+camel_data_cache_get_type(void)
+{
+	static CamelType camel_data_cache_type = CAMEL_INVALID_TYPE;
+	
+	if (camel_data_cache_type == CAMEL_INVALID_TYPE) {
+		camel_data_cache_type = camel_type_register(
+			CAMEL_OBJECT_TYPE, "CamelDataCache",
+			sizeof (CamelDataCache),
+			sizeof (CamelDataCacheClass),
+			(CamelObjectClassInitFunc) data_cache_class_init,
+			NULL,
+			(CamelObjectInitFunc) data_cache_init,
+			(CamelObjectFinalizeFunc) data_cache_finalise);
+	}
+
+	return camel_data_cache_type;
+}
+
+/**
+ * camel_data_cache_new:
+ * @path: Base path of cache, subdirectories will be created here.
+ * @flags: Open flags, none defined.
+ * @ex: 
+ * 
+ * Create a new data cache.
+ * 
+ * Return value: A new cache object, or NULL if the base path cannot
+ * be written to.
+ **/
+CamelDataCache *
+camel_data_cache_new(const char *path, guint32 flags, CamelException *ex)
+{
+	CamelDataCache *cdc;
+
+	if (camel_file_util_mkdir(path, 0700) == -1) {
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
+				     _("Unable to create cache path"));
+		return NULL;
+	}
+
+	cdc = (CamelDataCache *)camel_object_new(CAMEL_DATA_CACHE_TYPE);
+
+	cdc->path = g_strdup(path);
+	cdc->flags = flags;
+	cdc->expire_age = -1;
+	cdc->expire_access = -1;
+
+	return cdc;
+}
+
+/**
+ * camel_data_cache_set_expire_age:
+ * @cdc: 
+ * @when: Timeout for age expiry, or -1 to disable.
+ * 
+ * Set the cache expiration policy for aged entries.
+ * 
+ * Items in the cache older than @when seconds may be
+ * flushed at any time.  Items are expired in a lazy
+ * manner, so it is indeterminate when the items will
+ * physically be removed.
+ *
+ * Note you can set both an age and an access limit.  The
+ * age acts as a hard limit on cache entries.
+ **/
+void
+camel_data_cache_set_expire_age(CamelDataCache *cdc, time_t when)
+{
+	cdc->expire_age = when;
+}
+
+/**
+ * camel_data_cache_set_expire_access:
+ * @cdc: 
+ * @when: Timeout for access, or -1 to disable access expiry.
+ * 
+ * Set the cache expiration policy for access times.
+ *
+ * Items in the cache which haven't been accessed for @when
+ * seconds may be expired at any time.  Items are expired in a lazy
+ * manner, so it is indeterminate when the items will
+ * physically be removed.
+ *
+ * Note you can set both an age and an access limit.  The
+ * age acts as a hard limit on cache entries.
+ **/
+void
+camel_data_cache_set_expire_access(CamelDataCache *cdc, time_t when)
+{
+	cdc->expire_access = when;
+}
+
+static void
+data_cache_expire(CamelDataCache *cdc, const char *path, const char *keep, time_t now)
+{
+	DIR *dir;
+	struct dirent *d;
+	GString *s;
+	struct stat st;
+	char *oldpath;
+	CamelStream *stream;
+
+	dir = opendir(path);
+	if (dir == NULL)
+		return;
+
+	s = g_string_new("");
+	while ( (d = readdir(dir)) ) {
+		if (strcmp(d->d_name, keep) == 0)
+			continue;
+
+		g_string_sprintf(s, "%s/%s", path, d->d_name);
+		dd(printf("Checking '%s' for expiry\n", s->str));
+		if (stat(s->str, &st) == 0
+		    && S_ISREG(st.st_mode)
+		    && ((cdc->expire_age != -1 && st.st_mtime + cdc->expire_age < now)
+			|| (cdc->expire_access != -1 && st.st_atime + cdc->expire_access < now))) {
+			dd(printf("Has expired!  Removing!\n"));
+			unlink(s->str);
+			if (g_hash_table_lookup_extended(cdc->priv->busy_path, s->str, (void **)&oldpath, (void **)&stream)) {
+				g_hash_table_remove(cdc->priv->busy_path, path);
+				g_hash_table_remove(cdc->priv->busy_stream, stream);
+				g_free(oldpath);
+			}
+		}
+	}
+	g_string_free(s, TRUE);
+	closedir(dir);
+}
+
+/* Since we have to stat the directory anyway, we use this opportunity to
+   lazily expire old data.
+   If it is this directories 'turn', and we haven't done it for CYCLE_TIME seconds,
+   then we perform an expiry run */
+static char *
+data_cache_path(CamelDataCache *cdc, int create, const char *path, const char *key)
+{
+	char *dir, *real, *tmp;
+	guint32 hash;
+
+	hash = g_str_hash(key);
+	hash = (hash>>5)&CAMEL_DATA_CACHE_MASK;
+	dir = alloca(strlen(cdc->path) + strlen(path) + 8);
+	sprintf(dir, "%s/%s/%02x", cdc->path, path, hash);
+	if (access(dir, F_OK) == -1) {
+		if (create)
+			camel_file_util_mkdir(dir, 0700);
+	} else if (cdc->priv->expire_inc == hash
+		   && (cdc->expire_age != -1 || cdc->expire_access != -1)) {
+		time_t now;
+
+		dd(printf("Checking expire cycle time on dir '%s'\n", dir));
+
+		now = time(0);
+		if (cdc->priv->expire_last[hash] + CAMEL_DATA_CACHE_CYCLE_TIME < now) {
+			data_cache_expire(cdc, dir, key, now);
+			cdc->priv->expire_last[hash] = now;
+		}
+		cdc->priv->expire_inc = (cdc->priv->expire_inc + 1) & CAMEL_DATA_CACHE_MASK;
+	}
+
+	tmp = camel_file_util_safe_filename(key);
+	real = g_strdup_printf("%s/%s", dir, tmp);
+	g_free(tmp);
+
+	return real;
+}
+
+static void
+stream_finalised(CamelObject *o, void *event_data, void *data)
+{
+	CamelDataCache *cdc = data;
+	char *key;
+
+	CDC_LOCK(cdc, lock);
+	key = g_hash_table_lookup(cdc->priv->busy_stream, o);
+	if (key) {
+		g_hash_table_remove(cdc->priv->busy_path, key);
+		g_hash_table_remove(cdc->priv->busy_path, o);
+		g_free(key);
+	}
+	CDC_UNLOCK(cdc, lock);
+}
+
+/**
+ * camel_data_cache_add:
+ * @cdc: 
+ * @path: Relative path of item to add.
+ * @key: Key of item to add.
+ * @ex: 
+ * 
+ * Add a new item to the cache.
+ *
+ * The key and the path combine to form a unique key used to store
+ * the item.
+ * 
+ * Potentially, expiry processing will be performed while this call
+ * is executing.
+ *
+ * Return value: A CamelStream (file) opened in read-write mode.
+ * The caller must unref this when finished.
+ **/
+CamelStream *
+camel_data_cache_add(CamelDataCache *cdc, const char *path, const char *key, CamelException *ex)
+{
+	char *real, *oldpath;
+	CamelStream *stream;
+
+	CDC_LOCK(cdc, lock);
+
+	real = data_cache_path(cdc, TRUE, path, key);
+	if (g_hash_table_lookup_extended(cdc->priv->busy_path, real, (void **)&oldpath, (void **)&stream)) {
+		g_hash_table_remove(cdc->priv->busy_path, oldpath);
+		g_hash_table_remove(cdc->priv->busy_stream, stream);
+		unlink(oldpath);
+		g_free(oldpath);
+	}
+
+	stream = camel_stream_fs_new_with_name(real, O_RDWR|O_CREAT|O_TRUNC, 0600);
+	if (stream) {
+		camel_object_hook_event((CamelObject *)stream, "finalize", stream_finalised, cdc);
+		g_hash_table_insert(cdc->priv->busy_stream, stream, real);
+		g_hash_table_insert(cdc->priv->busy_path, real, stream);
+	} else {
+		g_free(real);
+	}
+
+	CDC_UNLOCK(cdc, lock);
+
+	return stream;
+}
+
+/**
+ * camel_data_cache_get:
+ * @cdc: 
+ * @path: Path to the (sub) cache the item exists in.
+ * @key: Key for the cache item.
+ * @ex: 
+ * 
+ * Lookup an item in the cache.  If the item exists, a stream
+ * is returned for the item.  The stream may be shared by
+ * multiple callers, so ensure the stream is in a valid state
+ * through external locking.
+ * 
+ * Return value: A cache item, or NULL if the cache item does not exist.
+ **/
+CamelStream *
+camel_data_cache_get(CamelDataCache *cdc, const char *path, const char *key, CamelException *ex)
+{
+	char *real;
+	CamelStream *stream;
+
+	CDC_LOCK(cdc, lock);
+
+	real = data_cache_path(cdc, FALSE, path, key);
+	stream = g_hash_table_lookup(cdc->priv->busy_path, real);
+	if (stream) {
+		camel_object_ref((CamelObject *)stream);
+		g_free(real);
+	} else {
+		stream = camel_stream_fs_new_with_name(real, O_RDWR, 0600);
+		if (stream) {
+			camel_object_hook_event((CamelObject *)stream, "finalize", stream_finalised, cdc);
+			g_hash_table_insert(cdc->priv->busy_stream, stream, real);
+			g_hash_table_insert(cdc->priv->busy_path, real, stream);
+		}
+	}
+
+	CDC_UNLOCK(cdc, lock);
+
+	return stream;
+}
+
+/**
+ * camel_data_cache_remove:
+ * @cdc: 
+ * @path: 
+ * @key: 
+ * @ex: 
+ * 
+ * Remove/expire a cache item.
+ * 
+ * Return value: 
+ **/
+int
+camel_data_cache_remove(CamelDataCache *cdc, const char *path, const char *key, CamelException *ex)
+{
+	CamelStream *stream;
+	char *real, *oldpath;
+	int ret;
+
+	CDC_LOCK(cdc, lock);
+
+	real = data_cache_path(cdc, FALSE, path, key);
+	if (g_hash_table_lookup_extended(cdc->priv->busy_path, real, (void **)&oldpath, (void **)&stream)) {
+		g_hash_table_remove(cdc->priv->busy_path, path);
+		g_hash_table_remove(cdc->priv->busy_stream, stream);
+		g_free(oldpath);
+	}
+
+	/* maybe we were a mem stream */
+	if (unlink(real) == -1 && errno != ENOENT) {
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
+				     _("Could not remove cache entry: %s: %s"),
+				     real, strerror(errno));
+		ret = -1;
+	} else {
+		ret = 0;
+	}
+
+	g_free(real);
+
+	CDC_UNLOCK(cdc, lock);
+
+	return ret;
+}
+
+/**
+ * camel_data_cache_rename:
+ * @cache: 
+ * @old: 
+ * @new: 
+ * @ex: 
+ * 
+ * Rename a cache path.  All cache items accessed from the old path
+ * are accessible using the new path.
+ *
+ * CURRENTLY UNIMPLEMENTED
+ * 
+ * Return value: -1 on error.
+ **/
+int camel_data_cache_rename(CamelDataCache *cache,
+			    const char *old, const char *new, CamelException *ex)
+{
+	/* blah dont care yet */
+	return -1;
+}
+
+/**
+ * camel_data_cache_clear:
+ * @cache: 
+ * @path: Path to clear, or NULL to clear all items in
+ * all paths.
+ * @ex: 
+ * 
+ * Clear all items in a given cache path or all items in the cache.
+ * 
+ * CURRENTLY_UNIMPLEMENTED
+ *
+ * Return value: -1 on error.
+ **/
+int
+camel_data_cache_clear(CamelDataCache *cache, const char *path, CamelException *ex)
+{
+	/* nor for this? */
+	return -1;
+}
diff --git a/camel/camel-data-cache.h b/camel/camel-data-cache.h
new file mode 100644
index 0000000000..ab6c29a622
--- /dev/null
+++ b/camel/camel-data-cache.h
@@ -0,0 +1,98 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/* camel-data-cache.h: Class for a Camel filesystem cache
+ *
+ * Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * Copyright (C) 2001 Ximian, Inc. (www.ximian.com)
+ *
+ * This program is free software; you can redistribute it and/or 
+ * modify it under the terms of version 2 of the GNU General Public 
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+
+#ifndef CAMEL_DATA_CACHE_H
+#define CAMEL_DATA_CACHE_H 1
+
+#ifdef __cplusplus
+extern "C" {
+#pragma }
+#endif /* __cplusplus }*/
+
+#include <glib.h>
+
+#include <camel/camel-stream.h>
+#include <camel/camel-exception.h>
+
+#define CAMEL_DATA_CACHE_TYPE     (camel_data_cache_get_type ())
+#define CAMEL_DATA_CACHE(obj)     (CAMEL_CHECK_CAST((obj), CAMEL_DATA_CACHE_TYPE, CamelFolder))
+#define CAMEL_DATA_CACHE_CLASS(k) (CAMEL_CHECK_CLASS_CAST ((k), CAMEL_DATA_CACHE_TYPE, CamelFolderClass))
+#define CAMEL_IS_DATA_CACHE(o)    (CAMEL_CHECK_TYPE((o), CAMEL_DATA_CACHE_TYPE))
+
+typedef struct _CamelDataCache CamelDataCache;
+typedef struct _CamelDataCacheClass CamelDataCacheClass;
+
+struct _CamelDataCache {
+	CamelObject parent_object;
+
+	struct _CamelDataCachePrivate *priv;
+
+	char *path;
+	guint32 flags;
+
+	time_t expire_age;
+	time_t expire_access;
+};
+
+struct _CamelDataCacheClass {
+	CamelObjectClass parent_class;
+
+	/* None are virtual yet */
+#if 0
+	/* Virtual methods */
+	CamelStream *(*add)(CamelDataCache *cmc, const char *path, const char *key, CamelException *ex);
+	CamelStream *(*get)(CamelDataCache *cmc, const char *path, const char *key, CamelException *ex);
+	int (*close)(CamelDataCache *cmc, CamelStream *stream, CamelException *ex);
+	int (*remove)(CamelDataCache *cmc, const char *path, const char *key, CamelException *ex);
+
+	int (*clear)(CamelDataCache *cmc, const char *path, CamelException *ex);
+#endif
+};
+
+/* public methods */
+CamelDataCache *camel_data_cache_new(const char *path, guint32 flags, CamelException *ex);
+
+void camel_data_cache_set_expire_age(CamelDataCache *cache, time_t when);
+void camel_data_cache_set_expire_access(CamelDataCache *cdc, time_t when);
+
+int             camel_data_cache_rename(CamelDataCache *cache,
+					const char *old, const char *new, CamelException *ex);
+
+CamelStream    *camel_data_cache_add(CamelDataCache *cache,
+				     const char *path, const char *key, CamelException *ex);
+CamelStream    *camel_data_cache_get(CamelDataCache *cache,
+				     const char *path, const char *key, CamelException *ex);
+int             camel_data_cache_remove(CamelDataCache *cache,
+					const char *path, const char *key, CamelException *ex);
+
+int             camel_data_cache_clear(CamelDataCache *cache,
+				       const char *path, CamelException *ex);
+
+/* Standard Camel function */
+CamelType camel_data_cache_get_type (void);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* CAMEL_DATA_CACHE_H */
diff --git a/camel/camel-exception.c b/camel/camel-exception.c
index c5eec66908..1bb6c22550 100644
--- a/camel/camel-exception.c
+++ b/camel/camel-exception.c
@@ -29,6 +29,7 @@
 
 #include <glib.h>
 #include "camel-exception.h"
+#include "e-util/e-memory.h"
 
 /* i dont know why gthread_mutex stuff even exists, this is easier */
 
@@ -46,6 +47,8 @@ static pthread_mutex_t exception_mutex = PTHREAD_MUTEX_INITIALIZER;
 #define CAMEL_EXCEPTION_UNLOCK(e) 
 #endif
 
+static EMemChunk *exception_chunks = NULL;
+
 /**
  * camel_exception_new: allocate a new exception object. 
  * 
@@ -59,12 +62,19 @@ camel_exception_new (void)
 {
 	CamelException *ex;
 
-	ex = g_new (CamelException, 1);
+	CAMEL_EXCEPTION_LOCK(exception);
+
+	if (exception_chunks == NULL)
+		exception_chunks = e_memchunk_new(16, sizeof(CamelException));
+
+	ex = e_memchunk_alloc(exception_chunks);
 	ex->desc = NULL;
 
 	/* set the Exception Id to NULL */
 	ex->id = CAMEL_EXCEPTION_NONE;
 
+	CAMEL_EXCEPTION_UNLOCK(exception);
+
 	return ex;
 }
 
@@ -129,7 +139,12 @@ camel_exception_free (CamelException *exception)
 	
 	if (exception->desc)
 		g_free (exception->desc);
-	g_free (exception);
+
+	CAMEL_EXCEPTION_LOCK(exception);
+
+	e_memchunk_free(exception_chunks, exception);
+
+	CAMEL_EXCEPTION_UNLOCK(exception);
 }
 
 /**
diff --git a/camel/camel-file-utils.c b/camel/camel-file-utils.c
index 9331c64bfa..eed36f1383 100644
--- a/camel/camel-file-utils.c
+++ b/camel/camel-file-utils.c
@@ -1,6 +1,5 @@
-/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-
-/* 
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *
  * Authors:
  *   Michael Zucchi <notzed@ximian.com>
  *   Dan Winship <danw@ximian.com>
@@ -24,6 +23,14 @@
 
 
 #include "camel-file-utils.h"
+#include "camel-url.h"
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <string.h>
 
 #include <netinet/in.h>
 
@@ -279,4 +286,37 @@ camel_file_util_decode_string (FILE *in, char **str)
 	return 0;
 }
 
+/* Make a directory heirarchy.
+   Always use full paths */
+int
+camel_file_util_mkdir(const char *path, mode_t mode)
+{
+	char *copy, *p;
 
+	g_assert(path && path[0] == '/');
+
+	p = copy = alloca(strlen(path)+1);
+	strcpy(copy, path);
+	do {
+		p = strchr(p + 1, '/');
+		if (p)
+			*p = '\0';
+		if (access(copy, F_OK) == -1) {
+			if (mkdir(copy, mode) == -1)
+				return -1;
+		}
+		if (p)
+			*p = '/';
+	} while (p);
+
+	return 0;
+}
+
+char *
+camel_file_util_safe_filename(const char *name)
+{
+	if (name == NULL)
+		return NULL;
+	
+	return camel_url_encode(name, TRUE, "/?()'*");
+}
diff --git a/camel/camel-file-utils.h b/camel/camel-file-utils.h
index b45063752a..e261ead16b 100644
--- a/camel/camel-file-utils.h
+++ b/camel/camel-file-utils.h
@@ -47,6 +47,9 @@ int camel_file_util_decode_off_t (FILE *in, off_t *);
 int camel_file_util_encode_string (FILE *out, const char *);
 int camel_file_util_decode_string (FILE *in, char **);
 
+int camel_file_util_mkdir(const char *path, mode_t mode);
+char *camel_file_util_safe_filename(const char *name);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
diff --git a/camel/camel-mime-parser.c b/camel/camel-mime-parser.c
index a53e0716a7..ea7b75f5ca 100644
--- a/camel/camel-mime-parser.c
+++ b/camel/camel-mime-parser.c
@@ -1523,6 +1523,16 @@ folder_scan_init(void)
 	return s;
 }
 
+static void
+drop_states(struct _header_scan_state *s)
+{
+	while (s->parts) {
+		folder_scan_drop_step(s);
+	}
+	s->unstep = 0;
+	s->state = HSCAN_INITIAL;
+}
+
 static int
 folder_scan_init_with_fd(struct _header_scan_state *s, int fd)
 {
@@ -1530,6 +1540,7 @@ folder_scan_init_with_fd(struct _header_scan_state *s, int fd)
 
 	len = read(fd, s->inbuf, SCAN_BUF);
 	if (len>=0) {
+		drop_states(s);
 		s->inend = s->inbuf+len;
 		s->inptr = s->inbuf;
 		s->inend[0] = '\n';
@@ -1555,6 +1566,7 @@ folder_scan_init_with_stream(struct _header_scan_state *s, CamelStream *stream)
 
 	len = camel_stream_read(stream, s->inbuf, SCAN_BUF);
 	if (len >= 0) {
+		drop_states(s);
 		s->inend = s->inbuf+len;
 		s->inptr = s->inbuf;
 		s->inend[0] = '\n';
diff --git a/camel/providers/nntp/Makefile.am b/camel/providers/nntp/Makefile.am
index 1244c04e52..7279c40be8 100644
--- a/camel/providers/nntp/Makefile.am
+++ b/camel/providers/nntp/Makefile.am
@@ -18,23 +18,37 @@ INCLUDES = -I../..				\
 	-DG_LOG_DOMAIN=\"camel-nntp-provider\"
 
 libcamelnntp_la_SOURCES = 			\
-	camel-nntp-auth.c			\
-	camel-nntp-folder.c			\
-	camel-nntp-grouplist.c			\
-	camel-nntp-newsrc.c			\
 	camel-nntp-provider.c			\
 	camel-nntp-store.c			\
-	camel-nntp-utils.c
+	camel-nntp-folder.c			\
+	camel-nntp-stream.c			\
+	camel-nntp-summary.c
+
+#	camel-nntp-auth.c			\
+#	camel-nntp-folder.c			\
+#	camel-nntp-grouplist.c			\
+#	camel-nntp-newsrc.c			\
+#	camel-nntp-provider.c			\
+#	camel-nntp-store.c			\
+#	camel-nntp-utils.c
 
 libcamelnntpinclude_HEADERS =			\
-	camel-nntp-auth.h			\
-	camel-nntp-folder.h			\
-	camel-nntp-grouplist.h			\
-	camel-nntp-newsrc.h			\
-	camel-nntp-resp-codes.h			\
 	camel-nntp-store.h			\
-	camel-nntp-types.h			\
-	camel-nntp-utils.h
+	camel-nntp-folder.h			\
+	camel-nntp-stream.h			\
+	camel-nntp-summary.h
+
+#	camel-nntp-auth.h			\
+#	camel-nntp-folder.h			\
+#	camel-nntp-grouplist.h			\
+#	camel-nntp-newsrc.h			\
+#	camel-nntp-resp-codes.h			\
+#	camel-nntp-store.h			\
+#	camel-nntp-types.h			\
+#	camel-nntp-utils.h
+
+noinst_HEADERS =				\
+	camel-nntp-private.h
 
 libcamelnntp_la_LDFLAGS = -version-info 0:0:0
 
diff --git a/camel/providers/nntp/camel-nntp-folder.c b/camel/providers/nntp/camel-nntp-folder.c
index e3b4da6b40..47d3111a01 100644
--- a/camel/providers/nntp/camel-nntp-folder.c
+++ b/camel/providers/nntp/camel-nntp-folder.c
@@ -1,10 +1,10 @@
 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-/* camel-nntp-folder.c : Abstract class for an email folder */
-
-/* 
- * Author : Chris Toshok <toshok@ximian.com> 
+/* camel-nntp-folder.c : Class for a news folder
+ *
+ * Authors : Chris Toshok <toshok@ximian.com> 
+ *           Michael Zucchi <notzed@ximian.com>
  *
- * Copyright (C) 2000 Ximian .
+ * Copyright (C) 2001 Ximian .
  *
  * This program is free software; you can redistribute it and/or 
  * modify it under the terms of version 2 of the GNU General Public 
@@ -33,21 +33,20 @@
 #include <string.h>
 #include <fcntl.h>
 
-#include "camel-folder-summary.h"
-#include "camel-nntp-resp-codes.h"
+#include "camel/string-utils.h"
+#include "camel/camel-stream-mem.h"
+#include "camel/camel-data-wrapper.h"
+#include "camel/camel-mime-message.h"
+#include "camel/camel-folder-search.h"
+#include "camel/camel-exception.h"
+#include "camel/camel-session.h"
+#include "camel/camel-data-cache.h"
+
+#include "camel-nntp-summary.h"
 #include "camel-nntp-store.h"
 #include "camel-nntp-folder.h"
 #include "camel-nntp-store.h"
-#include "camel-nntp-utils.h"
-
-#include "string-utils.h"
-#include "camel-stream-mem.h"
-#include "camel-data-wrapper.h"
-#include "camel-mime-message.h"
-#include "camel-folder-summary.h"
-#include "camel-folder-search.h"
-
-#include "camel-exception.h"
+#include "camel-nntp-private.h"
 
 static CamelFolderClass *parent_class=NULL;
 
@@ -56,131 +55,131 @@ static CamelFolderClass *parent_class=NULL;
 #define CF_CLASS(so) CAMEL_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(so))
 #define CNNTPS_CLASS(so) CAMEL_STORE_CLASS (CAMEL_OBJECT_GET_CLASS(so))
 
-
 static void
-nntp_folder_sync (CamelFolder *folder, gboolean expunge, 
-		  CamelException *ex)
+nntp_folder_sync (CamelFolder *folder, gboolean expunge, CamelException *ex)
 {
-	CamelNNTPStore *store;
+	CamelNNTPStore *nntp_store;
+	CamelFolderChangeInfo *changes = NULL;
+	CamelNNTPFolder *nntp_folder;
 
-	camel_folder_summary_save (folder->summary);
+	nntp_store = (CamelNNTPStore *)folder->parent_store;
+	nntp_folder = (CamelNNTPFolder *)folder;
 
-	store = CAMEL_NNTP_STORE (camel_folder_get_parent_store (folder));
+	CAMEL_NNTP_STORE_LOCK(nntp_store, command_lock);
 
-	if (store->newsrc)
-		camel_nntp_newsrc_write (store->newsrc);
+	if (camel_nntp_summary_check((CamelNNTPSummary *)folder->summary, nntp_folder->changes, ex) != -1)
+		camel_folder_summary_save (folder->summary);
+
+	if (camel_folder_change_info_changed(nntp_folder->changes)) {
+		changes = nntp_folder->changes;
+		nntp_folder->changes = camel_folder_change_info_new();
+	}
+
+	CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
+
+	if (changes) {
+		camel_object_trigger_event((CamelObject *)folder, "folder_changed", changes);
+		camel_folder_change_info_free(changes);
+	}
 }
 
 static void
-nntp_folder_set_message_flags (CamelFolder *folder, const char *uid,
-			       guint32 flags, guint32 set)
+nntp_folder_set_message_flags(CamelFolder *folder, const char *uid, guint32 flags, guint32 set)
 {
         ((CamelFolderClass *)parent_class)->set_message_flags(folder, uid, flags, set);
-
-	if (flags & set & CAMEL_MESSAGE_SEEN) {
-		int article_num;
-		CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE (camel_folder_get_parent_store (folder));
-
-		sscanf (uid, "%d", &article_num);
-
-		camel_nntp_newsrc_mark_article_read (nntp_store->newsrc,
-						     folder->name,
-						     article_num);
-	}
 }
 
 static CamelMimeMessage *
-nntp_folder_get_message (CamelFolder *folder, const gchar *uid, CamelException *ex)
+nntp_folder_get_message (CamelFolder *folder, const char *uid, CamelException *ex)
 {
-	CamelStream *message_stream = NULL;
 	CamelMimeMessage *message = NULL;
-	CamelStore *parent_store;
-	char *buf;
-	int buf_len;
-	int buf_alloc;
-	int status;
-	gboolean done;
-	char *message_id;
-
-	/* get the parent store */
-	parent_store = camel_folder_get_parent_store (folder);
-
-	message_id = strchr (uid, ',');
-	if (message_id) {
-		message_id++;
-		status = camel_nntp_command (CAMEL_NNTP_STORE( parent_store ), ex, NULL, "ARTICLE %s", message_id);
+	CamelNNTPStore *nntp_store;
+	CamelFolderChangeInfo *changes;
+	CamelNNTPFolder *nntp_folder;
+	CamelStream *stream = NULL;
+	int ret;
+	char *line;
+	const char *msgid;
+
+	nntp_store = (CamelNNTPStore *)folder->parent_store;
+	nntp_folder = (CamelNNTPFolder *)folder;
+
+	CAMEL_NNTP_STORE_LOCK(nntp_store, command_lock);
+
+	msgid = strchr(uid, ',');
+	if (msgid == 0) {
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
+				     _("Internal error: uid in invalid format: %s"), uid);
+		goto fail;
 	}
-
-	/* if the message_id was not found, raise an exception and return */
-	if (message_id == NULL || status == NNTP_NO_SUCH_ARTICLE) {
-		camel_exception_setv (ex, 
-				     CAMEL_EXCEPTION_FOLDER_INVALID_UID,
-				     _("Message %s not found."),
-				      uid);
-		return NULL;
-	}
-	else if (status != NNTP_ARTICLE_FOLLOWS) {
-		/* XXX */
-		g_warning ("weird nntp error %d\n", status);
-		return NULL;
+	msgid++;
+
+	/* Lookup in cache, NEWS is global messageid's so use a global cache path */
+	stream = camel_data_cache_get(nntp_store->cache, "cache", msgid, NULL);
+	if (stream == NULL) {
+		/* Not in cache, retrieve and put in cache */
+		if (camel_nntp_store_set_folder(nntp_store, folder, nntp_folder->changes, ex) == -1)
+			goto fail;
+
+		ret = camel_nntp_command(nntp_store, &line, "article %s", msgid);
+		if (ret == -1)
+			goto error;
+
+		if (ret == 220) {
+			stream = camel_data_cache_add(nntp_store->cache, "cache", msgid, NULL);
+			if (stream) {
+				if (camel_stream_write_to_stream((CamelStream *)nntp_store->stream, stream) == -1)
+					goto error;
+				if (camel_stream_reset(stream) == -1)
+					goto error;
+			} else {
+				stream = (CamelStream *)nntp_store->stream;
+				camel_object_ref((CamelObject *)stream);
+			}
+		}
 	}
 
-	/* this could probably done fairly easily with an nntp stream that
-	   returns eof after '.' */
+	if (stream) {
+		message = camel_mime_message_new();
+		if (camel_data_wrapper_construct_from_stream((CamelDataWrapper *)message, stream) == -1)
+			goto error;
 
-	/* XXX ick ick ick.  read the entire message into a buffer and
-	   then create a stream_mem for it. */
-	buf_alloc = 2048;
-	buf_len = 0;
-	buf = g_malloc(buf_alloc);
-	done = FALSE;
+		CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
 
-	buf[0] = 0;
+		camel_object_unref((CamelObject *)stream);
+		return message;
+	}
 
-	while (!done) {
-		int line_length;
-		char *line;
+	camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Cannot get message %s: %s"), uid, line);
 
-		if (camel_remote_store_recv_line (CAMEL_REMOTE_STORE (parent_store), &line, ex) < 0) {
-			g_warning ("recv_line failed while building message\n");
-			break;
-		}
+error:
+	if (errno == EINTR)
+		camel_exception_setv(ex, CAMEL_EXCEPTION_USER_CANCEL, _("User cancelled"));
+	else
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Cannot get message %s: %s"), uid, strerror(errno));
 
-		/* XXX check exception */
+fail:
+	if (message)
+		camel_object_unref((CamelObject *)message);
 
-		line_length = strlen ( line );
+	if (stream)
+		camel_object_unref((CamelObject *)stream);
 
-		if (!strcmp(line, ".")) {
-			done = TRUE;
-			g_free (line);
-		}
-		else {
-			if (buf_len + line_length > buf_alloc) {
-				buf_alloc *= 2;
-				buf = g_realloc (buf, buf_alloc);
-			}
-			strcat(buf, line);
-			strcat(buf, "\n");
-			buf_len += strlen(line) + 1;
-			g_free (line);
-		}
+	if (camel_folder_change_info_changed(nntp_folder->changes)) {
+		changes = nntp_folder->changes;
+		nntp_folder->changes = camel_folder_change_info_new();
+	} else {
+		changes = NULL;
 	}
 
-	/* create a stream bound to the message */
-	message_stream = camel_stream_mem_new_with_buffer(buf, buf_len);
-
-	message = camel_mime_message_new ();
-	camel_data_wrapper_construct_from_stream (CAMEL_DATA_WRAPPER(message), message_stream);
-
-	camel_object_unref (CAMEL_OBJECT (message_stream));
+	CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
 
-#if 0
-	gtk_signal_connect (CAMEL_OBJECT (message), "message_changed", message_changed, folder);
-#endif
-
-	g_free (buf);
+	if (changes) {
+		camel_object_trigger_event((CamelObject *)folder, "folder_changed", changes);
+		camel_folder_change_info_free(changes);
+	}
 
-	return message;
+	return NULL;
 }
 
 static GPtrArray*
@@ -188,7 +187,9 @@ nntp_folder_search_by_expression (CamelFolder *folder, const char *expression, C
 {
 	CamelNNTPFolder *nntp_folder = CAMEL_NNTP_FOLDER (folder);
 	GPtrArray *matches, *summary;
-	
+
+	CAMEL_NNTP_FOLDER_LOCK(nntp_folder, search_lock);
+
 	if(nntp_folder->search == NULL)
 		nntp_folder->search = camel_folder_search_new();
 	
@@ -198,11 +199,54 @@ nntp_folder_search_by_expression (CamelFolder *folder, const char *expression, C
 
 	matches = camel_folder_search_execute_expression(nntp_folder->search, expression, ex);
 
+	CAMEL_NNTP_FOLDER_UNLOCK(nntp_folder, search_lock);
+
 	camel_folder_free_summary(folder, summary);
 
 	return matches;
 }
 
+static GPtrArray *
+nntp_folder_search_by_uids(CamelFolder *folder, const char *expression, GPtrArray *uids, CamelException *ex)
+{
+	CamelNNTPFolder *nntp_folder = CAMEL_NNTP_FOLDER(folder);
+	GPtrArray *summary, *matches;
+	int i;
+
+	/* NOTE: could get away without the search lock by creating a new
+	   search object each time */
+
+	summary = g_ptr_array_new();
+	for (i=0;i<uids->len;i++) {
+		CamelMessageInfo *info;
+
+		info = camel_folder_get_message_info(folder, uids->pdata[i]);
+		if (info)
+			g_ptr_array_add(summary, info);
+	}
+
+	if (summary->len == 0)
+		return summary;
+
+	CAMEL_NNTP_FOLDER_LOCK(folder, search_lock);
+
+	if (nntp_folder->search == NULL)
+		nntp_folder->search = camel_folder_search_new();
+
+	camel_folder_search_set_folder(nntp_folder->search, folder);
+	camel_folder_search_set_summary(nntp_folder->search, summary);
+
+	matches = camel_folder_search_execute_expression(nntp_folder->search, expression, ex);
+
+	CAMEL_NNTP_FOLDER_UNLOCK(folder, search_lock);
+
+	for (i=0;i<summary->len;i++)
+		camel_folder_free_message_info(folder, summary->pdata[i]);
+	g_ptr_array_free(summary, TRUE);
+
+	return matches;
+}
+
 static void
 nntp_folder_search_free(CamelFolder *folder, GPtrArray *result)
 {
@@ -213,15 +257,19 @@ nntp_folder_search_free(CamelFolder *folder, GPtrArray *result)
 }
 
 static void           
-nntp_folder_finalize (CamelObject *object)
+nntp_folder_init(CamelNNTPFolder *nntp_folder, CamelNNTPFolderClass *klass)
 {
-	CamelNNTPFolder *nntp_folder = CAMEL_NNTP_FOLDER (object);
+	nntp_folder->changes = camel_folder_change_info_new();
+}
 
-	g_free (nntp_folder->summary_file_path);
+static void           
+nntp_folder_finalise (CamelNNTPFolder *nntp_folder)
+{
+	g_free(nntp_folder->storage_path);
 }
 
 static void
-camel_nntp_folder_class_init (CamelNNTPFolderClass *camel_nntp_folder_class)
+nntp_folder_class_init (CamelNNTPFolderClass *camel_nntp_folder_class)
 {
 	CamelFolderClass *camel_folder_class = CAMEL_FOLDER_CLASS (camel_nntp_folder_class);
 
@@ -234,6 +282,7 @@ camel_nntp_folder_class_init (CamelNNTPFolderClass *camel_nntp_folder_class)
 	camel_folder_class->set_message_flags = nntp_folder_set_message_flags;
 	camel_folder_class->get_message = nntp_folder_get_message;
 	camel_folder_class->search_by_expression = nntp_folder_search_by_expression;
+	camel_folder_class->search_by_uids = nntp_folder_search_by_uids;
 	camel_folder_class->search_free = nntp_folder_search_free;
 }
 
@@ -246,46 +295,97 @@ camel_nntp_folder_get_type (void)
 		camel_nntp_folder_type = camel_type_register (CAMEL_FOLDER_TYPE, "CamelNNTPFolder",
 							      sizeof (CamelNNTPFolder),
 							      sizeof (CamelNNTPFolderClass),
-							      (CamelObjectClassInitFunc) camel_nntp_folder_class_init,
+							      (CamelObjectClassInitFunc) nntp_folder_class_init,
 							      NULL,
-							      (CamelObjectInitFunc) NULL,
-							      (CamelObjectFinalizeFunc) nntp_folder_finalize);
+							      (CamelObjectInitFunc) nntp_folder_init,
+							      (CamelObjectFinalizeFunc) nntp_folder_finalise);
 	}
 	
 	return camel_nntp_folder_type;
 }
 
+
+/* not yet */
+/* Idea is we update in stages, but this requires a different xover command, etc */
+#ifdef ASYNC_SUMMARY
+struct _folder_check_msg {
+	CamelSessionThreadMsg msg;
+	CamelNNTPFolder *folder;
+};
+
+static void
+folder_check(CamelSession *session, CamelSessionThreadMsg *msg)
+{
+	struct _folder_check_msg *m = (struct _folder_check_msg *)msg;
+	CamelException *ex;
+	CamelNNTPStore *nntp_store;
+
+	nntp_store = (CamelNNTPStore *)m->folder->parent.parent_store;
+
+	CAMEL_NNTP_STORE_LOCK(nntp_store, command_lock);
+
+	ex = camel_exception_new();
+	camel_nntp_summary_check((CamelNNTPSummary *)m->folder->parent.summary, m->folder->changes, ex);
+	camel_exception_free(ex);
+
+	CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
+}
+
+static void
+folder_check_free(CamelSession *session, CamelSessionThreadMsg *msg)
+{
+	struct _folder_check_msg *m = (struct _folder_check_msg *)msg;
+
+	camel_object_unref((CamelObject *)m->folder);
+}
+
+static CamelSessionThreadOps folder_check_ops = {
+	folder_check,
+	folder_check_free,
+};
+#endif
+
 CamelFolder *
 camel_nntp_folder_new (CamelStore *parent, const char *folder_name, CamelException *ex)
 {
-	CamelFolder *folder = CAMEL_FOLDER (camel_object_new (CAMEL_NNTP_FOLDER_TYPE));
-	CamelNNTPFolder *nntp_folder = CAMEL_NNTP_FOLDER (folder);
-	const gchar *root_dir_path;
+	CamelFolder *folder;
+	CamelNNTPFolder *nntp_folder;
+	char *root;
+	CamelService *service;
+#ifdef ASYNC_SUMMARY
+	struct _folder_check_msg *m;
+#endif
 
-	camel_folder_construct (folder, parent, folder_name, folder_name);
-	folder->folder_flags |= (CAMEL_FOLDER_HAS_SUMMARY_CAPABILITY |
-				 CAMEL_FOLDER_HAS_SEARCH_CAPABILITY);
+	service = (CamelService *)parent;
+	root = camel_session_get_storage_path(service->session, service, ex);
+	if (root == NULL)
+		return NULL;
+
+	/* If this doesn't work, stuff wont save, but let it continue anyway */
+	(void) camel_mkdir_hier(root, 0777);
+
+	folder = (CamelFolder *) camel_object_new (CAMEL_NNTP_FOLDER_TYPE);
+	nntp_folder = (CamelNNTPFolder *)folder;
 
-	root_dir_path = camel_nntp_store_get_toplevel_dir (CAMEL_NNTP_STORE(folder->parent_store));
-	nntp_folder->summary_file_path = g_strdup_printf ("%s/%s-ev-summary",
-							  root_dir_path,
-							  folder->name);
+	camel_folder_construct (folder, parent, folder_name, folder_name);
+	folder->folder_flags |= CAMEL_FOLDER_HAS_SUMMARY_CAPABILITY|CAMEL_FOLDER_HAS_SEARCH_CAPABILITY;
 
-	folder->summary = camel_folder_summary_new ();
-	camel_folder_summary_set_filename (folder->summary,
-					   nntp_folder->summary_file_path);
+	nntp_folder->storage_path = g_strdup_printf("%s/%s", root, folder->full_name);
+	g_free(root);
 
+	folder->summary = (CamelFolderSummary *)camel_nntp_summary_new(nntp_folder);
 	camel_folder_summary_load (folder->summary);
-		
-	camel_nntp_get_headers (CAMEL_FOLDER( folder )->parent_store,
-				nntp_folder, ex);
-	if (camel_exception_get_id (ex)) {
-		camel_object_unref (CAMEL_OBJECT (folder));
-		return NULL;
+#ifdef ASYNC_SUMMARY
+	m = camel_session_thread_msg_new(service->session, &folder_check_ops, sizeof(*m));
+	m->folder = nntp_folder;
+	camel_object_ref((CamelObject *)folder);
+	camel_session_thread_queue(service->session, &m->msg, 0);
+#else
+	if (camel_nntp_summary_check((CamelNNTPSummary *)folder->summary, nntp_folder->changes, ex) == -1) {
+		camel_object_unref((CamelObject *)folder);
+		folder = NULL;
 	}
+#endif
 
-	/* XXX check return value */
-	camel_folder_summary_save (folder->summary);
-	
 	return folder;
 }
diff --git a/camel/providers/nntp/camel-nntp-folder.h b/camel/providers/nntp/camel-nntp-folder.h
index 223323cb95..300d61762d 100644
--- a/camel/providers/nntp/camel-nntp-folder.h
+++ b/camel/providers/nntp/camel-nntp-folder.h
@@ -32,7 +32,7 @@ extern "C" {
 #pragma }
 #endif /* __cplusplus }*/
 
-#include "camel-folder.h"
+#include "camel/camel-folder.h"
 
 /*  #include "camel-store.h" */
 
@@ -41,25 +41,23 @@ extern "C" {
 #define CAMEL_NNTP_FOLDER_CLASS(k) (CAMEL_CHECK_CLASS_CAST ((k), CAMEL_NNTP_FOLDER_TYPE, CamelNNTPFolderClass))
 #define CAMEL_IS_NNTP_FOLDER(o)    (CAMEL_CHECK_TYPE((o), CAMEL_NNTP_FOLDER_TYPE))
 
+typedef struct _CamelNNTPFolder {
+	CamelFolder parent;
 
-typedef struct {
-	CamelFolder parent_object;
+	struct _CamelNNTPFolderPrivate *priv;
 
-	gchar *summary_file_path;  /* contains the messages summary */
-	CamelFolderSummary *summary;
+	struct _CamelFolderChangeInfo *changes;
+	char *storage_path;
 	CamelFolderSearch *search;
 } CamelNNTPFolder;
 
-
-
-typedef struct {
-	CamelFolderClass parent_class;
+typedef struct _CamelNNTPFolderClass {
+	CamelFolderClass parent;
 
 	/* Virtual methods */	
 	
 } CamelNNTPFolderClass;
 
-
 /* public methods */
 
 /* Standard Camel function */
diff --git a/camel/providers/nntp/camel-nntp-private.h b/camel/providers/nntp/camel-nntp-private.h
new file mode 100644
index 0000000000..95b29ba6b9
--- /dev/null
+++ b/camel/providers/nntp/camel-nntp-private.h
@@ -0,0 +1,74 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*-
+ *  camel-nntp-private.h: Private info for nntp.
+ *
+ * Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * Copyright 1999, 2000 Ximian, Inc. (www.ximian.com)
+ *
+ * This program is free software; you can redistribute it and/or 
+ * modify it under the terms of version 2 of the GNU General Public 
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+#ifndef CAMEL_NNTP_PRIVATE_H
+#define CAMEL_NNTP_PRIVATE_H 1
+
+#ifdef __cplusplus
+extern "C" {
+#pragma }
+#endif /* __cplusplus }*/
+
+/* need a way to configure and save this data, if this header is to
+   be installed.  For now, dont install it */
+
+#include "config.h"
+
+#ifdef ENABLE_THREADS
+#include "e-util/e-msgport.h"
+#endif
+
+struct _CamelNNTPStorePrivate {
+#ifdef ENABLE_THREADS
+	EMutex *command_lock;	/* for locking the command stream for a complete operation */
+#endif
+};
+
+#ifdef ENABLE_THREADS
+#define CAMEL_NNTP_STORE_LOCK(f, l) (e_mutex_lock(((CamelNNTPStore *)f)->priv->l))
+#define CAMEL_NNTP_STORE_UNLOCK(f, l) (e_mutex_unlock(((CamelNNTPStore *)f)->priv->l))
+#else
+#define CAMEL_NNTP_STORE_LOCK(f, l)
+#define CAMEL_NNTP_STORE_UNLOCK(f, l)
+#endif
+
+struct _CamelNNTPFolderPrivate {
+#ifdef ENABLE_THREADS
+	GMutex *search_lock;	/* for locking the search object */
+	GMutex *cache_lock;     /* for locking the cache object */
+#endif
+};
+
+#ifdef ENABLE_THREADS
+#define CAMEL_NNTP_FOLDER_LOCK(f, l) (g_mutex_lock(((CamelNNTPFolder *)f)->priv->l))
+#define CAMEL_NNTP_FOLDER_UNLOCK(f, l) (g_mutex_unlock(((CamelNNTPFolder *)f)->priv->l))
+#else
+#define CAMEL_NNTP_FOLDER_LOCK(f, l)
+#define CAMEL_NNTP_FOLDER_UNLOCK(f, l)
+#endif
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* CAMEL_NNTP_PRIVATE_H */
+
diff --git a/camel/providers/nntp/camel-nntp-store.c b/camel/providers/nntp/camel-nntp-store.c
index 7ac3fcdc82..feca770e37 100644
--- a/camel/providers/nntp/camel-nntp-store.c
+++ b/camel/providers/nntp/camel-nntp-store.c
@@ -3,7 +3,10 @@
 
 /* 
  *
- * Copyright (C) 2000 Ximian, Inc. <toshok@ximian.com>
+ * Copyright (C) 2001 Ximian, Inc. <www.ximain.com>
+ *
+ * Authors: Christopher Toshok <toshok@ximian.com>
+ *    	    Michael Zucchi <notzed@ximian.com>
  *
  * This program is free software; you can redistribute it and/or 
  * modify it under the terms of version 2 of the GNU General Public 
@@ -29,17 +32,22 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "libgnome/libgnome.h"
+#include "camel/camel-exception.h"
+#include "camel/camel-url.h"
+#include "camel/string-utils.h"
+#include "camel/camel-stream-mem.h"
+#include "camel/camel-session.h"
+#include "camel/camel-data-cache.h"
 
-#include "camel-nntp-resp-codes.h"
-#include "camel-folder-summary.h"
+#include "camel-nntp-stream.h"
+#include "camel-nntp-summary.h"
 #include "camel-nntp-store.h"
-#include "camel-nntp-grouplist.h"
 #include "camel-nntp-folder.h"
-#include "camel-nntp-auth.h"
-#include "camel-exception.h"
-#include "camel-url.h"
-#include "string-utils.h"
+#include "camel-nntp-private.h"
+
+#define w(x)
+extern int camel_verbose_debug;
+#define dd(x) (camel_verbose_debug?(x):0)
 
 #define NNTP_PORT 119
 
@@ -57,249 +65,73 @@ static CamelServiceClass *service_class = NULL;
 #define CF_CLASS(so) CAMEL_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(so))
 #define CNNTPF_CLASS(so) CAMEL_NNTP_FOLDER_CLASS (CAMEL_OBJECT_GET_CLASS(so))
 
-static gboolean ensure_news_dir_exists (CamelNNTPStore *store);
-
-static int
-camel_nntp_store_set_mode (CamelNNTPStore *store, CamelException *ex)
-{
-	int rc;
-
-	CAMEL_NNTP_STORE_LOCK(store);
-	rc = camel_nntp_command(store, ex, NULL, "MODE READER");
-	CAMEL_NNTP_STORE_UNLOCK(store);
-
-	return rc;
-}
-
-static void
-camel_nntp_store_get_extensions (CamelNNTPStore *store, CamelException *ex)
-{
-	int rc;
-
-	store->extensions = 0;
-
-	CAMEL_NNTP_STORE_LOCK(store);
-	rc = camel_nntp_command(store, ex, NULL, "LIST EXTENSIONS");
-	if (rc == NNTP_LIST_FOLLOWS || rc == NNTP_EXTENSIONS_SUPPORTED) {
-		gboolean done = FALSE;
-		CamelException ex;
-
-		camel_exception_init (&ex);
-
-		while (!done) {
-			char *line;
-
-			if (camel_remote_store_recv_line (CAMEL_REMOTE_STORE (store), &line, &ex) < 0)
-				break; /* XXX */
-
-			if (*line == '.') {
-				done = TRUE;
-			}
-			else {
-#define CHECK_EXT(name,val) if (!strcasecmp (line, (name))) store->extensions |= (val)
-
-				CHECK_EXT ("SEARCH",     CAMEL_NNTP_EXT_SEARCH);
-				CHECK_EXT ("SETGET",     CAMEL_NNTP_EXT_SETGET);
-				CHECK_EXT ("OVER",       CAMEL_NNTP_EXT_OVER);
-				CHECK_EXT ("XPATTEXT",   CAMEL_NNTP_EXT_XPATTEXT);
-				CHECK_EXT ("XACTIVE",    CAMEL_NNTP_EXT_XACTIVE);
-				CHECK_EXT ("LISTMOTD",   CAMEL_NNTP_EXT_LISTMOTD);
-				CHECK_EXT ("LISTSUBSCR", CAMEL_NNTP_EXT_LISTSUBSCR);
-				CHECK_EXT ("LISTPNAMES", CAMEL_NNTP_EXT_LISTPNAMES);
-
-#undef CHECK_EXT
-			}
-
-			g_free (line);
-		}
-	}
-	CAMEL_NNTP_STORE_UNLOCK(store);
-
-#ifdef DUMP_EXTENSIONS
-	g_print ("NNTP Extensions:");
-#define DUMP_EXT(name,val) if (store->extensions & (val)) g_print (" %s", name);
-	DUMP_EXT ("SEARCH",     CAMEL_NNTP_EXT_SEARCH);
-	DUMP_EXT ("SETGET",     CAMEL_NNTP_EXT_SETGET);
-	DUMP_EXT ("OVER",       CAMEL_NNTP_EXT_OVER);
-	DUMP_EXT ("XPATTEXT",   CAMEL_NNTP_EXT_XPATTEXT);
-	DUMP_EXT ("XACTIVE",    CAMEL_NNTP_EXT_XACTIVE);
-	DUMP_EXT ("LISTMOTD",   CAMEL_NNTP_EXT_LISTMOTD);
-	DUMP_EXT ("LISTSUBSCR", CAMEL_NNTP_EXT_LISTSUBSCR);
-	DUMP_EXT ("LISTPNAMES", CAMEL_NNTP_EXT_LISTPNAMES);
-	g_print ("\n");
-#undef DUMP_EXT
-#endif
-}
-
-static void
-camel_nntp_store_get_overview_fmt (CamelNNTPStore *store, CamelException *ex)
-{
-	int status;
-	int i;
-	gboolean done = FALSE;
-	
-	CAMEL_NNTP_STORE_LOCK(store);
-	status = camel_nntp_command (store, ex, NULL,
-				     "LIST OVERVIEW.FMT");
-
-	if (status != NNTP_LIST_FOLLOWS) {
-		if (store->extensions & CAMEL_NNTP_EXT_OVER) {
-			/* if we can't get the overview format, we should
-			   disable OVER support */
-			g_warning ("server reported support of OVER but LIST OVERVIEW.FMT failed."
-				   "  disabling OVER.\n");
-			store->extensions &= ~CAMEL_NNTP_EXT_OVER;
-		}
-		CAMEL_NNTP_STORE_UNLOCK(store);
-		return;
-	}
-	else {
-		if (!(store->extensions & CAMEL_NNTP_EXT_OVER)) {
-			g_warning ("server didn't report support of OVER but LIST OVERVIEW.FMT worked."
-				   "  enabling OVER.\n");
-			store->extensions |= CAMEL_NNTP_EXT_OVER;
-		}
-	}
-
-	/* start at 1 because the article number is always first */
-	store->num_overview_fields = 1;
-	
-	for (i = 0; i < CAMEL_NNTP_OVER_LAST; i ++) {
-		store->overview_field [i].index = -1;
-	}
-
-	while (!done) {
-		char *line;
-
-		if (camel_remote_store_recv_line (CAMEL_REMOTE_STORE (store), &line, ex) < 0)
-			break; /* XXX */
-
-		if (*line == '.') {
-			done = TRUE;
-		}
-		else {
-			CamelNNTPOverField *over_field = NULL;
-			char *colon = NULL;;
-
-			if (!strncasecmp (line, "From:", 5)) {
-				over_field = &store->overview_field [ CAMEL_NNTP_OVER_FROM ];
-				over_field->index = store->num_overview_fields;
-				colon = line + 5;
-			}
-			else if (!strncasecmp (line, "Subject:", 7)) {
-				over_field = &store->overview_field [ CAMEL_NNTP_OVER_SUBJECT ];
-				over_field->index = store->num_overview_fields;
-				colon = line + 7;
-			}
-			else if (!strncasecmp (line, "Date:", 5)) {
-				over_field = &store->overview_field [ CAMEL_NNTP_OVER_DATE ];
-				over_field->index = store->num_overview_fields;
-				colon = line + 5;
-			}
-			else if (!strncasecmp (line, "Message-ID:", 11)) {
-				over_field = &store->overview_field [ CAMEL_NNTP_OVER_MESSAGE_ID ];
-				over_field->index = store->num_overview_fields;
-				colon = line + 11;
-			}
-			else if (!strncasecmp (line, "References:", 11)) {
-				over_field = &store->overview_field [ CAMEL_NNTP_OVER_REFERENCES ];
-				over_field->index = store->num_overview_fields;
-				colon = line + 11;
-			}
-			else if (!strncasecmp (line, "Bytes:", 6)) {
-				over_field = &store->overview_field [ CAMEL_NNTP_OVER_BYTES ];
-				over_field->index = store->num_overview_fields;
-				colon = line + 11;
-			}
-		
-			if (colon && !strncmp (colon + 1, "full", 4))
-				over_field->full = TRUE;
-
-			store->num_overview_fields ++;
-		}
-
-		g_free (line);
-	}
-
-	for (i = 0; i < CAMEL_NNTP_OVER_LAST; i ++) {
-		if (store->overview_field [i].index == -1) {
-			g_warning ("server's OVERVIEW.FMT doesn't support minimum set we require,"
-				   " disabling OVER support.\n");
-			store->extensions &= ~CAMEL_NNTP_EXT_OVER;
-		}
-	}
-	CAMEL_NNTP_STORE_UNLOCK(store);
-}
-
 static gboolean
 nntp_store_connect (CamelService *service, CamelException *ex)
 {
-	char *buf;
-	int resp_code;
+	unsigned char *line;
+	unsigned int len;
+	int ret = FALSE;
 	CamelNNTPStore *store = CAMEL_NNTP_STORE (service);
 
-	if (!ensure_news_dir_exists(store)) {
-		camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
-				      _("Could not open directory for news server: %s"),
-				      strerror (errno));
-		return FALSE;
-	}
+	CAMEL_NNTP_STORE_LOCK(store, command_lock);
 
-	if (CAMEL_SERVICE_CLASS (remote_store_class)->connect (service, ex) == FALSE)
-		return FALSE;
+	/* setup store-wide cache */
+	if (store->cache == NULL) {
+		char *root;
 
-	/* Read the greeting */
-	if (camel_remote_store_recv_line (CAMEL_REMOTE_STORE (service), &buf, ex) < 0) {
-		return FALSE;
-	}
+		root = camel_session_get_storage_path(service->session, service, ex);
+		if (root == NULL)
+			goto fail;
 
-	resp_code = atoi (buf);
-	/* Check to see if we have any permissions on this server. */
-	if (resp_code == NNTP_NO_PERMISSION)
-		return FALSE;
+		store->cache = camel_data_cache_new(root, 0, ex);
+		g_free(root);
+		if (store->cache == NULL)
+			goto fail;
 
-	/* check if posting is allowed. */
-	if (resp_code == NNTP_GREETING_POSTING_OK) {
-		g_print ("posting allowed\n");
-		store->posting_allowed = TRUE;
-	}
-	else if (resp_code == NNTP_GREETING_NO_POSTING) {
-		g_print ("no posting allowed\n");
-		store->posting_allowed = FALSE;
-	}
-	else {
-		g_warning ("unexpected server greeting code %d, no posting allowed\n", resp_code);
-		store->posting_allowed = FALSE;
+		/* Default cache expiry - 2 weeks old, or not visited in 5 days */
+		camel_data_cache_set_expire_age(store->cache, 60*60*24*14);
+		camel_data_cache_set_expire_access(store->cache, 60*60*24*5);
 	}
 
-	g_free (buf);
+	if (CAMEL_SERVICE_CLASS (remote_store_class)->connect (service, ex) == FALSE)
+		goto fail;
 
-	/* set 'reader' mode */
-	camel_nntp_store_set_mode(store, ex);
+	store->stream = (CamelNNTPStream *)camel_nntp_stream_new(((CamelRemoteStore *)service)->ostream);
+	if (camel_nntp_stream_line(store->stream, &line, &len) == -1)
+		goto fail;
 
-	/* get a list of extensions that the server supports */
-	camel_nntp_store_get_extensions (store, ex);
+	len = strtoul(line, (char **)&line, 10);
+	if (len != 200 && len != 201)
+		goto fail;
 
-	/* try to get the overview.fmt */
-	camel_nntp_store_get_overview_fmt (store, ex);
+	/* set 'reader' mode & ignore return code */
+	camel_nntp_command(store, (char **)&line, "mode reader");
+	ret = TRUE;
+fail:
+	CAMEL_NNTP_STORE_UNLOCK(store, command_lock);
 
-	return TRUE;
+	return ret;
 }
 
 static gboolean
 nntp_store_disconnect (CamelService *service, gboolean clean, CamelException *ex)
 {
 	CamelNNTPStore *store = CAMEL_NNTP_STORE (service);
+	char *line;
 
-	if (clean)
-		camel_nntp_command (store, ex, NULL, "QUIT");
+	CAMEL_NNTP_STORE_LOCK(store, command_lock);
 
-	if (store->newsrc)
-		camel_nntp_newsrc_write (store->newsrc);
+	if (clean)
+		camel_nntp_command (store, &line, "quit");
 
 	if (!service_class->disconnect (service, clean, ex))
 		return FALSE;
 
+	camel_object_unref((CamelObject *)store->stream);
+	store->stream = NULL;
+
+	CAMEL_NNTP_STORE_UNLOCK(store, command_lock);
+
 	return TRUE;
 }
 
@@ -334,211 +166,51 @@ nntp_store_query_auth_types (CamelService *service, CamelException *ex)
 }
 
 static CamelFolder *
-nntp_store_get_folder (CamelStore *store, const gchar *folder_name,
-		       guint32 flags, CamelException *ex)
+nntp_store_get_folder(CamelStore *store, const char *folder_name, guint32 flags, CamelException *ex)
 {
 	CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE (store);
+	CamelFolder *folder;
 
-	/* if we haven't already read our .newsrc, read it now */
-	if (!nntp_store->newsrc)
-		nntp_store->newsrc = 
-		camel_nntp_newsrc_read_for_server (CAMEL_SERVICE(store)->url->host);
-
-	if (!nntp_store->newsrc) {
-		camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
-				      _("Unable to open or create .newsrc file for %s: %s"),
-				      CAMEL_SERVICE(store)->url->host,
-				      strerror(errno));
-		return NULL;
-	}
-
-	return camel_nntp_folder_new (store, folder_name, ex);
-}
-
-#ifdef INFO_AS_TREE
-static void
-build_folder_info (CamelNNTPStore *nntp_store, CamelFolderInfo **root,
-		   CamelFolderInfo *parent, CamelNNTPGroupListEntry *entry,
-		   char *prefix, char *suffix,
-		   GHashTable *name_to_info)
-{
-	CamelURL *url = CAMEL_SERVICE (nntp_store)->url;
-	char *dot;
-	if ((dot = strchr (suffix, '.'))) {
-		/* it's an internal node, figure out the next node in
-                   the chain */
-		CamelFolderInfo *node;
-		char *node_name, *node_full_name;
-
-		node_name = g_malloc0 (dot - suffix + 1);
-		strncpy (node_name, suffix, dot - suffix);
-		node_full_name = g_strdup_printf ("%s.%s", prefix, node_name);
-
-		node = g_hash_table_lookup (name_to_info, node_full_name);
-		if (!node) {
-			/* we need to add one */
-			node = g_new0 (CamelFolderInfo, 1);
-			node->name = g_strdup (node_name);
-			node->full_name = g_strdup (node_full_name);
-			node->url = NULL;
-			node->unread_message_count = -1;
-
-			if (parent) {
-				if (parent->child) {
-					node->sibling = parent->child;
-					parent->child = node;
-				}
-				else {
-					parent->child = node;
-				}
-			}
-			else {
-				if (*root) {
-					*root = node;
-				}
-				else {
-					node->sibling = *root;
-					*root = node;
-				}
-			}
-
-			g_hash_table_insert (name_to_info, node_full_name, node);
-		}
-
-		build_folder_info (nntp_store, root, node, entry, node_full_name, dot + 1, name_to_info);
-	}
-	else {
-		/* it's a leaf node, make the CamelFolderInfo and
-                   append it to @parent's list of children. */
-		CamelFolderInfo *new_group;
-
-		new_group = g_new0 (CamelFolderInfo, 1);
-		new_group->name = g_strdup (entry->group_name);
-		new_group->full_name = g_strdup (entry->group_name);
-		new_group->url = g_strdup_printf ("nntp://%s%s%s/%s",
-						  url->user ? url->user : "",
-						  url->user ? "@" : "",
-						  url->host, (char *)entry->group_name);
-
-		new_group->unread_message_count = (entry->high - entry->low - 
-						   camel_nntp_newsrc_get_num_articles_read (nntp_store->newsrc, entry->group_name));
-
-		if (parent) {
-			if (parent->child) {
-				new_group->sibling = parent->child;
-				parent->child = new_group;
-			}
-			else {
-				parent->child = new_group;
-			}
-		}
-		else {
-			if (*root) {
-				*root = new_group;
-			}
-			else {
-				new_group->sibling = *root;
-				*root = new_group;
-			}
-		}
-	}
-}
-#endif
-
-static CamelFolderInfo *
-build_folder_info_from_grouplist (CamelNNTPStore *nntp_store, const char *top)
-{
-	GList *g;
-	CamelFolderInfo *groups = NULL;
-#ifdef INFO_AS_TREE
-	GHashTable *hash = g_hash_table_new (g_str_hash, g_str_equal);
-#else
-	CamelFolderInfo *last = NULL, *fi;
-	CamelURL *url = CAMEL_SERVICE (nntp_store)->url;
-#endif
+	CAMEL_NNTP_STORE_LOCK(nntp_store, command_lock);
 
-	for (g = nntp_store->group_list->group_list; g; g = g_list_next (g)) {
-		CamelNNTPGroupListEntry *entry = g->data;
-
-		if (!top || !strncmp (top, entry->group_name, strlen (top))) {
-#ifdef INFO_AS_TREE
-			build_folder_info (nntp_store, &groups, NULL, entry,
-					   "", entry->group_name, hash);
-#else
-
-			fi = g_new0 (CamelFolderInfo, 1);
-			fi->name = g_strdup (entry->group_name);
-			fi->full_name = g_strdup (entry->group_name);
-			fi->url = g_strdup_printf ("nntp://%s%s%s/%s",
-						   url->user ? url->user : "",
-						   url->user ? "@" : "",
-						   url->host, (char *)entry->group_name);
-
-			fi->unread_message_count = (entry->high - entry->low - 
-						    camel_nntp_newsrc_get_num_articles_read (
-							     nntp_store->newsrc, entry->group_name));
-			camel_folder_info_build_path(fi, '/');
+	folder = camel_nntp_folder_new(store, folder_name, ex);
 
-			if (last)
-				last->sibling = fi;
-			else
-				groups = fi;
-			last = fi;
-#endif
-		}
-	}
+	CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
 
-	return groups;
+	return folder;
 }
 
 static CamelFolderInfo *
-nntp_store_get_folder_info (CamelStore *store, const char *top,
-			    guint32 flags,
-			    CamelException *ex)
+nntp_store_get_folder_info(CamelStore *store, const char *top, guint32 flags, CamelException *ex)
 {
 	CamelURL *url = CAMEL_SERVICE (store)->url;
 	CamelNNTPStore *nntp_store = (CamelNNTPStore *)store;
-	GPtrArray *names;
 	CamelFolderInfo *groups = NULL, *last = NULL, *fi;
-	int i;
-
-	/* if we haven't already read our .newsrc, read it now */
-	if (!nntp_store->newsrc)
-		nntp_store->newsrc = 
-		camel_nntp_newsrc_read_for_server (CAMEL_SERVICE(store)->url->host);
-
-	if (!nntp_store->newsrc) {
-		camel_exception_setv (ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
-				      _("Unable to open or create .newsrc file for %s: %s"),
-				      CAMEL_SERVICE(store)->url->host,
-				      strerror(errno));
-		return NULL;
-	}
+	unsigned int len;
+	unsigned char *line, *space;
+	int ret = -1;
 
-	if (!(flags & CAMEL_STORE_FOLDER_INFO_SUBSCRIBED)) {
-		if (!nntp_store->group_list)
-			nntp_store->group_list = camel_nntp_grouplist_fetch (nntp_store, ex);
-		if (camel_exception_is_set (ex)) {
-			return NULL;
-		}
-		else {
-			fi = build_folder_info_from_grouplist (nntp_store, top);
-			return fi;
-		}
+	CAMEL_NNTP_STORE_LOCK(nntp_store, command_lock);
+
+	ret = camel_nntp_command(nntp_store, (char **)&line, "list");
+	if (ret != 215) {
+		ret = -1;
+		goto error;
 	}
 
-	if (top == NULL) {
-		/* return the list of groups */
-		names = camel_nntp_newsrc_get_subscribed_group_names (nntp_store->newsrc);
-		for (i = 0; i < names->len; i++) {
-			fi = g_new0 (CamelFolderInfo, 1);
-			fi->name = g_strdup (names->pdata[i]);
-			fi->full_name = g_strdup (names->pdata[i]);
-			fi->url = g_strdup_printf ("nntp://%s%s%s/%s",
-						   url->user ? url->user : "",
-						   url->user ? "@" : "",
-						   url->host, (char *)names->pdata[i]);
-			/* FIXME */
+	while ( (ret = camel_nntp_stream_line(nntp_store->stream, &line, &len)) > 0) {
+		space = strchr(line, ' ');
+		if (space)
+			*space = 0;
+
+		if (top == NULL || top[0] == 0 || strcmp(top, line) == 0) {
+			fi = g_malloc0(sizeof(*fi));
+			fi->name = g_strdup(line);
+			fi->full_name = g_strdup(line);
+			if (url->user)
+				fi->url = g_strdup_printf ("nntp://%s@%s/%s", url->user, url->host, line);
+			else
+				fi->url = g_strdup_printf ("nntp://%s/%s", url->host, line);
 			fi->unread_message_count = -1;
 			camel_folder_info_build_path(fi, '/');
 
@@ -548,23 +220,22 @@ nntp_store_get_folder_info (CamelStore *store, const char *top,
 				groups = fi;
 			last = fi;
 		}
-		camel_nntp_newsrc_free_group_names (nntp_store->newsrc, names);
-
-		return groups;
-	}
-	else {
-		/* getting a specific group */
-
-		fi = g_new0 (CamelFolderInfo, 1);
-		fi->name = g_strdup (top);
-		fi->full_name = g_strdup (top);
-		fi->url = g_strdup_printf ("nntp://%s/%s", url->host, top);
-		/* FIXME */
-		fi->unread_message_count = -1;
-		camel_folder_info_build_path(fi, '/');
-
-		return fi;
 	}
+
+	if (ret < 0)
+		goto error;
+
+	CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
+
+	return groups;
+
+error:
+	CAMEL_NNTP_STORE_UNLOCK(nntp_store, command_lock);
+
+	if (groups)
+		camel_store_free_folder_info(store, groups);
+
+	return NULL;
 }
 
 static gboolean
@@ -572,7 +243,11 @@ nntp_store_folder_subscribed (CamelStore *store, const char *folder_name)
 {
 	CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE (store);
 
-	return camel_nntp_newsrc_group_is_subscribed (nntp_store->newsrc, folder_name);
+	nntp_store = nntp_store;
+
+	/* FIXME: implement */
+
+	return TRUE;
 }
 
 static void
@@ -581,7 +256,9 @@ nntp_store_subscribe_folder (CamelStore *store, const char *folder_name,
 {
 	CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE (store);
 
-	camel_nntp_newsrc_subscribe_group (nntp_store->newsrc, folder_name);
+	nntp_store = nntp_store;
+
+	/* FIXME: implement */
 }
 
 static void
@@ -590,22 +267,32 @@ nntp_store_unsubscribe_folder (CamelStore *store, const char *folder_name,
 {
 	CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE (store);
 
-	camel_nntp_newsrc_unsubscribe_group (nntp_store->newsrc, folder_name);
+	nntp_store = nntp_store;
+
+	/* FIXME: implement */
 }
 
 static void
-finalize (CamelObject *object)
+nntp_store_finalise (CamelObject *object)
 {
 	CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE (object);
-	if (nntp_store->newsrc)
-		camel_nntp_newsrc_write (nntp_store->newsrc);
+	struct _CamelNNTPStorePrivate *p = nntp_store->priv;
+
+	camel_service_disconnect((CamelService *)object, TRUE, NULL);
+
+	camel_object_unref((CamelObject *)nntp_store->mem);
+	nntp_store->mem = NULL;
+	if (nntp_store->stream)
+		camel_object_unref((CamelObject *)nntp_store->stream);
+
 #ifdef ENABLE_THREADS
-	e_mutex_destroy(nntp_store->command_lock);
+	e_mutex_destroy(p->command_lock);
 #endif
+	g_free(p);
 }
 
 static void
-camel_nntp_store_class_init (CamelNNTPStoreClass *camel_nntp_store_class)
+nntp_store_class_init (CamelNNTPStoreClass *camel_nntp_store_class)
 {
 	CamelStoreClass *camel_store_class = CAMEL_STORE_CLASS (camel_nntp_store_class);
 	CamelServiceClass *camel_service_class = CAMEL_SERVICE_CLASS (camel_nntp_store_class);
@@ -630,21 +317,23 @@ camel_nntp_store_class_init (CamelNNTPStoreClass *camel_nntp_store_class)
 	camel_store_class->unsubscribe_folder = nntp_store_unsubscribe_folder;
 }
 
-
-
 static void
-camel_nntp_store_init (gpointer object, gpointer klass)
+nntp_store_init (gpointer object, gpointer klass)
 {
 	CamelRemoteStore *remote_store = CAMEL_REMOTE_STORE (object);
 	CamelNNTPStore *nntp_store = CAMEL_NNTP_STORE(object);
 	CamelStore *store = CAMEL_STORE (object);
+	struct _CamelNNTPStorePrivate *p;
 
 	remote_store->default_port = NNTP_PORT;
 
 	store->flags = CAMEL_STORE_SUBSCRIPTIONS;
 
+	nntp_store->mem = (CamelStreamMem *)camel_stream_mem_new();
+
+	p = nntp_store->priv = g_malloc0(sizeof(*p));
 #ifdef ENABLE_THREADS
-	nntp_store->command_lock = e_mutex_new(E_MUTEX_REC);
+	p->command_lock = e_mutex_new(E_MUTEX_REC);
 #endif
 }
 
@@ -657,238 +346,113 @@ camel_nntp_store_get_type (void)
 		camel_nntp_store_type = camel_type_register (CAMEL_REMOTE_STORE_TYPE, "CamelNNTPStore",
 							     sizeof (CamelNNTPStore),
 							     sizeof (CamelNNTPStoreClass),
-							     (CamelObjectClassInitFunc) camel_nntp_store_class_init,
+							     (CamelObjectClassInitFunc) nntp_store_class_init,
 							     NULL,
-							     (CamelObjectInitFunc) camel_nntp_store_init,
-							     (CamelObjectFinalizeFunc) finalize);
+							     (CamelObjectInitFunc) nntp_store_init,
+							     (CamelObjectFinalizeFunc) nntp_store_finalise);
 	}
 	
 	return camel_nntp_store_type;
 }
 
-
-/**
- * camel_nntp_command: Send a command to a NNTP server.
- * @store: the NNTP store
- * @ret: a pointer to return the full server response in
- * @fmt: a printf-style format string, followed by arguments
- *
- * This command sends the command specified by @fmt and the following
- * arguments to the connected NNTP store specified by @store. It then
- * reads the server's response and parses out the status code. If
- * the caller passed a non-NULL pointer for @ret, camel_nntp_command
- * will set it to point to an buffer containing the rest of the
- * response from the NNTP server. (If @ret was passed but there was
- * no extended response, @ret will be set to NULL.) The caller must
- * free this buffer when it is done with it.
- *
- * Return value: the response code of the nntp command.
- **/
-static int
-camel_nntp_command_send_recv (CamelNNTPStore *store, CamelException *ex, char **ret, char *cmd)
+/* enter owning lock */
+int camel_nntp_store_set_folder(CamelNNTPStore *store, CamelFolder *folder, CamelFolderChangeInfo *changes, CamelException *ex)
 {
-	char *respbuf;
-	int resp_code;
-	gboolean again;
+	int ret;
 
-	do {
-		again = FALSE;
+	if (store->current_folder && strcmp(folder->full_name, store->current_folder) == 0)
+		return 0;
 
-		/* Send the command */
-		if (camel_remote_store_send_string (CAMEL_REMOTE_STORE (store), ex, cmd) < 0) {
-			return NNTP_PROTOCOL_ERROR;
-		}
-
-		/* Read the response */
-		if (camel_remote_store_recv_line (CAMEL_REMOTE_STORE (store), &respbuf, ex) < 0) {
-			if (ret)
-				*ret = g_strdup (g_strerror (errno));
-			return NNTP_PROTOCOL_ERROR;
-		}
+	/* FIXME: Do something with changeinfo */
+	ret = camel_nntp_summary_check((CamelNNTPSummary *)folder->summary, changes, ex);
 
-		resp_code = atoi (respbuf);
+	g_free(store->current_folder);
+	store->current_folder = g_strdup(folder->full_name);
 
-		/* this is kind of a gross hack, but since an auth challenge
-		   can pop up at any time, and we want to shield this from our
-		   callers, we handle authentication here. */
-		if (resp_code == NNTP_AUTH_REQUIRED) {
-			resp_code = camel_nntp_auth_authenticate (store, ex);
-			if (resp_code != NNTP_AUTH_ACCEPTED) {
-				return resp_code;
-			}
-
-			/* need to resend our command here */
-			again = TRUE;
-		}
-	} while (again);
-
-	if (ret) {
-		*ret = strchr (respbuf, ' ');
-		if (*ret)
-			*ret = g_strdup (*ret + 1);
-	}
-	g_free (respbuf);
-
-	return resp_code;
+	return ret;
 }
 
+/* Enter owning lock */
 int
-camel_nntp_command (CamelNNTPStore *store, CamelException *ex, char **ret, char *fmt, ...)
+camel_nntp_command(CamelNNTPStore *store, char **line, const char *fmt, ...)
 {
-	char *cmdbuf;
+	const unsigned char *p, *ps;
+	unsigned char c;
 	va_list ap;
-	int resp_code;
-	char *real_fmt;
-
-	real_fmt = g_strdup_printf ("%s\r\n", fmt);
-
-	va_start (ap, fmt);
-	cmdbuf = g_strdup_vprintf (real_fmt, ap);
-	va_end (ap);
+	char *s;
+	int d;
+	unsigned int u, u2;
 
-	g_free (real_fmt);
+	e_mutex_assert_locked(store->priv->command_lock);
 
-	resp_code = camel_nntp_command_send_recv (store, ex, ret, cmdbuf);
+	if (!camel_remote_store_connected((CamelRemoteStore *)store, NULL))
+		return -1;
 
-	if(camel_exception_get_id(ex) == 
-	   CAMEL_EXCEPTION_SERVICE_NOT_CONNECTED) {
-		/* the connect might have timed out, give it another shot.. */
-		camel_exception_clear(ex);
-		if(nntp_store_connect(CAMEL_SERVICE(store), ex))
-			resp_code = 
-				camel_nntp_command_send_recv (store, ex, ret, cmdbuf);
-		/* that's it, no more tries */
+	/* Check for unprocessed data, ! */
+	if (store->stream->mode == CAMEL_NNTP_STREAM_DATA) {
+		g_warning("Unprocessed data left in stream, flushing");
+		while (camel_nntp_stream_getd(store->stream, (unsigned char **)&p, &u) > 0)
+			;
 	}
-
-	g_free (cmdbuf);
-
-	return resp_code;
-}
-
-void
-camel_nntp_store_subscribe_group (CamelStore *store,
-				  const gchar *group_name)
-{
-	gchar *root_dir = camel_nntp_store_get_toplevel_dir(CAMEL_NNTP_STORE(store));
-	char *ret = NULL;
-	CamelException *ex = camel_exception_new();
-
-	if (camel_exception_get_id (ex)) {
-		g_free (root_dir);
-		camel_exception_free (ex);
-		return;
-	}
-
-	if (camel_nntp_command ( CAMEL_NNTP_STORE (store),
-				 ex, &ret, "GROUP %s", group_name) == NNTP_GROUP_SELECTED) {
-		/* we create an empty summary file here, so that when
-                   the group is opened we'll know we need to build it. */
-		gchar *summary_file;
-		int fd;
-		summary_file = g_strdup_printf ("%s/%s-ev-summary", root_dir, group_name);
-		
-		fd = open (summary_file, O_CREAT | O_RDWR, 0666);
-		close (fd);
-
-		g_free (summary_file);
-	}
-	if (ret) g_free (ret);
-
-	g_free (root_dir);
-	camel_exception_free (ex);
-}
-
-void
-camel_nntp_store_unsubscribe_group (CamelStore *store,
-				    const gchar *group_name)
-{
-	gchar *root_dir = camel_nntp_store_get_toplevel_dir(CAMEL_NNTP_STORE(store));
-	gchar *summary_file;
-
-	summary_file = g_strdup_printf ("%s/%s-ev-summary", root_dir, group_name);
-	if (g_file_exists (summary_file))
-		unlink (summary_file);
-	g_free (summary_file);
-
-	g_free (root_dir);
-}
-
-GList *
-camel_nntp_store_list_subscribed_groups(CamelStore *store)
-{
-	GList *group_name_list = NULL;
-	struct stat stat_buf;
-	gint stat_error = 0;
-	gchar *entry_name;
-	gchar *full_entry_name;
-	gchar *real_group_name;
-	struct dirent *dir_entry;
-	DIR *dir_handle;
-	gchar *root_dir = camel_nntp_store_get_toplevel_dir(CAMEL_NNTP_STORE(store));
-
-	dir_handle = opendir (root_dir);
-	g_return_val_if_fail (dir_handle, NULL);
-
-	/* read the first entry in the directory */
-	dir_entry = readdir (dir_handle);
-	while ((stat_error != -1) && (dir_entry != NULL)) {
-
-		/* get the name of the next entry in the dir */
-		entry_name = dir_entry->d_name;
-		full_entry_name = g_strdup_printf ("%s/%s", root_dir, entry_name);
-		stat_error = stat (full_entry_name, &stat_buf);
-		g_free (full_entry_name);
-
-		/* is it a normal file ending in -ev-summary ? */
-		if ((stat_error != -1) && S_ISREG (stat_buf.st_mode)) {
-			gboolean summary_suffix_found;
-
-			real_group_name = string_prefix (entry_name, "-ev-summary",
-							 &summary_suffix_found);
-
-			if (summary_suffix_found)
-				/* add the folder name to the list */
-				group_name_list = g_list_append (group_name_list, 
-								 real_group_name);
+	camel_nntp_stream_set_mode(store->stream, CAMEL_NNTP_STREAM_LINE);
+
+	va_start(ap, fmt);
+	ps = p = fmt;
+	while ( (c = *p++) ) {
+		switch (c) {
+		case '%':
+			c = *p++;
+			camel_stream_write((CamelStream *)store->mem, ps, p-ps-(c=='%'?1:2));
+			ps = p;
+			switch (c) {
+			case 's':
+				s = va_arg(ap, char *);
+				camel_stream_write((CamelStream *)store->mem, s, strlen(s));
+				break;
+			case 'd':
+				d = va_arg(ap, int);
+				camel_stream_printf((CamelStream *)store->mem, "%d", d);
+				break;
+			case 'u':
+				u = va_arg(ap, unsigned int);
+				camel_stream_printf((CamelStream *)store->mem, "%u", u);
+				break;
+			case 'm':
+				s = va_arg(ap, char *);
+				camel_stream_printf((CamelStream *)store->mem, "<%s>", s);
+				break;
+			case 'r':
+				u = va_arg(ap, unsigned int);
+				u2 = va_arg(ap, unsigned int);
+				if (u == u2)
+					camel_stream_printf((CamelStream *)store->mem, "%u", u);
+				else
+					camel_stream_printf((CamelStream *)store->mem, "%u-%u", u, u2);
+				break;
+			default:
+				g_warning("Passing unknown format to nntp_command: %c\n", c);
+				g_assert(0);
+			}
 		}
-		/* read next entry */
-		dir_entry = readdir (dir_handle);
 	}
 
-	closedir (dir_handle);
-
-	return group_name_list;
-}
+	camel_stream_write((CamelStream *)store->mem, ps, p-ps-1);
+	dd(printf("NNTP_COMMAND: '%.*s'\n", (int)store->mem->buffer->len, store->mem->buffer->data));
+	camel_stream_write((CamelStream *)store->mem, "\r\n", 2);
+	camel_stream_write((CamelStream *)store->stream, store->mem->buffer->data, store->mem->buffer->len);
+	camel_stream_reset((CamelStream *)store->mem);
+	/* FIXME: hack */
+	g_byte_array_set_size(store->mem->buffer, 0);
 
-gchar *
-camel_nntp_store_get_toplevel_dir (CamelNNTPStore *store)
-{
-	CamelURL *url = CAMEL_SERVICE (store)->url;
-	char *top_dir;
+	if (camel_nntp_stream_line(store->stream, (unsigned char **)line, &u) == -1)
+		return -1;
 
-	g_assert(url != NULL);
+	u = strtoul(*line, NULL, 10);
 
-	top_dir = g_strdup_printf( "%s/evolution/news/%s",
-				   g_get_home_dir (),
-				   url->host );
+	/* Handle all switching to data mode here, to make callers job easier */
+	if (u == 215 || (u >= 220 && u <=224) || (u >= 230 && u <= 231))
+		camel_nntp_stream_set_mode(store->stream, CAMEL_NNTP_STREAM_DATA);
 
-	return top_dir;
+	return u;
 }
 
-static gboolean
-ensure_news_dir_exists (CamelNNTPStore *store)
-{
-	gchar *dir = camel_nntp_store_get_toplevel_dir (store);
-
-	if (access (dir, F_OK) == 0) {
-		g_free (dir);
-		return TRUE;
-	}
-
-	if (camel_mkdir_hier (dir, S_IRWXU) == -1) {
-		g_free (dir);
-		return FALSE;
-	}
-
-	return TRUE;
-}
diff --git a/camel/providers/nntp/camel-nntp-store.h b/camel/providers/nntp/camel-nntp-store.h
index 0f373dd0bc..04a69b02e7 100644
--- a/camel/providers/nntp/camel-nntp-store.h
+++ b/camel/providers/nntp/camel-nntp-store.h
@@ -30,41 +30,15 @@ extern "C" {
 #pragma }
 #endif /* __cplusplus }*/
 
-#include "camel-remote-store.h"
-#include "camel-nntp-newsrc.h"
-#include "camel-nntp-types.h"
-
-#include "config.h"
-
-#ifdef ENABLE_THREADS
-#include "e-util/e-msgport.h"
-#endif
+#include "camel/camel-remote-store.h"
+#include "camel/camel-exception.h"
+#include "camel/camel-folder.h"
 
 #define CAMEL_NNTP_STORE_TYPE     (camel_nntp_store_get_type ())
 #define CAMEL_NNTP_STORE(obj)     (CAMEL_CHECK_CAST((obj), CAMEL_NNTP_STORE_TYPE, CamelNNTPStore))
 #define CAMEL_NNTP_STORE_CLASS(k) (CAMEL_CHECK_CLASS_CAST ((k), CAMEL_NNTP_STORE_TYPE, CamelNNTPStoreClass))
 #define CAMEL_IS_NNTP_STORE(o)    (CAMEL_CHECK_TYPE((o), CAMEL_NNTP_STORE_TYPE))
 
-enum {
-	CAMEL_NNTP_OVER_FROM,
-	CAMEL_NNTP_OVER_SUBJECT,
-	CAMEL_NNTP_OVER_DATE,
-	CAMEL_NNTP_OVER_MESSAGE_ID,
-	CAMEL_NNTP_OVER_REFERENCES,
-	CAMEL_NNTP_OVER_BYTES,
-
-	CAMEL_NNTP_OVER_LAST
-};
-
-struct CamelNNTPOverField {
-	int index;
-	gboolean full; /* full in the OVER sense - the field name
-                          precedes the ':' in the XOVER list. */
-};
-
-struct CamelNNTPStore {
-	CamelRemoteStore parent_object;	
-
 #define CAMEL_NNTP_EXT_SEARCH     (1<<0)
 #define CAMEL_NNTP_EXT_SETGET     (1<<1)
 #define CAMEL_NNTP_EXT_OVER       (1<<2)
@@ -73,51 +47,38 @@ struct CamelNNTPStore {
 #define CAMEL_NNTP_EXT_LISTMOTD   (1<<5)
 #define CAMEL_NNTP_EXT_LISTSUBSCR (1<<6)
 #define CAMEL_NNTP_EXT_LISTPNAMES (1<<7)
+
+typedef struct _CamelNNTPStore CamelNNTPStore;
+typedef struct _CamelNNTPStoreClass CamelNNTPStoreClass;
+
+struct _CamelNNTPStore {
+	CamelRemoteStore parent_object;	
+
+	struct _CamelNNTPStorePrivate *priv;
+
 	guint32 extensions;
 
 	gboolean posting_allowed;
 
-	int num_overview_fields;
-	CamelNNTPOverField overview_field[ CAMEL_NNTP_OVER_LAST ];
+	struct _CamelNNTPStream *stream;
+	struct _CamelStreamMem *mem;
 
-	CamelNNTPNewsrc *newsrc;
-	CamelNNTPGroupList *group_list;
+	struct _CamelDataCache *cache;
 
-#ifdef ENABLE_THREADS
-	EMutex *command_lock;
-#endif
+	char *current_folder;
 };
 
-struct CamelNNTPStoreClass {
+struct _CamelNNTPStoreClass {
 	CamelRemoteStoreClass parent_class;
 
 };
 
-#ifdef ENABLE_THREADS
-#define CAMEL_NNTP_STORE_LOCK(f) (e_mutex_lock(((CamelNNTPStore *) f)->command_lock))
-#define CAMEL_NNTP_STORE_UNLOCK(f) (e_mutex_unlock(((CamelNNTPStore *) f)->command_lock))
-#else
-#define CAMEL_NNTP_STORE_LOCK(f)
-#define CAMEL_NNTP_STORE_UNLOCK(f)
-#endif
-
-/* public methods */
-void camel_nntp_store_open (CamelNNTPStore *store, CamelException *ex);
-void camel_nntp_store_close (CamelNNTPStore *store, gboolean expunge,
-			     CamelException *ex);
-
-void camel_nntp_store_subscribe_group (CamelStore *store, const gchar *group_name);
-void camel_nntp_store_unsubscribe_group (CamelStore *store, const gchar *group_name);
-GList *camel_nntp_store_list_subscribed_groups(CamelStore *store);
-
-gchar *camel_nntp_store_get_toplevel_dir (CamelNNTPStore *store);
-
-/* support functions */
-int camel_nntp_command (CamelNNTPStore *store, CamelException *ex, char **ret, char *fmt, ...);
-
 /* Standard Camel function */
 CamelType camel_nntp_store_get_type (void);
 
+int camel_nntp_command(CamelNNTPStore *store, char **line, const char *fmt, ...);
+int camel_nntp_store_set_folder(CamelNNTPStore *store, CamelFolder *folder, CamelFolderChangeInfo *changes, CamelException *ex);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
diff --git a/camel/providers/nntp/camel-nntp-stream.c b/camel/providers/nntp/camel-nntp-stream.c
new file mode 100644
index 0000000000..9072a8b75a
--- /dev/null
+++ b/camel/providers/nntp/camel-nntp-stream.c
@@ -0,0 +1,462 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*-
+ *
+ * Author:
+ *  Michael Zucchi <notzed@ximian.com>
+ *
+ * Copyright 1999, 2000 Ximian, Inc. (www.ximian.com)
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <errno.h>
+
+#include <string.h>
+#include <stdio.h>
+
+#include <glib.h>
+
+#include "camel-nntp-stream.h"
+
+extern int camel_verbose_debug;
+#define dd(x) (camel_verbose_debug?(x):0)
+
+static CamelObjectClass *parent_class = NULL;
+
+/* Returns the class for a CamelStream */
+#define CS_CLASS(so) CAMEL_NNTP_STREAM_CLASS(CAMEL_OBJECT_GET_CLASS(so))
+
+#define CAMEL_NNTP_STREAM_SIZE (4096)
+#define CAMEL_NNTP_STREAM_LINE (1024) /* maximum line size */
+
+static int
+stream_fill(CamelNNTPStream *is)
+{
+	int left = 0;
+
+	if (is->source) {
+		left = is->end - is->ptr;
+		memcpy(is->buf, is->ptr, left);
+		is->end = is->buf + left;
+		is->ptr = is->buf;
+		left = camel_stream_read(is->source, is->end, CAMEL_NNTP_STREAM_SIZE - (is->end - is->buf));
+		if (left > 0) {
+			is->end += left;
+			is->end[0] = '\n';
+			return is->end - is->ptr;
+		} else {
+			dd(printf("NNTP_STREAM_FILL(ERROR): '%s'\n", strerror(errno)));
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static ssize_t
+stream_read(CamelStream *stream, char *buffer, size_t n)
+{
+	CamelNNTPStream *is = (CamelNNTPStream *)stream;
+	char *o, *oe;
+	unsigned char *p, *e, c;
+	int state;
+
+	if (is->mode != CAMEL_NNTP_STREAM_DATA || n == 0)
+		return 0;
+
+	o = buffer;
+	oe = buffer + n;
+	state = is->state;
+
+	/* Need to copy/strip '.'s and whatnot */
+	p = is->ptr;
+	e = is->end;
+
+	switch(state) {
+	state_0:
+	case 0:		/* start of line, always read at least 3 chars */
+		while (e - p < 3) {
+			is->ptr = p;
+			if (stream_fill(is) == -1)
+				return -1;
+			p = is->ptr;
+			e = is->end;
+		}
+		if (p[0] == '.') {
+			if (p[1] == '\r' && p[2] == '\n') {
+				is->ptr = p+3;
+				is->mode = CAMEL_NNTP_STREAM_EOD;
+				is->state = 0;
+				dd(printf("NNTP_STREAM_READ(%d):\n%.*s\n", o-buffer, o-buffer, buffer));
+				return o-buffer;
+			}
+			p++;
+		}
+		state = 1;
+		/* FALLS THROUGH */
+	case 1:		/* looking for next sol */
+		while (o < oe) {
+			c = *p++;
+			if (c == '\n') {
+				/* end of input sentinal check */
+				if (p > e) {
+					is->ptr = e;
+					if (stream_fill(is) == -1)
+						return -1;
+					p = is->ptr;
+					e = is->end;
+				} else {
+					*o++ = '\n';
+					state = 0;
+					goto state_0;
+				}
+			} else if (c != '\r') {
+				*o++ = c;
+			}
+		}
+		break;
+	}
+
+	is->ptr = p;
+	is->state = state;
+
+	dd(printf("NNTP_STREAM_READ(%d):\n%.*s\n", o-buffer, o-buffer, buffer));
+
+	return o-buffer;
+}
+
+static ssize_t
+stream_write(CamelStream *stream, const char *buffer, size_t n)
+{
+	CamelNNTPStream *is = (CamelNNTPStream *)stream;
+
+	return camel_stream_write(is->source, buffer, n);
+}
+
+static int
+stream_close(CamelStream *stream)
+{
+	/* nop? */
+	return 0;
+}
+
+static int
+stream_flush(CamelStream *stream)
+{
+	/* nop? */
+	return 0;
+}
+
+static gboolean
+stream_eos(CamelStream *stream)
+{
+	CamelNNTPStream *is = (CamelNNTPStream *)stream;
+
+	return is->mode != CAMEL_NNTP_STREAM_DATA;
+}
+
+static int
+stream_reset(CamelStream *stream)
+{
+	/* nop?  reset literal mode? */
+	return 0;
+}
+
+static void
+camel_nntp_stream_class_init (CamelStreamClass *camel_nntp_stream_class)
+{
+	CamelStreamClass *camel_stream_class = (CamelStreamClass *)camel_nntp_stream_class;
+
+	parent_class = camel_type_get_global_classfuncs( CAMEL_OBJECT_TYPE );
+
+	/* virtual method definition */
+	camel_stream_class->read = stream_read;
+	camel_stream_class->write = stream_write;
+	camel_stream_class->close = stream_close;
+	camel_stream_class->flush = stream_flush;
+	camel_stream_class->eos = stream_eos;
+	camel_stream_class->reset = stream_reset;
+}
+
+static void
+camel_nntp_stream_init(CamelNNTPStream *is, CamelNNTPStreamClass *isclass)
+{
+	/* +1 is room for appending a 0 if we need to for a line */
+	is->ptr = is->end = is->buf = g_malloc(CAMEL_NNTP_STREAM_SIZE+1);
+	is->lineptr = is->linebuf = g_malloc(CAMEL_NNTP_STREAM_LINE+1);
+	is->lineend = is->linebuf + CAMEL_NNTP_STREAM_LINE;
+
+	/* init sentinal */
+	is->ptr[0] = '\n';
+
+	is->state = 0;
+	is->mode = CAMEL_NNTP_STREAM_LINE;
+}
+
+static void
+camel_nntp_stream_finalise(CamelNNTPStream *is)
+{
+	g_free(is->buf);
+	g_free(is->linebuf);
+	if (is->source)
+		camel_object_unref((CamelObject *)is->source);
+}
+
+CamelType
+camel_nntp_stream_get_type (void)
+{
+	static CamelType camel_nntp_stream_type = CAMEL_INVALID_TYPE;
+
+	if (camel_nntp_stream_type == CAMEL_INVALID_TYPE) {
+		camel_nntp_stream_type = camel_type_register( camel_stream_get_type(),
+							    "CamelNNTPStream",
+							    sizeof( CamelNNTPStream ),
+							    sizeof( CamelNNTPStreamClass ),
+							    (CamelObjectClassInitFunc) camel_nntp_stream_class_init,
+							    NULL,
+							    (CamelObjectInitFunc) camel_nntp_stream_init,
+							    (CamelObjectFinalizeFunc) camel_nntp_stream_finalise );
+	}
+
+	return camel_nntp_stream_type;
+}
+
+/**
+ * camel_nntp_stream_new:
+ *
+ * Returns a NULL stream.  A null stream is always at eof, and
+ * always returns success for all reads and writes.
+ *
+ * Return value: the stream
+ **/
+CamelStream *
+camel_nntp_stream_new(CamelStream *source)
+{
+	CamelNNTPStream *is;
+
+	is = (CamelNNTPStream *)camel_object_new(camel_nntp_stream_get_type ());
+	camel_object_ref((CamelObject *)source);
+	is->source = source;
+
+	return (CamelStream *)is;
+}
+
+/* Get one line from the nntp stream */
+int
+camel_nntp_stream_line(CamelNNTPStream *is, unsigned char **data, unsigned int *len)
+{
+	register unsigned char c, *p, *o, *oe;
+	int newlen, oldlen;
+	unsigned char *e;
+
+	if (is->mode == CAMEL_NNTP_STREAM_EOD) {
+		*data = is->linebuf;
+		*len = 0;
+		return 0;
+	}
+
+	o = is->linebuf;
+	oe = is->lineend - 1;
+	p = is->ptr;
+	e = is->end;
+
+	/* Data mode, convert leading '..' to '.', and stop when we reach a solitary '.' */
+	if (is->mode == CAMEL_NNTP_STREAM_DATA) {
+		/* need at least 3 chars in buffer */
+		while (e-p < 3) {
+			is->ptr = p;
+			if (stream_fill(is) == -1)
+				return -1;
+			p = is->ptr;
+			e = is->end;
+		}
+
+		/* check for isolated '.\r\n' or begging of line '.' */
+		if (p[0] == '.') {
+			if (p[1] == '\r' && p[2] == '\n') {
+				is->ptr = p+3;
+				is->mode = CAMEL_NNTP_STREAM_EOD;
+				*data = is->linebuf;
+				*len = 0;
+				is->linebuf[0] = 0;
+
+				dd(printf("NNTP_STREAM_LINE(END)\n"));
+
+				return 0;
+			}
+			p++;
+		}
+	}
+
+	while (1) {
+		while (o < oe) {
+			c = *p++;
+			if (c == '\n') {
+				/* sentinal? */
+				if (p> e) {
+					is->ptr = e;
+					if (stream_fill(is) == -1)
+						return -1;
+					p = is->ptr;
+					e = is->end;
+				} else {
+					is->ptr = p;
+					*data = is->linebuf;
+					*len = o - is->linebuf;
+					*o = 0;
+
+					dd(printf("NNTP_STREAM_LINE(%d): '%s'\n", *len, *data));
+
+					return 1;
+				}
+			} else if (c != '\r') {
+				*o++ = c;
+			}
+		}
+
+		/* limit this for bad server data? */
+		oldlen = o - is->linebuf;
+		newlen = (is->lineend - is->linebuf) * 3 / 2;
+		is->lineptr = is->linebuf = g_realloc(is->linebuf, newlen);
+		is->lineend = is->linebuf + newlen;
+		oe = is->lineend - 1;
+		o = is->linebuf + oldlen;
+	}
+
+	return -1;
+}
+
+/* returns -1 on error, 0 if last lot of data, >0 if more remaining */
+int camel_nntp_stream_gets(CamelNNTPStream *is, unsigned char **start, unsigned int *len)
+{
+	int max;
+	unsigned char *end;
+
+	*len = 0;
+
+	max = is->end - is->ptr;
+	if (max == 0) {
+		max = stream_fill(is);
+		if (max <= 0)
+			return max;
+	}
+
+	*start = is->ptr;
+	end = memchr(is->ptr, '\n', max);
+	if (end)
+		max = (end - is->ptr) + 1;
+	*start = is->ptr;
+	*len = max;
+	is->ptr += max;
+
+	dd(printf("NNTP_STREAM_GETS(%s,%d): '%.*s'\n", end==NULL?"more":"last", *len, (int)*len, *start));
+
+	return end == NULL?1:0;
+}
+
+void camel_nntp_stream_set_mode(CamelNNTPStream *is, camel_nntp_stream_mode_t mode)
+{
+	is->mode = mode;
+}
+
+/* returns -1 on erorr, 0 if last data, >0 if more data left */
+int camel_nntp_stream_getd(CamelNNTPStream *is, unsigned char **start, unsigned int *len)
+{
+	unsigned char *p, *e, *s;
+	int state;
+
+	*len = 0;
+
+	if (is->mode == CAMEL_NNTP_STREAM_EOD)
+		return 0;
+
+	if (is->mode == CAMEL_NNTP_STREAM_LINE) {
+		g_warning("nntp_stream reading data in line mode\n");
+		return 0;
+	}
+
+	state = is->state;
+	p = is->ptr;
+	e = is->end;
+
+	while (e - p < 3) {
+		is->ptr = p;
+		if (stream_fill(is) == -1)
+			return -1;
+		p = is->ptr;
+		e = is->end;
+	}
+
+	s = p;
+
+	do {
+		switch(state) {
+		case 0:
+			/* check leading '.', ... */
+			if (p[0] == '.') {
+				if (p[1] == '\r' && p[2] == '\n') {
+					is->ptr = p+3;
+					*len = p-s;
+					*start = s;
+					is->mode = CAMEL_NNTP_STREAM_EOD;
+					is->state = 0;
+
+					dd(printf("NNTP_STREAM_GETD(%s,%d): '%.*s'\n", "last", *len, (int)*len, *start));
+
+					return 0;
+				}
+
+				/* If at start, just skip '.', else return data upto '.' but skip it */
+				if (p == s) {
+					s++;
+					p++;
+				} else {
+					is->ptr = p+1;
+					*len = p-s;
+					*start = s;
+					is->state = 1;
+
+					dd(printf("NNTP_STREAM_GETD(%s,%d): '%.*s'\n", "more", *len, (int)*len, *start));
+
+					return 1;
+				}
+			}
+			state = 1;
+		case 1:
+			/* Scan for sentinal */
+			while ((*p++)!='\n')
+				;
+
+			if (p > e) {
+				p = e;
+			} else {
+				state = 0;
+			}
+			break;
+		}
+	} while ((e-p) >= 3);
+
+	is->state = state;
+	is->ptr = p;
+	*len = p-s;
+	*start = s;
+
+	dd(printf("NNTP_STREAM_GETD(%s,%d): '%.*s'\n", "more", *len, (int)*len, *start));
+
+	return 1;
+}
diff --git a/camel/providers/nntp/camel-nntp-stream.h b/camel/providers/nntp/camel-nntp-stream.h
new file mode 100644
index 0000000000..e7b732beff
--- /dev/null
+++ b/camel/providers/nntp/camel-nntp-stream.h
@@ -0,0 +1,66 @@
+/*
+ * Copyright (C) 2001 Ximian Inc.
+ *
+ * Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _CAMEL_NNTP_STREAM_H
+#define _CAMEL_NNTP_STREAM_H
+
+#include <camel/camel-stream.h>
+
+#define CAMEL_NNTP_STREAM(obj)         CAMEL_CHECK_CAST (obj, camel_nntp_stream_get_type (), CamelNNTPStream)
+#define CAMEL_NNTP_STREAM_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_nntp_stream_get_type (), CamelNNTPStreamClass)
+#define CAMEL_IS_NNTP_STREAM(obj)      CAMEL_CHECK_TYPE (obj, camel_nntp_stream_get_type ())
+
+typedef struct _CamelNNTPStreamClass CamelNNTPStreamClass;
+typedef struct _CamelNNTPStream CamelNNTPStream;
+
+typedef enum {
+	CAMEL_NNTP_STREAM_LINE,
+	CAMEL_NNTP_STREAM_DATA,
+	CAMEL_NNTP_STREAM_EOD,	/* end of data, acts as if end of stream */
+} camel_nntp_stream_mode_t;
+
+struct _CamelNNTPStream {
+	CamelStream parent;
+
+	CamelStream *source;
+
+	camel_nntp_stream_mode_t mode;
+	int state;
+
+	unsigned char *buf, *ptr, *end;
+	unsigned char *linebuf, *lineptr, *lineend;
+};
+
+struct _CamelNNTPStreamClass {
+	CamelStreamClass parent_class;
+};
+
+guint		 camel_nntp_stream_get_type	(void);
+
+CamelStream     *camel_nntp_stream_new		(CamelStream *source);
+
+
+void		 camel_nntp_stream_set_mode     (CamelNNTPStream *is, camel_nntp_stream_mode_t mode);
+
+int              camel_nntp_stream_line		(CamelNNTPStream *is, unsigned char **data, unsigned int *len);
+int 		 camel_nntp_stream_gets		(CamelNNTPStream *is, unsigned char **start, unsigned int *len);
+int 		 camel_nntp_stream_getd		(CamelNNTPStream *is, unsigned char **start, unsigned int *len);
+
+#endif /* ! _CAMEL_NNTP_STREAM_H */
diff --git a/camel/providers/nntp/camel-nntp-summary.c b/camel/providers/nntp/camel-nntp-summary.c
new file mode 100644
index 0000000000..9f6ff5f095
--- /dev/null
+++ b/camel/providers/nntp/camel-nntp-summary.c
@@ -0,0 +1,581 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8; fill-column: 160 -*- */
+/*
+ *  Copyright (C) 2000 Ximian Inc.
+ *
+ *  Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <ctype.h>
+#include <sys/stat.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "camel/camel-file-utils.h"
+#include "camel/camel-mime-message.h"
+#include "camel/camel-stream-null.h"
+#include "camel/camel-operation.h"
+#include "camel/camel-data-cache.h"
+
+#include "camel-nntp-summary.h"
+#include "camel-nntp-folder.h"
+#include "camel-nntp-store.h"
+#include "camel-nntp-stream.h"
+
+#define w(x)
+#define io(x)
+#define d(x) /*(printf("%s(%d): ", __FILE__, __LINE__),(x))*/
+extern int camel_verbose_debug;
+#define dd(x) (camel_verbose_debug?(x):0)
+
+#define CAMEL_NNTP_SUMMARY_VERSION (0x200)
+
+static int xover_setup(CamelNNTPSummary *cns, CamelException *ex);
+static int add_range_xover(CamelNNTPSummary *cns, unsigned int high, unsigned int low, CamelFolderChangeInfo *changes, CamelException *ex);
+static int add_range_head(CamelNNTPSummary *cns, unsigned int high, unsigned int low, CamelFolderChangeInfo *changes, CamelException *ex);
+
+enum _xover_t {
+	XOVER_STRING = 0,
+	XOVER_MSGID,
+	XOVER_SIZE,
+};
+
+struct _xover_header {
+	struct _xover_header *next;
+
+	const char *name;
+	unsigned int skip:8;
+	enum _xover_t type:8;
+};
+
+struct _CamelNNTPSummaryPrivate {
+	char *uid;
+
+	struct _xover_header *xover; /* xoverview format */
+	int xover_setup;
+};
+
+#define _PRIVATE(o) (((CamelNNTPSummary *)(o))->priv)
+
+static CamelMessageInfo * message_info_new (CamelFolderSummary *, struct _header_raw *);
+static int summary_header_load(CamelFolderSummary *, FILE *);
+static int summary_header_save(CamelFolderSummary *, FILE *);
+
+static void camel_nntp_summary_class_init (CamelNNTPSummaryClass *klass);
+static void camel_nntp_summary_init       (CamelNNTPSummary *obj);
+static void camel_nntp_summary_finalise   (CamelObject *obj);
+static CamelFolderSummaryClass *camel_nntp_summary_parent;
+
+CamelType
+camel_nntp_summary_get_type(void)
+{
+	static CamelType type = CAMEL_INVALID_TYPE;
+	
+	if (type == CAMEL_INVALID_TYPE) {
+		type = camel_type_register(camel_folder_summary_get_type(), "CamelNNTPSummary",
+					   sizeof (CamelNNTPSummary),
+					   sizeof (CamelNNTPSummaryClass),
+					   (CamelObjectClassInitFunc) camel_nntp_summary_class_init,
+					   NULL,
+					   (CamelObjectInitFunc) camel_nntp_summary_init,
+					   (CamelObjectFinalizeFunc) camel_nntp_summary_finalise);
+	}
+	
+	return type;
+}
+
+static void
+camel_nntp_summary_class_init(CamelNNTPSummaryClass *klass)
+{
+	CamelFolderSummaryClass *sklass = (CamelFolderSummaryClass *) klass;
+	
+	camel_nntp_summary_parent = CAMEL_FOLDER_SUMMARY_CLASS(camel_type_get_global_classfuncs(camel_folder_summary_get_type()));
+
+	sklass->message_info_new  = message_info_new;
+	sklass->summary_header_load = summary_header_load;
+	sklass->summary_header_save = summary_header_save;
+}
+
+static void
+camel_nntp_summary_init(CamelNNTPSummary *obj)
+{
+	struct _CamelNNTPSummaryPrivate *p;
+	struct _CamelFolderSummary *s = (CamelFolderSummary *)obj;
+
+	p = _PRIVATE(obj) = g_malloc0(sizeof(*p));
+
+	/* subclasses need to set the right instance data sizes */
+	s->message_info_size = sizeof(CamelMessageInfo);
+	s->content_info_size = sizeof(CamelMessageContentInfo);
+
+	/* and a unique file version */
+	s->version += CAMEL_NNTP_SUMMARY_VERSION;
+}
+
+static void
+camel_nntp_summary_finalise(CamelObject *obj)
+{
+	CamelNNTPSummary *cns = CAMEL_NNTP_SUMMARY(obj);
+	struct _xover_header *xover, *xn;
+
+	xover = cns->priv->xover;
+	while (xover) {
+		xn = xover->next;
+		g_free(xover);
+		xover = xn;
+	}
+
+	g_free(cns->priv);
+}
+
+CamelNNTPSummary *
+camel_nntp_summary_new(CamelNNTPFolder *folder)
+{
+	CamelNNTPSummary *cns = (CamelNNTPSummary *)camel_object_new(camel_nntp_summary_get_type());
+	char *path;
+
+	cns->folder = folder;
+	path = g_strdup_printf("%s.ev-summary", folder->storage_path);
+	camel_folder_summary_set_filename((CamelFolderSummary *)cns, path);
+	g_free(path);
+
+	camel_folder_summary_set_build_content((CamelFolderSummary *)cns, FALSE);
+
+	return cns;
+}
+
+static CamelMessageInfo *
+message_info_new(CamelFolderSummary *s, struct _header_raw *h)
+{
+	CamelMessageInfo *mi;
+	CamelNNTPSummary *cns = (CamelNNTPSummary *)s;
+
+	/* error to call without this setup */
+	if (cns->priv->uid == NULL)
+		return NULL;
+
+	/* we shouldn't be here if we already have this uid */
+	g_assert(camel_folder_summary_uid(s, cns->priv->uid) == NULL);
+
+	mi = ((CamelFolderSummaryClass *)camel_nntp_summary_parent)->message_info_new(s, h);
+	if (mi) {
+		camel_message_info_set_uid(mi, cns->priv->uid);
+		cns->priv->uid = NULL;
+	}
+	
+	return mi;
+}
+
+static int summary_header_load(CamelFolderSummary *s, FILE *in)
+{
+	CamelNNTPSummary *cns = CAMEL_NNTP_SUMMARY(s);
+
+	if (((CamelFolderSummaryClass *)camel_nntp_summary_parent)->summary_header_load(s, in) == -1
+	    || camel_file_util_decode_fixed_int32(in, &cns->high) == -1
+	    || camel_file_util_decode_fixed_int32(in, &cns->low) == -1)
+		return -1;
+
+	return 0;
+}
+
+static int summary_header_save(CamelFolderSummary *s, FILE *out)
+{
+	CamelNNTPSummary *cns = CAMEL_NNTP_SUMMARY(s);
+
+	if (((CamelFolderSummaryClass *)camel_nntp_summary_parent)->summary_header_save(s, out) == -1
+	    || camel_file_util_encode_fixed_int32(out, cns->high) == -1
+	    || camel_file_util_encode_fixed_int32(out, cns->low) == -1)
+		return -1;
+
+	return 0;
+}
+
+/* Assumes we have the stream */
+int camel_nntp_summary_check(CamelNNTPSummary *cns, CamelFolderChangeInfo *changes, CamelException *ex)
+{
+	CamelNNTPStore *store;
+	CamelFolder *folder;
+	CamelFolderSummary *s;
+	int ret, i;
+	char *line;
+	unsigned int n, f, l;
+	int count;
+
+	if (xover_setup(cns, ex) == -1)
+		return -1;
+
+	folder = (CamelFolder *)cns->folder;
+	store = (CamelNNTPStore *)folder->parent_store;
+	s = (CamelFolderSummary *)cns;
+
+	ret = camel_nntp_command(store, &line, "group %s", folder->full_name);
+	if (ret == 411) {
+		camel_exception_setv(ex, CAMEL_EXCEPTION_FOLDER_INVALID,
+				     _("No such folder: %s"), line);
+		return -1;
+	} else if (ret != 211) {
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SERVICE_UNAVAILABLE,
+				     _("Could not get group: %s"), line);
+		return -1;
+	}
+
+	line +=3;
+	n = strtoul(line, &line, 10);
+	f = strtoul(line, &line, 10);
+	l = strtoul(line, &line, 10);
+
+	dd(printf("nntp_summary: got last '%u' first '%u'\n"
+		  "nntp_summary: high '%u' low '%u'\n", l, f, cns->high, cns->low));
+
+	if (cns->low == f && cns->high == l) {
+		dd(printf("nntp_summary: no work to do!\n"));
+		return 0;
+	}
+
+	/* Need to work out what to do with our messages */
+
+	/* Check for messages no longer on the server */
+	if (cns->low != f) {
+		count = camel_folder_summary_count(s);
+		for (i = 0; i < count; i++) {
+			CamelMessageInfo *mi = camel_folder_summary_index(s, i);
+
+			if (mi) {
+				const char *uid = camel_message_info_uid(mi);
+				const char *msgid;
+
+				n = strtoul(uid, NULL, 10);
+				if (n < f || n > l) {
+					dd(printf("nntp_summary: %u is lower/higher than lowest/highest article, removed\n", n));
+					/* Since we use a global cache this could prematurely remove
+					   a cached message that might be in another folder - not that important as
+					   it is a true cache */
+					msgid = strchr(uid, ',');
+					if (msgid)
+						camel_data_cache_remove(store->cache, "cache", msgid+1, NULL);
+					camel_folder_change_info_remove_uid(changes, uid);
+					camel_folder_summary_remove(s, mi);
+					count--;
+					i--;
+				}
+				
+				camel_folder_summary_info_free(s, mi);
+			}
+		}
+		cns->low = f;
+	}
+
+	if (cns->high < l) {
+		if (cns->high < f)
+			cns->high = f-1;
+
+		if (cns->priv->xover) {
+			ret = add_range_xover(cns, l, cns->high+1, changes, ex);
+		} else {
+			ret = add_range_head(cns, l, cns->high+1, changes, ex);
+		}
+	}
+
+	camel_folder_summary_touch(s);
+
+	return ret;
+}
+
+static struct {
+	const char *name;
+	int type;
+} headers[] = {
+	{ "subject", 0 },
+	{ "from", 0 },
+	{ "date", 0 },
+	{ "message-id", 1 },
+	{ "references", 0 },
+	{ "bytes", 2 },
+};
+
+static int
+xover_setup(CamelNNTPSummary *cns, CamelException *ex)
+{
+	CamelNNTPStore *store;
+	CamelFolder *folder;
+	CamelFolderSummary *s;
+	int ret, i;
+	char *line;
+	unsigned int len;
+	unsigned char c, *p;
+	struct _xover_header *xover, *last;
+
+	if (cns->priv->xover_setup)
+		return 0;
+
+	/* manual override */
+	if (getenv("CAMEL_NNTP_DISABLE_XOVER") != NULL) {
+		cns->priv->xover_setup = TRUE;
+		return 0;
+	}
+
+	folder = (CamelFolder *)cns->folder;
+	store = (CamelNNTPStore *)folder->parent_store;
+	s = (CamelFolderSummary *)cns;
+
+	ret = camel_nntp_command(store, &line, "list overview.fmt");
+	if (ret == -1) {
+		camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM,
+				     _("NNTP Command failed: %s"), strerror(errno));
+		return -1;
+	}
+
+	cns->priv->xover_setup = TRUE;
+
+	/* unsupported command? */
+	if (ret != 215)
+		return 0;
+
+	last = (struct _xover_header *)&cns->priv->xover;
+
+	/* supported command */
+	while ((ret = camel_nntp_stream_line(store->stream, (unsigned char **)&line, &len)) > 0) {
+		p = line;
+		xover = g_malloc0(sizeof(*xover));
+		last->next = xover;
+		last = xover;
+		while ((c = *p++)) {
+			if (c == ':') {
+				p[-1] = 0;
+				for (i=0;i<sizeof(headers)/sizeof(headers[0]);i++) {
+					if (strcmp(line, headers[i].name) == 0) {
+						xover->name = headers[i].name;
+						if (strncmp(p, "full", 4) == 0)
+							xover->skip = strlen(xover->name)+1;
+						else
+							xover->skip = 0;
+						xover->type = headers[i].type;
+						break;
+					}
+				}
+				break;
+			} else {
+				p[-1] = tolower(c);
+			}
+		}
+	}
+
+	return ret;
+}
+
+static int
+add_range_xover(CamelNNTPSummary *cns, unsigned int high, unsigned int low, CamelFolderChangeInfo *changes, CamelException *ex)
+{
+	CamelNNTPStore *store;
+	CamelFolder *folder;
+	CamelFolderSummary *s;
+	CamelMessageInfo *mi;
+	struct _header_raw *headers = NULL;
+	char *line, *tab;
+	int len, ret;
+	unsigned int n, count, total, size;
+	struct _xover_header *xover;
+	time_t last, now;
+
+	folder = (CamelFolder *)cns->folder;
+	store = (CamelNNTPStore *)folder->parent_store;
+	s = (CamelFolderSummary *)cns;
+
+	camel_operation_start(NULL, _("%s: Scanning new messages"), ((CamelService *)store)->url->host);
+
+	ret = camel_nntp_command(store, &line, "xover %r", low, high);
+	if (ret != 224) {
+		camel_operation_end(NULL);
+		return -1;
+	}
+
+	last = time(0);
+	count = 0;
+	total = high-low+1;
+	while ((ret = camel_nntp_stream_line(store->stream, (unsigned char **)&line, &len)) > 0) {
+		camel_operation_progress(NULL, (count * 100) / total);
+		count++;
+		n = strtoul(line, &tab, 10);
+		if (*tab != '\t')
+			continue;
+		tab++;
+		xover = cns->priv->xover;
+		size = 0;
+		for (;tab[0] && xover;xover = xover->next) {
+			line = tab;
+			tab = strchr(line, '\t');
+			if (tab)
+				*tab++ = 0;
+			else
+				tab = line+strlen(line);
+
+			/* do we care about this column? */
+			if (xover->name) {
+				line += xover->skip;
+				if (line < tab) {
+					header_raw_append(&headers, xover->name, line, -1);
+					switch(xover->type) {
+					case XOVER_STRING:
+						break;
+					case XOVER_MSGID:
+						cns->priv->uid = g_strdup_printf("%u,%s", n, line);
+						break;
+					case XOVER_SIZE:
+						size = strtoul(line, NULL, 10);
+						break;
+					}
+				}
+			}
+		}
+
+		/* truncated line? ignore? */
+		if (xover == NULL) {
+			mi = camel_folder_summary_uid(s, cns->priv->uid);
+			if (mi == NULL) {
+				mi = camel_folder_summary_add_from_header(s, headers);
+				if (mi) {
+					mi->size = size;
+					cns->high = n;
+					camel_folder_change_info_add_uid(changes, camel_message_info_uid(mi));
+				}
+			} else {
+				camel_folder_summary_info_free(s, mi);
+			}
+		}
+
+		if (cns->priv->uid) {
+			g_free(cns->priv->uid);
+			cns->priv->uid = NULL;
+		}
+
+		header_raw_clear(&headers);
+
+		now = time(0);
+		if (last + 2 < now) {
+			camel_object_trigger_event((CamelObject *)folder, "folder_changed", changes);
+			camel_folder_change_info_clear(changes);
+			last = now;
+		}
+	}
+
+	camel_operation_end(NULL);
+
+	return ret;
+}
+
+static int
+add_range_head(CamelNNTPSummary *cns, unsigned int high, unsigned int low, CamelFolderChangeInfo *changes, CamelException *ex)
+{
+	CamelNNTPStore *store;
+	CamelFolder *folder;
+	CamelFolderSummary *s;
+	int i, ret = -1;
+	char *line, *msgid;
+	unsigned int n, count, total;
+	CamelMessageInfo *mi;
+	CamelMimeParser *mp;
+	time_t now, last;
+
+	folder = (CamelFolder *)cns->folder;
+	store = (CamelNNTPStore *)folder->parent_store;
+	s = (CamelFolderSummary *)cns;
+
+	mp = camel_mime_parser_new();
+
+	camel_operation_start(NULL, _("%s: Scanning new messages"), ((CamelService *)store)->url->host);
+
+	last = time(0);
+	count = 0;
+	total = high-low+1;
+	for (i=low;i<high+1;i++) {
+		camel_operation_progress(NULL, (count * 100) / total);
+		count++;
+		ret = camel_nntp_command(store, &line, "head %u", i);
+		/* unknown article, ignore */
+		if (ret == 423)
+			continue;
+		else if (ret == -1)
+			goto error;
+		else if (ret != 221) {
+			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Unknown server response: %s"), line);
+			goto ioerror;
+		}
+		line += 3;
+		n = strtoul(line, &line, 10);
+		if (n != i)
+			g_warning("retrieved message '%d' when i expected '%d'?\n", n, i);
+
+		if ((msgid = strchr(line, '<')) && (line = strchr(msgid+1, '>'))){
+			line[1] = 0;
+			cns->priv->uid = g_strdup_printf("%u,%s\n", n, msgid);
+			mi = camel_folder_summary_uid(s, cns->priv->uid);
+			if (mi == NULL) {
+				if (camel_mime_parser_init_with_stream(mp, (CamelStream *)store->stream) == -1)
+					goto error;
+				mi = camel_folder_summary_add_from_parser(s, mp);
+				while (camel_mime_parser_step(mp, NULL, NULL) != HSCAN_EOF)
+					;
+				if (mi == NULL) {
+					goto error;
+				}
+				cns->high = i;
+				camel_folder_change_info_add_uid(changes, camel_message_info_uid(mi));
+			} else {
+				/* already have, ignore */
+				camel_folder_summary_info_free(s, mi);
+			}
+			if (cns->priv->uid) {
+				g_free(cns->priv->uid);
+				cns->priv->uid = NULL;
+			}
+		}
+
+		now = time(0);
+		if (last + 2 < now) {
+			camel_object_trigger_event((CamelObject *)folder, "folder_changed", changes);
+			camel_folder_change_info_clear(changes);
+			last = now;
+		}
+	}
+
+	ret = 0;
+error:
+
+	if (ret == -1) {
+		if (errno == EINTR)
+			camel_exception_setv(ex, CAMEL_EXCEPTION_USER_CANCEL, _("Use cancel"));
+		else
+			camel_exception_setv(ex, CAMEL_EXCEPTION_SYSTEM, _("Operation failed: %s"), strerror(errno));
+	}
+ioerror:
+
+	if (cns->priv->uid) {
+		g_free(cns->priv->uid);
+		cns->priv->uid = NULL;
+	}
+	camel_object_unref((CamelObject *)mp);
+
+	camel_operation_end(NULL);
+
+	return ret;
+}
diff --git a/camel/providers/nntp/camel-nntp-summary.h b/camel/providers/nntp/camel-nntp-summary.h
new file mode 100644
index 0000000000..b82283452e
--- /dev/null
+++ b/camel/providers/nntp/camel-nntp-summary.h
@@ -0,0 +1,67 @@
+/*
+ *  Copyright (C) 2000 Ximian Inc.
+ *
+ *  Authors: Michael Zucchi <notzed@ximian.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of version 2 of the GNU General Public
+ * License as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _CAMEL_NNTP_SUMMARY_H
+#define _CAMEL_NNTP_SUMMARY_H
+
+#include <camel/camel-folder-summary.h>
+#include <camel/camel-folder.h>
+#include <camel/camel-exception.h>
+#include <libibex/ibex.h>
+
+#define CAMEL_NNTP_SUMMARY(obj)         CAMEL_CHECK_CAST (obj, camel_nntp_summary_get_type (), CamelNNTPSummary)
+#define CAMEL_NNTP_SUMMARY_CLASS(klass) CAMEL_CHECK_CLASS_CAST (klass, camel_nntp_summary_get_type (), CamelNNTPSummaryClass)
+#define CAMEL_IS_LOCAL_SUMMARY(obj)      CAMEL_CHECK_TYPE (obj, camel_nntp_summary_get_type ())
+
+typedef struct _CamelNNTPSummary      CamelNNTPSummary;
+typedef struct _CamelNNTPSummaryClass CamelNNTPSummaryClass;
+
+struct _CamelNNTPSummary {
+	CamelFolderSummary parent;
+
+	struct _CamelNNTPSummaryPrivate *priv;
+
+	struct _CamelNNTPFolder *folder;
+
+	guint32 high, low;
+};
+
+struct _CamelNNTPSummaryClass {
+	CamelFolderSummaryClass parent_class;
+};
+
+guint	camel_nntp_summary_get_type	(void);
+CamelNNTPSummary *camel_nntp_summary_new(struct _CamelNNTPFolder *folder);
+
+int camel_nntp_summary_check(CamelNNTPSummary *cns, CamelFolderChangeInfo *, CamelException *ex);
+
+#if 0
+/* load/check the summary */
+int camel_nntp_summary_load(CamelNNTPSummary *cls, CamelException *ex);
+/* check for new/removed messages */
+int camel_nntp_summary_check(CamelNNTPSummary *cls, CamelFolderChangeInfo *, CamelException *ex);
+/* perform a folder sync or expunge, if needed */
+int camel_nntp_summary_sync(CamelNNTPSummary *cls, gboolean expunge, CamelFolderChangeInfo *, CamelException *ex);
+/* add a new message to the summary */
+CamelMessageInfo *camel_nntp_summary_add(CamelNNTPSummary *cls, CamelMimeMessage *msg, const CamelMessageInfo *info, CamelFolderChangeInfo *, CamelException *ex);
+#endif
+
+#endif /* ! _CAMEL_NNTP_SUMMARY_H */
+