Apply upstream patches for indexing.

Reviewed by:	ler
Obtained from:	dovecot GH

2 files changed, 293 insertions, 1 deletions

diff --git a/mail/dovecot/Makefile b/mail/dovecot/Makefile
index ffa8b3d1fcb7..844e12415135 100644
--- a/mail/dovecot/Makefile
+++ b/mail/dovecot/Makefile
@@ -13,7 +13,7 @@
 
 PORTNAME=	dovecot
 PORTVERSION=	2.2.31
-PORTREVISION=	1
+PORTREVISION=	2
 CATEGORIES=	mail ipv6
 MASTER_SITES=	https://www.dovecot.org/releases/2.2/
 
diff --git a/mail/dovecot/files/patch-UPSTREAM-indexing b/mail/dovecot/files/patch-UPSTREAM-indexing
new file mode 100644
index 000000000000..6e072d8dbb67
--- /dev/null
+++ b/mail/dovecot/files/patch-UPSTREAM-indexing
@@ -0,0 +1,292 @@
+From 892d85b7c6c36eb3ae6d0a78e4028a5b88b9f3a5 Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen@dovecot.fi>
+Date: Sat, 5 Aug 2017 14:07:58 +0900
+Subject: [PATCH] lib-index: Fix modseq tracking for
+ MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ
+
+This is used to increase modseq for mails when their private flags change.
+Use an already existing MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL() that
+does this properly.
+
+(This change preserves another bug, which is fixed in the next commit.)
+---
+ src/lib-index/mail-transaction-log-file.c | 7 +------
+ 1 file changed, 1 insertion(+), 6 deletions(-)
+
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index ab412d6ca..520541b8a 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1013,9 +1013,6 @@ static bool
+ flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+ 			       unsigned int count, unsigned int version)
+ {
+-	const uint8_t internal_flags =
+-		MAIL_INDEX_MAIL_FLAG_BACKEND | MAIL_INDEX_MAIL_FLAG_DIRTY;
+-
+ 	/* Hide internal flags from modseqs if the log file's version
+ 	   is new enough. This allows upgrading without the modseqs suddenly
+ 	   shrinking. */
+@@ -1023,9 +1020,7 @@ flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+ 		return TRUE;
+ 
+ 	for (unsigned int i = 0; i < count; i++) {
+-		uint8_t changed_flags = u->add_flags | u->remove_flags;
+-
+-		if ((changed_flags & ~internal_flags) != 0)
++		if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u))
+ 			return TRUE;
+ 	}
+ 	return FALSE;
+From ace341ac567376f37ded043c4c0f2c46b9aaecb1 Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen@dovecot.fi>
+Date: Sat, 5 Aug 2017 14:11:17 +0900
+Subject: [PATCH] lib-index: Fix modseq tracking with multiple flag updates
+
+The earlier code was checking only the first flag record update. If the
+first one had only internal flag changes but (some of) the rest didn't,
+the modseq wasn't counted correctly. This was probably pretty rare.
+---
+ src/lib-index/mail-transaction-log-file.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index 520541b8a..094cbb2d5 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1020,7 +1020,7 @@ flag_updates_have_non_internal(const struct mail_transaction_flag_update *u,
+ 		return TRUE;
+ 
+ 	for (unsigned int i = 0; i < count; i++) {
+-		if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u))
++		if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(&u[i]))
+ 			return TRUE;
+ 	}
+ 	return FALSE;
+From 7e78f1cfc10cc0540134b5507e08524a0fdd5c93 Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen@dovecot.fi>
+Date: Tue, 8 Aug 2017 16:54:42 +0300
+Subject: [PATCH] lib-index: Code cleanup - move code to
+ get_modseq_next_offset_at()
+
+In preparation for making the next commit smaller.
+---
+ src/lib-index/mail-transaction-log-file.c | 80 ++++++++++++++++++-------------
+ 1 file changed, 47 insertions(+), 33 deletions(-)
+
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index 094cbb2d5..b0512e027 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1255,15 +1255,57 @@ int mail_transaction_log_file_get_highest_modseq_at(
+ 	return 0;
+ }
+ 
++static int
++get_modseq_next_offset_at(struct mail_transaction_log_file *file,
++			  uint64_t modseq,
++			  uoff_t *cur_offset, uint64_t *cur_modseq,
++			  uoff_t *next_offset_r)
++{
++	const struct mail_transaction_header *hdr;
++	const char *reason;
++	int ret;
++
++	/* make sure we've read until end of file. this is especially important
++	   with non-head logs which might only have been opened without being
++	   synced. */
++	ret = mail_transaction_log_file_map(file, *cur_offset, (uoff_t)-1, &reason);
++	if (ret <= 0) {
++		mail_index_set_error(file->log->index,
++			"Failed to map transaction log %s for getting offset "
++			"for modseq=%llu with start_offset=%"PRIuUOFF_T": %s",
++			file->filepath, (unsigned long long)modseq,
++			*cur_offset, reason);
++		return -1;
++	}
++
++	/* check sync_highest_modseq again in case sync_offset was updated */
++	if (modseq >= file->sync_highest_modseq) {
++		*next_offset_r = file->sync_offset;
++		return 0;
++	}
++
++	i_assert(*cur_offset >= file->buffer_offset);
++	while (*cur_offset < file->sync_offset) {
++		if (log_get_synced_record(file, cur_offset, &hdr, &reason) < 0) {
++			mail_index_set_error(file->log->index,
++				"%s: %s", file->filepath, reason);
++			return -1;
++		}
++		mail_transaction_update_modseq(hdr, hdr + 1, cur_modseq,
++			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
++		if (*cur_modseq >= modseq)
++			break;
++	}
++	return 1;
++}
++
+ int mail_transaction_log_file_get_modseq_next_offset(
+ 		struct mail_transaction_log_file *file,
+ 		uint64_t modseq, uoff_t *next_offset_r)
+ {
+-	const struct mail_transaction_header *hdr;
+ 	struct modseq_cache *cache;
+ 	uoff_t cur_offset;
+ 	uint64_t cur_modseq;
+-	const char *reason;
+ 	int ret;
+ 
+ 	if (modseq == file->sync_highest_modseq) {
+@@ -1290,37 +1332,9 @@ int mail_transaction_log_file_get_modseq_next_offset(
+ 		cur_modseq = cache->highest_modseq;
+ 	}
+ 
+-	/* make sure we've read until end of file. this is especially important
+-	   with non-head logs which might only have been opened without being
+-	   synced. */
+-	ret = mail_transaction_log_file_map(file, cur_offset, (uoff_t)-1, &reason);
+-	if (ret <= 0) {
+-		mail_index_set_error(file->log->index,
+-			"Failed to map transaction log %s for getting offset "
+-			"for modseq=%llu with start_offset=%"PRIuUOFF_T": %s",
+-			file->filepath, (unsigned long long)modseq,
+-			cur_offset, reason);
+-		return -1;
+-	}
+-
+-	/* check sync_highest_modseq again in case sync_offset was updated */
+-	if (modseq >= file->sync_highest_modseq) {
+-		*next_offset_r = file->sync_offset;
+-		return 0;
+-	}
+-
+-	i_assert(cur_offset >= file->buffer_offset);
+-	while (cur_offset < file->sync_offset) {
+-		if (log_get_synced_record(file, &cur_offset, &hdr, &reason) < 0) {
+-			mail_index_set_error(file->log->index,
+-				"%s: %s", file->filepath, reason);
+-			return -1;
+-		}
+-		mail_transaction_update_modseq(hdr, hdr + 1, &cur_modseq,
+-			MAIL_TRANSACTION_LOG_HDR_VERSION(&file->hdr));
+-		if (cur_modseq >= modseq)
+-			break;
+-	}
++	if ((ret = get_modseq_next_offset_at(file, modseq, &cur_offset,
++					     &cur_modseq, next_offset_r)) <= 0)
++		return ret;
+ 	if (cur_offset == file->sync_offset) {
+ 		/* if we got to sync_offset, cur_modseq should be
+ 		   sync_highest_modseq */
+From f39bde1e1fb00905b71a4f2a793faea4b915e27b Mon Sep 17 00:00:00 2001
+From: Timo Sirainen <timo.sirainen@dovecot.fi>
+Date: Tue, 8 Aug 2017 16:56:02 +0300
+Subject: [PATCH] lib-index: Fix wrong mail_index_modseq_header automatically
+
+It happens only on the next sync, although that isn't actually guaranteed to
+happen. Still, it happens almost always so this should be good enough.
+---
+ src/lib-index/mail-index-sync.c                |  2 +-
+ src/lib-index/mail-transaction-log-file.c      | 26 ++++++++++++++++-----
+ src/lib-index/mail-transaction-log-private.h   |  1 +
+ src/lib-index/mail-transaction-log.c           |  3 +++
+ src/lib-index/test-mail-transaction-log-file.c | 32 ++++++++++++++++++++++++++
+ 5 files changed, 57 insertions(+), 7 deletions(-)
+
+diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
+index 24de5e119..f4d30eb03 100644
+--- src/lib-index/mail-index-sync.c
++++ src/lib-index/mail-index-sync.c
+@@ -361,7 +361,7 @@ mail_index_sync_begin_init(struct mail_index *index,
+ 	}
+ 
+ 	if (!mail_index_need_sync(index, flags, log_file_seq, log_file_offset) &&
+-	    !index->index_deleted) {
++	    !index->index_deleted && !index->need_recreate) {
+ 		if (locked)
+ 			mail_transaction_log_sync_unlock(index->log, "syncing determined unnecessary");
+ 		return 0;
+diff --git a/src/lib-index/mail-transaction-log-file.c b/src/lib-index/mail-transaction-log-file.c
+index 8410865a1..82723fe3a 100644
+--- src/lib-index/mail-transaction-log-file.c
++++ src/lib-index/mail-transaction-log-file.c
+@@ -1257,7 +1257,7 @@ int mail_transaction_log_file_get_highest_modseq_at(
+ 
+ static int
+ get_modseq_next_offset_at(struct mail_transaction_log_file *file,
+-			  uint64_t modseq,
++			  uint64_t modseq, bool use_highest,
+ 			  uoff_t *cur_offset, uint64_t *cur_modseq,
+ 			  uoff_t *next_offset_r)
+ {
+@@ -1279,7 +1279,7 @@ get_modseq_next_offset_at(struct mail_transaction_log_file *file,
+ 	}
+ 
+ 	/* check sync_highest_modseq again in case sync_offset was updated */
+-	if (modseq >= file->sync_highest_modseq) {
++	if (modseq >= file->sync_highest_modseq && use_highest) {
+ 		*next_offset_r = file->sync_offset;
+ 		return 0;
+ 	}
+@@ -1332,16 +1332,30 @@ int mail_transaction_log_file_get_modseq_next_offset(
+ 		cur_modseq = cache->highest_modseq;
+ 	}
+ 
+-	if ((ret = get_modseq_next_offset_at(file, modseq, &cur_offset,
++	if ((ret = get_modseq_next_offset_at(file, modseq, TRUE, &cur_offset,
+ 					     &cur_modseq, next_offset_r)) <= 0)
+ 		return ret;
+ 	if (cur_offset == file->sync_offset) {
+ 		/* if we got to sync_offset, cur_modseq should be
+ 		   sync_highest_modseq */
+ 		mail_index_set_error(file->log->index,
+-			"%s: Transaction log changed unexpectedly, "
+-			"can't get modseq", file->filepath);
+-		return -1;
++			"%s: Transaction log modseq tracking is corrupted - fixing",
++			file->filepath);
++		/* retry getting the offset by reading from the beginning
++		   of the file */
++		cur_offset = file->hdr.hdr_size;
++		cur_modseq = file->hdr.initial_modseq;
++		ret = get_modseq_next_offset_at(file, modseq, FALSE,
++						&cur_offset, &cur_modseq,
++						next_offset_r);
++		if (ret < 0)
++			return -1;
++		i_assert(ret != 0);
++		/* get it fixed on the next sync */
++		file->log->index->need_recreate = TRUE;
++		file->need_rotate = TRUE;
++		/* clear cache, since it's unreliable */
++		memset(file->modseq_cache, 0, sizeof(file->modseq_cache));
+ 	}
+ 
+ 	/* @UNSAFE: cache the value */
+diff --git a/src/lib-index/mail-transaction-log-private.h b/src/lib-index/mail-transaction-log-private.h
+index f56434b40..cba0b6b5b 100644
+--- src/lib-index/mail-transaction-log-private.h
++++ src/lib-index/mail-transaction-log-private.h
+@@ -81,6 +81,7 @@ struct mail_transaction_log_file {
+ 	unsigned int locked:1;
+ 	unsigned int locked_sync_offset_updated:1;
+ 	unsigned int corrupted:1;
++	unsigned int need_rotate:1;
+ };
+ 
+ struct mail_transaction_log {
+diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
+index 1738e367f..16d301f34 100644
+--- src/lib-index/mail-transaction-log.c
++++ src/lib-index/mail-transaction-log.c
+@@ -234,6 +234,9 @@ bool mail_transaction_log_want_rotate(struct mail_transaction_log *log)
+ {
+ 	struct mail_transaction_log_file *file = log->head;
+ 
++	if (file->need_rotate)
++		return TRUE;
++
+ 	if (file->hdr.major_version < MAIL_TRANSACTION_LOG_MAJOR_VERSION ||
+ 	    (file->hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+ 	     file->hdr.minor_version < MAIL_TRANSACTION_LOG_MINOR_VERSION)) {