Initial revision

svn path=/trunk/; revision=1768

1 files changed, 263 insertions, 0 deletions

diff --git a/libibex/words.c b/libibex/words.c
new file mode 100644
index 0000000000..4776634251
--- /dev/null
+++ b/libibex/words.c
@@ -0,0 +1,263 @@
+/*
+	Copyright 2000 Helix Code Inc.
+*/
+/* words.c: low-level indexing ops */
+
+#include <ctype.h>
+#include <errno.h>
+#include <string.h>
+
+#include <unicode.h>
+
+#include "ibex_internal.h"
+
+static signed char utf8_trans[] = {
+  'A', 'A', 'A', 'A', 'A', 'A', -1, 'C', 'E', 'E', 'E', 'E', 'I', 'I',
+  'I', 'I', -2, 'N', 'O', 'O', 'O', 'O', 'O', '*', 'O', 'U', 'U', 'U',
+  'U', 'Y', -3, -4, 'a', 'a', 'a', 'a', 'a', 'a', -5, 'c', 'e', 'e',
+  'e', 'e', 'i', 'i', 'i', 'i', -6, 'n', 'o', 'o', 'o', 'o', 'o', '/',
+  'o', 'u', 'u', 'u', 'u', 'y', -7, 'y', 'A', 'a', 'A', 'a', 'A', 'a',
+  'C', 'c', 'C', 'c', 'C', 'c', 'C', 'c', 'D', 'd', 'D', 'd', 'E', 'e',
+  'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'G', 'g', 'G', 'g', 'G', 'g',
+  'G', 'g', 'H', 'h', 'H', 'h', 'I', 'i', 'I', 'i', 'I', 'i', 'I', 'i',
+  'I', 'i', -8, -9, 'J', 'j', 'K', 'k', 'k', 'L', 'l', 'L', 'l', 'L',
+  'l', 'L', 'l', 'L', 'l', 'N', 'n', 'N', 'n', 'N', 'n', 'n', -10, -11,
+  'O', 'o', 'O', 'o', 'O', 'o', -12, -13, 'R', 'r', 'R', 'r', 'R', 'r',
+  'S', 'r', 'S', 's', 'S', 's', 'S', 's', 'T', 't', 'T', 't', 'T', 't',
+  'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'W', 'w',
+  'Y', 'y', 'Y', 'Z', 'z', 'Z', 'z', 'Z', 'z', 's'
+};
+
+static char *utf8_long_trans[] = {
+  "AE", "TH", "TH", "ss", "ae", "th", "th", "IJ", "ij", "NG", "ng", "OE", "oe"
+};
+
+/* This is a bit weird. It takes pointers to the start and end (actually
+ * just past the end) of a UTF-8-encoded word, and a buffer at least 1
+ * byte longer than the length of the word. It copies the word into the
+ * buffer in all lowercase without accents, and splits up ligatures.
+ * (Since any ligature would be a multi-byte character in UTF-8, splitting
+ * them into two US-ASCII characters won't overrun the buffer.)
+ *
+ * It is not safe to call this routine with bad UTF-8.
+ */
+static void
+normalize_word(char *start, char *end, char *buf)
+{
+  unsigned char *s, *d;
+  unicode_char_t uc;
+
+  s = (unsigned char *)start;
+  d = (unsigned char *)buf;
+  while (s < (unsigned char *)end)
+    {
+      if (*s < 0x80)
+	{
+	  /* US-ASCII character: copy unless it's an apostrophe. */
+	  if (*s != '\'')
+	    *d++ = tolower(*s);
+	  s++;
+	}
+      else
+	{
+	  char *next = unicode_get_utf8(s, &uc);
+	  if (uc >= 0xc0 && uc < 0xc0 + sizeof(utf8_trans))
+	    {
+	      signed char ch = utf8_trans[uc - 0xc0];
+	      if (ch > 0)
+		*d++ = tolower(ch);
+	      else
+		{
+		  *d++ = tolower(utf8_long_trans[-ch - 1][0]);
+		  *d++ = tolower(utf8_long_trans[-ch - 1][1]);
+		}
+	      s = next;
+	    }
+	  else
+	    {
+	      while (s < (unsigned char *)next)
+		*d++ = *s++;
+	    }
+	}
+    }
+  *d = '\0';
+}
+
+enum { IBEX_ALPHA, IBEX_NONALPHA, IBEX_INVALID, IBEX_INCOMPLETE };
+
+/* This incorporates parts of libunicode, because there's no way to
+ * force libunicode to not read past a certain point.
+ */
+static int
+utf8_category(char *sp, char **snp, char *send)
+{
+  unsigned char *p = (unsigned char *)sp, **np = (unsigned char **)snp;
+  unsigned char *end = (unsigned char *)send;
+
+  if (isascii(*p))
+    {
+      *np = p + 1;
+      if (isalpha(*p) || *p == '\'')
+	return IBEX_ALPHA;
+      return IBEX_NONALPHA;
+    }
+  else
+    {
+      unicode_char_t uc;
+      int more;
+
+      if ((*p & 0xe0) == 0xc0)
+	{
+	  more = 1;
+	  uc = *p & 0x1f;
+	}
+      else if ((*p & 0xf0) == 0xe0)
+	{
+	  more = 2;
+	  uc = *p & 0x0f;
+	}
+      else if ((*p & 0xf8) == 0xf0)
+	{
+	  more = 3;
+	  uc = *p & 0x07;
+	}
+      else if ((*p & 0xfc) == 0xf8)
+	{
+	  more = 4;
+	  uc = *p & 0x03;
+	}
+      else if ((*p & 0xfe) == 0xfc)
+	{
+	  more = 5;
+	  uc = *p & 0x01;
+	}
+      else
+	return IBEX_INVALID;
+
+      if (p + more > end)
+	return IBEX_INCOMPLETE;
+
+      while (more--)
+	{
+	  if ((*++p & 0xc0) != 0x80)
+	    return IBEX_INVALID;
+	  uc <<= 6;
+	  uc |= *p & 0x3f;
+	}
+
+      *np = p + 1;
+      if (unicode_isalpha(uc))
+	return IBEX_ALPHA;
+      else
+	return IBEX_NONALPHA;
+    }
+}
+
+static ibex_file *
+get_ibex_file(ibex *ib, char *name)
+{
+  ibex_file *ibf;
+
+  ibf = g_tree_lookup(ib->files, name);
+  if (!ibf)
+    {
+      ibf = g_malloc(sizeof(ibex_file));
+      ibf->name = strdup(name);
+      ibf->index = 0;
+      g_tree_insert(ib->files, ibf->name, ibf);
+      ib->dirty = TRUE;
+    }
+  return ibf;
+}
+
+static void
+ref_word(ibex *ib, ibex_file *ibf, char *word)
+{
+  GPtrArray *refs;
+
+  refs = g_hash_table_lookup(ib->words, word);
+  if (!refs)
+    {
+      refs = g_ptr_array_new();
+      g_hash_table_insert(ib->words, g_strdup(word), refs);
+      g_ptr_array_add(refs, ibf);
+      ib->dirty = TRUE;
+    }
+  else if (g_ptr_array_index(refs, refs->len - 1) != ibf)
+    {
+      g_ptr_array_add(refs, ibf);
+      ib->dirty = TRUE;
+    }
+}
+
+int
+ibex_index_buffer(ibex *ib, char *name, char *buffer,
+		  size_t len, size_t *unread)
+{
+  char *p, *q, *nq, *end, *word;
+  ibex_file *ibf = get_ibex_file(ib, name);
+  int wordsiz, cat;
+
+  end = buffer + len;
+  wordsiz = 20;
+  word = g_malloc(wordsiz);
+
+  p = buffer;
+  while (p < end)
+    {
+      while (p < end)
+	{
+	  cat = utf8_category(p, &q, end);
+	  if (cat != IBEX_NONALPHA)
+	    break;
+	  p = q;
+	}
+      if (p == end)
+	{
+	  g_free(word);
+	  return 0;
+	}
+      else if (cat == IBEX_INVALID)
+	{
+	  errno = EINVAL;
+	  g_free(word);
+	  return -1;
+	}
+      else if (cat == IBEX_INCOMPLETE)
+	q = end;
+
+      while (q < end)
+	{
+	  cat = utf8_category(q, &nq, end);
+	  if (cat != IBEX_ALPHA)
+	    break;
+	  q = nq;
+	}
+      if (cat == IBEX_INVALID || (cat == IBEX_INCOMPLETE && !unread))
+	{
+	  errno = EINVAL;
+	  g_free(word);
+	  return -1;
+	}
+      else if (cat == IBEX_INCOMPLETE || (q == end && unread))
+	{
+	  *unread = end - p;
+	  g_free(word);
+	  return 0;
+	}
+
+      if (wordsiz < q - p + 1)
+	{
+	  wordsiz = q - p + 1;
+	  word = g_realloc(word, wordsiz);
+	}
+      normalize_word(p, q, word);
+      ref_word(ib, ibf, word);
+      p = q;
+    }
+
+  if (unread)
+    *unread = 0;
+  g_free(word);
+  return 0;
+}