path: root/libibex/words.c

/*
    Copyright 2000 Helix Code Inc.
*/
/* words.c: low-level indexing ops */

#include <ctype.h>
#include <errno.h>
#include <string.h>

#include <unicode.h>

#include "ibex_internal.h"

static signed char utf8_trans[] = {
  'A', 'A', 'A', 'A', 'A', 'A', -1, 'C', 'E', 'E', 'E', 'E', 'I', 'I',
  'I', 'I', -2, 'N', 'O', 'O', 'O', 'O', 'O', '*', 'O', 'U', 'U', 'U',
  'U', 'Y', -3, -4, 'a', 'a', 'a', 'a', 'a', 'a', -5, 'c', 'e', 'e',
  'e', 'e', 'i', 'i', 'i', 'i', -6, 'n', 'o', 'o', 'o', 'o', 'o', '/',
  'o', 'u', 'u', 'u', 'u', 'y', -7, 'y', 'A', 'a', 'A', 'a', 'A', 'a',
  'C', 'c', 'C', 'c', 'C', 'c', 'C', 'c', 'D', 'd', 'D', 'd', 'E', 'e',
  'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'G', 'g', 'G', 'g', 'G', 'g',
  'G', 'g', 'H', 'h', 'H', 'h', 'I', 'i', 'I', 'i', 'I', 'i', 'I', 'i',
  'I', 'i', -8, -9, 'J', 'j', 'K', 'k', 'k', 'L', 'l', 'L', 'l', 'L',
  'l', 'L', 'l', 'L', 'l', 'N', 'n', 'N', 'n', 'N', 'n', 'n', -10, -11,
  'O', 'o', 'O', 'o', 'O', 'o', -12, -13, 'R', 'r', 'R', 'r', 'R', 'r',
  'S', 'r', 'S', 's', 'S', 's', 'S', 's', 'T', 't', 'T', 't', 'T', 't',
  'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'W', 'w',
  'Y', 'y', 'Y', 'Z', 'z', 'Z', 'z', 'Z', 'z', 's'
};

static char *utf8_long_trans[] = {
  "AE", "TH", "TH", "ss", "ae", "th", "th", "IJ", "ij", "NG", "ng", "OE", "oe"
};

/* This is a bit weird. It takes pointers to the start and end (actually
 * just past the end) of a UTF-8-encoded word, and a buffer at least 1
 * byte longer than the length of the word. It copies the word into the
 * buffer in all lowercase without accents, and splits up ligatures.
 * (Since any ligature would be a multi-byte character in UTF-8, splitting
 * them into two US-ASCII characters won't overrun the buffer.)
 *
 * It is not safe to call this routine with bad UTF-8.
 */
static void
normalize_word(char *start, char *end, char *buf)
{
  unsigned char *s, *d;
  unicode_char_t uc;

  s = (unsigned char *)start;
  d = (unsigned char *)buf;
  while (s < (unsigned char *)end)
    {
      if (*s < 0x80)
    {
      /* US-ASCII character: copy unless it's an apostrophe. */
      if (*s != '\'')
        *d++ = tolower(*s);
      s++;
    }
      else
    {
      char *next = unicode_get_utf8(s, &uc);
      if (uc >= 0xc0 && uc < 0xc0 + sizeof(utf8_trans))
        {
          signed char ch = utf8_trans[uc - 0xc0];
          if (ch > 0)
        *d++ = tolower(ch);
          else
        {
          *d++ = tolower(utf8_long_trans[-ch - 1][0]);
          *d++ = tolower(utf8_long_trans[-ch - 1][1]);
        }
          s = next;
        }
      else
        {
          while (s < (unsigned char *)next)
        *d++ = *s++;
        }
    }
    }
  *d = '\0';
}

enum { IBEX_ALPHA, IBEX_NONALPHA, IBEX_INVALID, IBEX_INCOMPLETE };

/* This incorporates parts of libunicode, because there's no way to
 * force libunicode to not read past a certain point.
 */
static int
utf8_category(char *sp, char **snp, char *send)
{
  unsigned char *p = (unsigned char *)sp, **np = (unsigned char **)snp;
  unsigned char *end = (unsigned char *)send;

  if (isascii(*p))
    {
      *np = p + 1;
      if (isalpha(*p) || *p == '\'')
    return IBEX_ALPHA;
      return IBEX_NONALPHA;
    }
  else
    {
      unicode_char_t uc;
      int more;

      if ((*p & 0xe0) == 0xc0)
    {
      more = 1;
      uc = *p & 0x1f;
    }
      else if ((*p & 0xf0) == 0xe0)
    {
      more = 2;
      uc = *p & 0x0f;
    }
      else if ((*p & 0xf8) == 0xf0)
    {
      more = 3;
      uc = *p & 0x07;
    }
      else if ((*p & 0xfc) == 0xf8)
    {
      more = 4;
      uc = *p & 0x03;
    }
      else if ((*p & 0xfe) == 0xfc)
    {
      more = 5;
      uc = *p & 0x01;
    }
      else
    return IBEX_INVALID;

      if (p + more > end)
    return IBEX_INCOMPLETE;

      while (more--)
    {
      if ((*++p & 0xc0) != 0x80)
        return IBEX_INVALID;
      uc <<= 6;
      uc |= *p & 0x3f;
    }

      *np = p + 1;
      if (unicode_isalpha(uc))
    return IBEX_ALPHA;
      else
    return IBEX_NONALPHA;
    }
}

static ibex_file *
get_ibex_file(ibex *ib, char *name)
{
  ibex_file *ibf;

  ibf = g_tree_lookup(ib->files, name);
  if (!ibf)
    {
      ibf = g_malloc(sizeof(ibex_file));
      ibf->name = strdup(name);
      ibf->index = 0;
      g_tree_insert(ib->files, ibf->name, ibf);
      ib->dirty = TRUE;
    }
  return ibf;
}

static void
ref_word(ibex *ib, ibex_file *ibf, char *word)
{
  GPtrArray *refs;

  refs = g_hash_table_lookup(ib->words, word);
  if (!refs)
    {
      refs = g_ptr_array_new();
      g_hash_table_insert(ib->words, g_strdup(word), refs);
      g_ptr_array_add(refs, ibf);
      ib->dirty = TRUE;
    }
  else if (g_ptr_array_index(refs, refs->len - 1) != ibf)
    {
      g_ptr_array_add(refs, ibf);
      ib->dirty = TRUE;
    }
}

int
ibex_index_buffer(ibex *ib, char *name, char *buffer,
          size_t len, size_t *unread)
{
  char *p, *q, *nq, *end, *word;
  ibex_file *ibf = get_ibex_file(ib, name);
  int wordsiz, cat;

  end = buffer + len;
  wordsiz = 20;
  word = g_malloc(wordsiz);

  p = buffer;
  while (p < end)
    {
      while (p < end)
    {
      cat = utf8_category(p, &q, end);
      if (cat != IBEX_NONALPHA)
        break;
      p = q;
    }
      if (p == end)
    {
      g_free(word);
      return 0;
    }
      else if (cat == IBEX_INVALID)
    {
      errno = EINVAL;
      g_free(word);
      return -1;
    }
      else if (cat == IBEX_INCOMPLETE)
    q = end;

      while (q < end)
    {
      cat = utf8_category(q, &nq, end);
      if (cat != IBEX_ALPHA)
        break;
      q = nq;
    }
      if (cat == IBEX_INVALID || (cat == IBEX_INCOMPLETE && !unread))
    {
      errno = EINVAL;
      g_free(word);
      return -1;
    }
      else if (cat == IBEX_INCOMPLETE || (q == end && unread))
    {
      *unread = end - p;
      g_free(word);
      return 0;
    }

      if (wordsiz < q - p + 1)
    {
      wordsiz = q - p + 1;
      word = g_realloc(word, wordsiz);
    }
      normalize_word(p, q, word);
      ref_word(ib, ibf, word);
      p = q;
    }

  if (unread)
    *unread = 0;
  g_free(word);
  return 0;
}