#include <glib.h>
#include <db.h>
#include <stdio.h>
#include <unicode.h>
#include <ctype.h>
#include "ibex_internal.h"
#define d(x)
/*
Uses 2 databases:
names - HASH - lists which files are stored
words - BTREE - index words to files
*/
static int
db_delete_name(DB *db, const char *name)
{
DBC *cursor;
int err, namelen;
DBT key, data;
printf("called to delete name %s\n", name);
return 0;
err = db->cursor(db, NULL, &cursor, 0);
if (err != 0) {
printf("Error occured?: %s\n", db_strerror(err));
return err;
}
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
namelen = strlen(name);
err = cursor->c_get(cursor, &key, &data, DB_FIRST);
while (err == 0) {
if (data.size == namelen && memcmp(data.data, name, namelen) == 0) {
d(printf("Found match, removing it\n"));
cursor->c_del(cursor, 0);
} else {
data.size = namelen;
data.data = (void *)name;
if (cursor->c_get(cursor, &key, &data, DB_GET_BOTH) == 0) {
d(printf("seek to match, removing it\n"));
cursor->c_del(cursor, 0);
} else {
d(printf("seek to match, not found\n"));
}
}
err = cursor->c_get(cursor, &key, &data, DB_NEXT_NODUP);
}
cursor->c_close(cursor);
return 0;
}
static int
db_index_words(DB *db, char *name, char **words)
{
DBT key, data;
int err = 0;
char *word;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
data.data = name;
data.size = strlen(name);
for (;(word=*words);words++) {
/* normalise word first ... */
key.data = word;
key.size = strlen(word);
err = db->put(db, NULL, &key, &data, 0);
if (err != 0)
printf("Error occured?: %s\n", db_strerror(err));
}
return err;
}
static int
db_index_word(DB *db, char *name, char *word)
{
DBT key, data;
int err = 0;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
data.data = name;
data.size = strlen(name);
key.data = word;
key.size = strlen(word);
err = db->put(db, NULL, &key, &data, 0);
if (err != 0)
printf("Error occured?: %s\n", db_strerror(err));
return err;
}
static GPtrArray *
db_find(DB *db, char *word)
{
DBT key, data;
int err;
DBC *cursor;
GPtrArray *matches = g_ptr_array_new();
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
err = db->cursor(db, NULL, &cursor, 0);
if (err != 0) {
printf("Error occured?: %s\n", db_strerror(err));
return matches;
}
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
key.size = strlen(word);
key.data = word;
err = cursor->c_get(cursor, &key, &data, DB_SET);
while (err == 0) {
char *name;
name = g_malloc(data.size+1);
memcpy(name, data.data, data.size);
name[data.size] = '\0';
g_ptr_array_add(matches, name);
err = cursor->c_get(cursor, &key, &data, DB_NEXT_DUP);
}
if (err != DB_NOTFOUND) {
printf("Error occured?: %s\n", db_strerror(err));
/* what to do? doesn't matter too much its just a search ... */
}
cursor->c_close(cursor);
return matches;
}
/* check that db contains key @word, optionally with contents @name */
static gboolean
db_contains_word(DB *db, char *name, char *word)
{
DBT key, data;
memset(&key, 0, sizeof(key));
memset(&data, 0, sizeof(data));
if (name != NULL) {
data.size = strlen(name);
data.data = name;
}
key.size = strlen(word);
key.data = word;
return db->get(db, NULL, &key, &data, name?DB_GET_BOTH:DB_SET) == 0;
}
static int
db_delete_word(DB *db, char *word)
{
DBT key;
memset(&key, 0, sizeof(key));
key.size = strlen(word);
key.data = word;
return db->del(db, NULL, &key, 0);
}
static signed char utf8_trans[] = {
'A', 'A', 'A', 'A', 'A', 'A', -1, 'C', 'E', 'E', 'E', 'E', 'I', 'I',
'I', 'I', -2, 'N', 'O', 'O', 'O', 'O', 'O', '*', 'O', 'U', 'U', 'U',
'U', 'Y', -3, -4, 'a', 'a', 'a', 'a', 'a', 'a', -5, 'c', 'e', 'e',
'e', 'e', 'i', 'i', 'i', 'i', -6, 'n', 'o', 'o', 'o', 'o', 'o', '/',
'o', 'u', 'u', 'u', 'u', 'y', -7, 'y', 'A', 'a', 'A', 'a', 'A', 'a',
'C', 'c', 'C', 'c', 'C', 'c', 'C', 'c', 'D', 'd', 'D', 'd', 'E', 'e',
'E', 'e', 'E', 'e', 'E', 'e', 'E', 'e', 'G', 'g', 'G', 'g', 'G', 'g',
'G', 'g', 'H', 'h', 'H', 'h', 'I', 'i', 'I', 'i', 'I', 'i', 'I', 'i',
'I', 'i', -8, -9, 'J', 'j', 'K', 'k', 'k', 'L', 'l', 'L', 'l', 'L',
'l', 'L', 'l', 'L', 'l', 'N', 'n', 'N', 'n', 'N', 'n', 'n', -10, -11,
'O', 'o', 'O', 'o', 'O', 'o', -12, -13, 'R', 'r', 'R', 'r', 'R', 'r',
'S', 'r', 'S', 's', 'S', 's', 'S', 's', 'T', 't', 'T', 't', 'T', 't',
'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'U', 'u', 'W', 'w',
'Y', 'y', 'Y', 'Z', 'z', 'Z', 'z', 'Z', 'z', 's'
};
static char *utf8_long_trans[] = {
"AE", "TH", "TH", "ss", "ae", "th", "th", "IJ", "ij",
"NG", "ng", "OE", "oe"
};
/* This is a bit weird. It takes pointers to the start and end (actually
* just past the end) of a UTF-8-encoded word, and a buffer at least 1
* byte longer than the length of the word. It copies the word into the
* buffer in all lowercase without accents, and splits up ligatures.
* (Since any ligature would be a multi-byte character in UTF-8, splitting
* them into two US-ASCII characters won't overrun the buffer.)
*
* It is not safe to call this routine with bad UTF-8.
*/
static void
ibex_normalise_word(char *start, char *end, char *buf)
{
unsigned char *s, *d;
unicode_char_t uc;
s = (unsigned char *)start;
d = (unsigned char *)buf;
while (s < (unsigned char *)end) {
if (*s < 0x80) {
/* US-ASCII character: copy unless it's
* an apostrophe.
*/
if (*s != '\'')
*d++ = tolower (*s);
s++;
} else {
char *next = unicode_get_utf8 (s, &uc);
if (uc >= 0xc0 && uc < 0xc0 + sizeof (utf8_trans)) {
signed char ch = utf8_trans[uc - 0xc0];
if (ch > 0)
*d++ = tolower (ch);
else {
*d++ = tolower (utf8_long_trans[-ch - 1][0]);
*d++ = tolower (utf8_long_trans[-ch - 1][1]);
}
s = next;
} else {
while (s < (unsigned char *)next)
*d++ = *s++;
}
}
}
*d = '\0';
}
enum { IBEX_ALPHA, IBEX_NONALPHA, IBEX_INVALID, IBEX_INCOMPLETE };
/* This incorporates parts of libunicode, because there's no way to
* force libunicode to not read past a certain point.
*/
static int
utf8_category (char *sp, char **snp, char *send)
{
unsigned char *p = (unsigned char *)sp, **np = (unsigned char **)snp;
unsigned char *end = (unsigned char *)send;
if (isascii (*p)) {
*np = p + 1;
if (isalpha (*p) || *p == '\'')
return IBEX_ALPHA;
return IBEX_NONALPHA;
} else {
unicode_char_t uc;
int more;
if ((*p & 0xe0) == 0xc0) {
more = 1;
uc = *p & 0x1f;
} else if ((*p & 0xf0) == 0xe0) {
more = 2;
uc = *p & 0x0f;
} else if ((*p & 0xf8) == 0xf0) {
more = 3;
uc = *p & 0x07;
} else if ((*p & 0xfc) == 0xf8) {
more = 4;
uc = *p & 0x03;
} else if ((*p & 0xfe) == 0xfc) {
more = 5;
uc = *p & 0x01;
} else
return IBEX_INVALID;
if (p + more > end)
return IBEX_INCOMPLETE;
while (more--) {
if ((*++p & 0xc0) != 0x80)
return IBEX_INVALID;
uc <<= 6;
uc |= *p & 0x3f;
}
*np = p + 1;
if (unicode_isalpha (uc))
return IBEX_ALPHA;
else
return IBEX_NONALPHA;
}
}
static void
do_insert_words(char *key, char *name, ibex *ib)
{
db_index_word(ib->words, name, key);
g_free(key);
}
static void
do_free_words(char *key, char *name, void *data)
{
g_free(key);
}
/**
* ibex_index_buffer: the lowest-level ibex indexing interface
* @ib: an ibex
* @name: the name of the file being indexed
* @buffer: a buffer containing data from the file
* @len: the length of @buffer
* @unread: an output argument containing the number of unread bytes
*
* This routine indexes up to @len bytes from @buffer into @ib.
* If @unread is NULL, the indexer assumes that the buffer ends on a
* word boundary, and will index all the way to the end of the
* buffer. If @unread is not NULL, and the buffer ends with an
* alphabetic character, the indexer will assume that the buffer has
* been cut off in the middle of a word, and return the number of
* un-indexed bytes at the end of the buffer in *@unread. The caller
* should then read in more data through whatever means it has
* and pass in the unread bytes from the original buffer, followed
* by the new data, on its next call.
*
* Return value: 0 on success, -1 on failure.
**/
int
ibex_index_buffer (ibex *ib, char *name, char *buffer, size_t len, size_t *unread)
{