Add full-text indexing using the GMime library for parsing.

This is based on the old notmuch-index-message.cc from early in
the history of notmuch, but considerably cleaned up now that
we have some experience with Xapian and know just what we want
to index, (rather than just blindly trying to index exactly
what sup does).

This does slow down notmuch_database_add_message a *lot*, but I've
got some ideas for getting some time back.

1 files changed, 26 insertions, 2 deletions

diff --git a/database.cc b/database.cc
index 71246eb..583bee8 100644
--- a/database.cc
+++ b/database.cc
@@ -114,6 +114,13 @@ prefix_t BOOLEAN_PREFIX_EXTERNAL[] = {
     { "id", "Q" }
 };
 
+prefix_t PROBABILISTIC_PREFIX[]= {
+    { "from", "XFROM" },
+    { "to", "XTO" },
+    { "attachment", "XATTACHMENT" },
+    { "subject", "XSUBJECT"}
+};
+
 int
 _internal_error (const char *format, ...)
 {
@@ -141,6 +148,10 @@ _find_prefix (const char *name)
 	if (strcmp (name, BOOLEAN_PREFIX_EXTERNAL[i].name) == 0)
 	    return BOOLEAN_PREFIX_EXTERNAL[i].prefix;
 
+    for (i = 0; i < ARRAY_SIZE (PROBABILISTIC_PREFIX); i++)
+	if (strcmp (name, PROBABILISTIC_PREFIX[i].name) == 0)
+	    return PROBABILISTIC_PREFIX[i].prefix;
+
     INTERNAL_ERROR ("No prefix exists for '%s'\n", name);
 
     return "";
@@ -478,14 +489,24 @@ notmuch_database_open (const char *path)
 	notmuch->xapian_db = new Xapian::WritableDatabase (xapian_path,
 							   Xapian::DB_CREATE_OR_OPEN);
 	notmuch->query_parser = new Xapian::QueryParser;
+	notmuch->term_gen = new Xapian::TermGenerator;
+	notmuch->term_gen->set_stemmer (Xapian::Stem ("english"));
+
 	notmuch->query_parser->set_default_op (Xapian::Query::OP_AND);
 	notmuch->query_parser->set_database (*notmuch->xapian_db);
+	notmuch->query_parser->set_stemmer (Xapian::Stem ("english"));
+	notmuch->query_parser->set_stemming_strategy (Xapian::QueryParser::STEM_SOME);
 
 	for (i = 0; i < ARRAY_SIZE (BOOLEAN_PREFIX_EXTERNAL); i++) {
 	    prefix_t *prefix = &BOOLEAN_PREFIX_EXTERNAL[i];
 	    notmuch->query_parser->add_boolean_prefix (prefix->name,
 						       prefix->prefix);
 	}
+
+	for (i = 0; i < ARRAY_SIZE (PROBABILISTIC_PREFIX); i++) {
+	    prefix_t *prefix = &PROBABILISTIC_PREFIX[i];
+	    notmuch->query_parser->add_prefix (prefix->name, prefix->prefix);
+	}
     } catch (const Xapian::Error &error) {
 	fprintf (stderr, "A Xapian exception occurred: %s\n",
 		 error.get_msg().c_str());
@@ -508,6 +529,7 @@ notmuch_database_close (notmuch_database_t *notmuch)
 {
     notmuch->xapian_db->flush ();
 
+    delete notmuch->term_gen;
     delete notmuch->query_parser;
     delete notmuch->xapian_db;
     talloc_free (notmuch);
@@ -924,9 +946,11 @@ notmuch_database_add_message (notmuch_database_t *notmuch,
 	{
 	    ret = NOTMUCH_STATUS_FILE_NOT_EMAIL;
 	    goto DONE;
-	} else {
-	    _notmuch_message_sync (message);
 	}
+
+	_notmuch_message_index_file (message, filename);
+
+	_notmuch_message_sync (message);
     } catch (const Xapian::Error &error) {
 	fprintf (stderr, "A Xapian exception occurred: %s.\n",
 		 error.get_msg().c_str());