From 863c1495141084dd1e66a73be35560b2531b71bf Mon Sep 17 00:00:00 2001 From: Tomi Ollila Date: Wed, 15 Feb 2012 11:17:30 +0200 Subject: test: add tests wrt ignoring user-specified files and directories Files and directories which are specified in 'new.ignore' in the config file shouldn't be indexed nor reported by `notmuch new'. This is basically Pieter's work with Austin's comments addressed. --- test/new | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'test/new') diff --git a/test/new b/test/new index 49f390d..e453684 100755 --- a/test/new +++ b/test/new @@ -153,4 +153,26 @@ rm -rf "${MAIL_DIR}"/two output=$(NOTMUCH_NEW) test_expect_equal "$output" "No new mail. Removed 3 messages." +# This test requires that notmuch new has been run at least once. +test_begin_subtest "Skip and report non-mail files" +generate_message +mkdir -p "${MAIL_DIR}"/.git && touch "${MAIL_DIR}"/.git/config +touch "${MAIL_DIR}"/ignored_file +touch "${MAIL_DIR}"/.ignored_hidden_file +output=$(NOTMUCH_NEW 2>&1) +test_expect_equal "$output" \ +"Note: Ignoring non-mail file: ${MAIL_DIR}/.git/config +Note: Ignoring non-mail file: ${MAIL_DIR}/.ignored_hidden_file +Note: Ignoring non-mail file: ${MAIL_DIR}/ignored_file +Added 1 new message to the database." + +test_begin_subtest "Ignore files and directories specified in new.ignore" +test_subtest_known_broken +generate_message +notmuch config set new.ignore .git ignored_file .ignored_hidden_file +touch "${MAIL_DIR}"/.git # change .git's mtime for notmuch new to rescan. +output=$(NOTMUCH_NEW 2>&1) +test_expect_equal "$output" "Added 1 new message to the database." + + test_done -- cgit v1.2.3 From ce1e720de64270a7cbb4bc3fba2c7fe081de3edc Mon Sep 17 00:00:00 2001 From: Tomi Ollila Date: Wed, 15 Feb 2012 11:17:31 +0200 Subject: add support for user-specified files & directories to ignore A new configuration key 'new.ignore' is used to determine which files and directories user wants not to be scanned as new mails. Mark the corresponding test as no longer broken. This work merges my previous attempts and Andreas Amann's work in id:"ylp7hi23mw8.fsf@tyndall.ie" --- notmuch-client.h | 9 +++++++++ notmuch-config.c | 30 +++++++++++++++++++++++++++++- notmuch-new.c | 45 +++++++++++++++++++++++++++++++++------------ test/new | 1 - 4 files changed, 71 insertions(+), 14 deletions(-) (limited to 'test/new') diff --git a/notmuch-client.h b/notmuch-client.h index 60828aa..f4a62cc 100644 --- a/notmuch-client.h +++ b/notmuch-client.h @@ -250,6 +250,15 @@ notmuch_config_set_new_tags (notmuch_config_t *config, const char *new_tags[], size_t length); +const char ** +notmuch_config_get_new_ignore (notmuch_config_t *config, + size_t *length); + +void +notmuch_config_set_new_ignore (notmuch_config_t *config, + const char *new_ignore[], + size_t length); + notmuch_bool_t notmuch_config_get_maildir_synchronize_flags (notmuch_config_t *config); diff --git a/notmuch-config.c b/notmuch-config.c index a124e34..1f01128 100644 --- a/notmuch-config.c +++ b/notmuch-config.c @@ -44,7 +44,10 @@ static const char new_config_comment[] = " The following options are supported here:\n" "\n" "\ttags A list (separated by ';') of the tags that will be\n" - "\t added to all messages incorporated by \"notmuch new\".\n"; + "\t added to all messages incorporated by \"notmuch new\".\n" + "\n" + "\tignore A list (separated by ';') of file and directory names\n" + "\t that will not be searched for messages by \"notmuch new\".\n"; static const char user_config_comment[] = " User configuration\n" @@ -105,6 +108,8 @@ struct _notmuch_config { size_t user_other_email_length; const char **new_tags; size_t new_tags_length; + const char **new_ignore; + size_t new_ignore_length; notmuch_bool_t maildir_synchronize_flags; const char **search_exclude_tags; size_t search_exclude_tags_length; @@ -264,6 +269,8 @@ notmuch_config_open (void *ctx, config->user_other_email_length = 0; config->new_tags = NULL; config->new_tags_length = 0; + config->new_ignore = NULL; + config->new_ignore_length = 0; config->maildir_synchronize_flags = TRUE; config->search_exclude_tags = NULL; config->search_exclude_tags_length = 0; @@ -361,6 +368,10 @@ notmuch_config_open (void *ctx, notmuch_config_set_new_tags (config, tags, 2); } + if (notmuch_config_get_new_ignore (config, &tmp) == NULL) { + notmuch_config_set_new_ignore (config, NULL, 0); + } + if (notmuch_config_get_search_exclude_tags (config, &tmp) == NULL) { if (is_new) { const char *tags[] = { "deleted", "spam" }; @@ -609,6 +620,14 @@ notmuch_config_get_new_tags (notmuch_config_t *config, size_t *length) &(config->new_tags_length), length); } +const char ** +notmuch_config_get_new_ignore (notmuch_config_t *config, size_t *length) +{ + return _config_get_list (config, "new", "ignore", + &(config->new_ignore), + &(config->new_ignore_length), length); +} + void notmuch_config_set_user_other_email (notmuch_config_t *config, const char *list[], @@ -627,6 +646,15 @@ notmuch_config_set_new_tags (notmuch_config_t *config, &(config->new_tags)); } +void +notmuch_config_set_new_ignore (notmuch_config_t *config, + const char *list[], + size_t length) +{ + _config_set_list (config, "new", "ignore", list, length, + &(config->new_ignore)); +} + const char ** notmuch_config_get_search_exclude_tags (notmuch_config_t *config, size_t *length) { diff --git a/notmuch-new.c b/notmuch-new.c index 8dbebb3..4f13535 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -39,6 +39,8 @@ typedef struct { int verbose; const char **new_tags; size_t new_tags_length; + const char **new_ignore; + size_t new_ignore_length; int total_files; int processed_files; @@ -181,6 +183,20 @@ _entries_resemble_maildir (struct dirent **entries, int count) return 0; } +/* Test if the file/directory is to be ignored. + */ +static notmuch_bool_t +_entry_in_ignore_list (const char *entry, add_files_state_t *state) +{ + size_t i; + + for (i = 0; i < state->new_ignore_length; i++) + if (strcmp (entry, state->new_ignore[i]) == 0) + return TRUE; + + return FALSE; +} + /* Examine 'path' recursively as follows: * * o Ask the filesystem for the mtime of 'path' (fs_mtime) @@ -320,15 +336,15 @@ add_files_recursive (notmuch_database_t *notmuch, } /* Ignore special directories to avoid infinite recursion. - * Also ignore the .notmuch directory and any "tmp" directory - * that appears within a maildir. + * Also ignore the .notmuch directory, any "tmp" directory + * that appears within a maildir and files/directories + * the user has configured to be ignored. */ - /* XXX: Eventually we'll want more sophistication to let the - * user specify files to be ignored. */ if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0 || (is_maildir && strcmp (entry->d_name, "tmp") == 0) || - strcmp (entry->d_name, ".notmuch") ==0) + strcmp (entry->d_name, ".notmuch") == 0 || + _entry_in_ignore_list (entry->d_name, state)) { continue; } @@ -369,6 +385,10 @@ add_files_recursive (notmuch_database_t *notmuch, entry = fs_entries[i]; + /* Ignore files & directories user has configured to be ignored */ + if (_entry_in_ignore_list (entry->d_name, state)) + continue; + /* Check if we've walked past any names in db_files or * db_subdirs. If so, these have been deleted. */ while (notmuch_filenames_valid (db_files) && @@ -650,7 +670,7 @@ add_files (notmuch_database_t *notmuch, * initialized to zero by the top-level caller before calling * count_files). */ static void -count_files (const char *path, int *count) +count_files (const char *path, int *count, add_files_state_t *state) { struct dirent *entry = NULL; char *next; @@ -672,13 +692,13 @@ count_files (const char *path, int *count) entry = fs_entries[i++]; /* Ignore special directories to avoid infinite recursion. - * Also ignore the .notmuch directory. + * Also ignore the .notmuch directory and files/directories + * the user has configured to be ignored. */ - /* XXX: Eventually we'll want more sophistication to let the - * user specify files to be ignored. */ if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0 || - strcmp (entry->d_name, ".notmuch") == 0) + strcmp (entry->d_name, ".notmuch") == 0 || + _entry_in_ignore_list (entry->d_name, state)) { continue; } @@ -699,7 +719,7 @@ count_files (const char *path, int *count) fflush (stdout); } } else if (S_ISDIR (st.st_mode)) { - count_files (next, count); + count_files (next, count, state); } free (next); @@ -841,6 +861,7 @@ notmuch_new_command (void *ctx, int argc, char *argv[]) return 1; add_files_state.new_tags = notmuch_config_get_new_tags (config, &add_files_state.new_tags_length); + add_files_state.new_ignore = notmuch_config_get_new_ignore (config, &add_files_state.new_ignore_length); add_files_state.synchronize_flags = notmuch_config_get_maildir_synchronize_flags (config); db_path = notmuch_config_get_database_path (config); @@ -856,7 +877,7 @@ notmuch_new_command (void *ctx, int argc, char *argv[]) int count; count = 0; - count_files (db_path, &count); + count_files (db_path, &count, &add_files_state); if (interrupted) return 1; diff --git a/test/new b/test/new index e453684..79a6c97 100755 --- a/test/new +++ b/test/new @@ -167,7 +167,6 @@ Note: Ignoring non-mail file: ${MAIL_DIR}/ignored_file Added 1 new message to the database." test_begin_subtest "Ignore files and directories specified in new.ignore" -test_subtest_known_broken generate_message notmuch config set new.ignore .git ignored_file .ignored_hidden_file touch "${MAIL_DIR}"/.git # change .git's mtime for notmuch new to rescan. -- cgit v1.2.3 From b32831e5b8e494da802f7c616c986fcd1e1e7b5f Mon Sep 17 00:00:00 2001 From: Pieter Praet Date: Wed, 1 Feb 2012 21:37:21 +0100 Subject: test: replace occurrences of $PWD with vars that are more stable Thanks to Dmitry Kurochkin for pointing this out: id:"87d39ymyb4.fsf@gmail.com" --- test/emacs | 2 +- test/new | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'test/new') diff --git a/test/emacs b/test/emacs index b74cfa9..13b0471 100755 --- a/test/emacs +++ b/test/emacs @@ -266,7 +266,7 @@ From: Notmuch Test Suite To: user@example.com Subject: Re: Testing message sent via SMTP In-Reply-To: -Fcc: $(pwd)/mail/sent +Fcc: ${MAIL_DIR}/sent --text follows this line-- On 01 Jan 2000 12:00:00 -0000, Notmuch Test Suite wrote: > This is a test that messages are sent via SMTP diff --git a/test/new b/test/new index 79a6c97..99f9913 100755 --- a/test/new +++ b/test/new @@ -117,10 +117,10 @@ test_expect_equal "$output" "No new mail. Removed 3 messages." test_begin_subtest "New symlink to directory" rm -rf "${MAIL_DIR}"/.notmuch -mv "${MAIL_DIR}" "$PWD"/actual_maildir +mv "${MAIL_DIR}" "${TMP_DIRECTORY}"/actual_maildir mkdir "${MAIL_DIR}" -ln -s "$PWD"/actual_maildir "${MAIL_DIR}"/symlink +ln -s "${TMP_DIRECTORY}"/actual_maildir "${MAIL_DIR}"/symlink output=$(NOTMUCH_NEW) test_expect_equal "$output" "Added 1 new message to the database." @@ -128,7 +128,7 @@ test_expect_equal "$output" "Added 1 new message to the database." test_begin_subtest "New symlink to a file" generate_message -external_msg_filename="$PWD"/external/"$(basename "$gen_msg_filename")" +external_msg_filename="${TMP_DIRECTORY}"/external/"$(basename "$gen_msg_filename")" mkdir -p "$(dirname "$external_msg_filename")" mv "$gen_msg_filename" "$external_msg_filename" ln -s "$external_msg_filename" "$gen_msg_filename" -- cgit v1.2.3 From 72c944655f9bc29809dc2f64d206d942d2198f6f Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 24 May 2012 18:01:10 -0400 Subject: test: Test notmuch new with a broken symlink --- test/new | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'test/new') diff --git a/test/new b/test/new index 99f9913..a7bc146 100755 --- a/test/new +++ b/test/new @@ -136,6 +136,16 @@ output=$(NOTMUCH_NEW) test_expect_equal "$output" "Added 1 new message to the database." +test_begin_subtest "Broken symlink aborts" +ln -s does-not-exist "${MAIL_DIR}/broken" +output=$(NOTMUCH_NEW 2>&1) +test_expect_equal "$output" \ +"Error reading directory ${MAIL_DIR}/broken: No such file or directory +Note: A fatal error was encountered: Something went wrong trying to read or write a file +No new mail." +rm "${MAIL_DIR}/broken" + + test_begin_subtest "New two-level directory" generate_message [dir]=two/levels -- cgit v1.2.3 From d99270c450d8f9ef3ecfbcbeeb99b581f36c9175 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 24 May 2012 18:01:11 -0400 Subject: new: Centralize file type stat-ing logic This moves our logic to get a file's type into one function. This has several benefits: we can support OSes and file systems that do not provide dirent.d_type or always return DT_UNKNOWN, complex symlink-handling logic has been replaced by a simple stat fall-through in one place, and the error message for un-stat-able file is more accurate (previously, the error always mentioned directories, even though a broken symlink is not a directory). --- notmuch-new.c | 103 +++++++++++++++++++++++++++++++++++----------------------- test/new | 2 +- 2 files changed, 64 insertions(+), 41 deletions(-) (limited to 'test/new') diff --git a/notmuch-new.c b/notmuch-new.c index 72dd558..c64f1a7 100644 --- a/notmuch-new.c +++ b/notmuch-new.c @@ -154,6 +154,48 @@ dirent_sort_strcmp_name (const struct dirent **a, const struct dirent **b) return strcmp ((*a)->d_name, (*b)->d_name); } +/* Return the type of a directory entry relative to path as a stat(2) + * mode. Like stat, this follows symlinks. Returns -1 and sets errno + * if the file's type cannot be determined (which includes dangling + * symlinks). + */ +static int +dirent_type (const char *path, const struct dirent *entry) +{ + struct stat statbuf; + char *abspath; + int err, saved_errno; + +#ifdef _DIRENT_HAVE_D_TYPE + /* Mapping from d_type to stat mode_t. We omit DT_LNK so that + * we'll fall through to stat and get the real file type. */ + static const mode_t modes[] = { + [DT_BLK] = S_IFBLK, + [DT_CHR] = S_IFCHR, + [DT_DIR] = S_IFDIR, + [DT_FIFO] = S_IFIFO, + [DT_REG] = S_IFREG, + [DT_SOCK] = S_IFSOCK + }; + if (entry->d_type < ARRAY_SIZE(modes) && modes[entry->d_type]) + return modes[entry->d_type]; +#endif + + abspath = talloc_asprintf (NULL, "%s/%s", path, entry->d_name); + if (!abspath) { + errno = ENOMEM; + return -1; + } + err = stat(abspath, &statbuf); + saved_errno = errno; + talloc_free (abspath); + if (err < 0) { + errno = saved_errno; + return -1; + } + return statbuf.st_mode & S_IFMT; +} + /* Test if the directory looks like a Maildir directory. * * Search through the array of directory entries to see if we can find all @@ -162,12 +204,12 @@ dirent_sort_strcmp_name (const struct dirent **a, const struct dirent **b) * Return 1 if the directory looks like a Maildir and 0 otherwise. */ static int -_entries_resemble_maildir (struct dirent **entries, int count) +_entries_resemble_maildir (const char *path, struct dirent **entries, int count) { int i, found = 0; for (i = 0; i < count; i++) { - if (entries[i]->d_type != DT_DIR && entries[i]->d_type != DT_UNKNOWN) + if (dirent_type (path, entries[i]) != S_IFDIR) continue; if (strcmp(entries[i]->d_name, "new") == 0 || @@ -250,7 +292,7 @@ add_files_recursive (notmuch_database_t *notmuch, notmuch_status_t status, ret = NOTMUCH_STATUS_SUCCESS; notmuch_message_t *message = NULL; struct dirent **fs_entries = NULL; - int i, num_fs_entries = 0; + int i, num_fs_entries = 0, entry_type; notmuch_directory_t *directory; notmuch_filenames_t *db_files = NULL; notmuch_filenames_t *db_subdirs = NULL; @@ -300,7 +342,7 @@ add_files_recursive (notmuch_database_t *notmuch, } /* Pass 1: Recurse into all sub-directories. */ - is_maildir = _entries_resemble_maildir (fs_entries, num_fs_entries); + is_maildir = _entries_resemble_maildir (path, fs_entries, num_fs_entries); for (i = 0; i < num_fs_entries; i++) { if (interrupted) @@ -308,17 +350,16 @@ add_files_recursive (notmuch_database_t *notmuch, entry = fs_entries[i]; - /* We only want to descend into directories. - * But symlinks can be to directories too, of course. - * - * And if the filesystem doesn't tell us the file type in the - * scandir results, then it might be a directory (and if not, - * then we'll stat and return immediately in the next level of - * recursion). */ - if (entry->d_type != DT_DIR && - entry->d_type != DT_LNK && - entry->d_type != DT_UNKNOWN) - { + /* We only want to descend into directories (and symlinks to + * directories). */ + entry_type = dirent_type (path, entry); + if (entry_type == -1) { + /* Be pessimistic, e.g. so we don't lose lots of mail just + * because a user broke a symlink. */ + fprintf (stderr, "Error reading file %s/%s: %s\n", + path, entry->d_name, strerror (errno)); + return NOTMUCH_STATUS_FILE_ERROR; + } else if (entry_type != S_IFDIR) { continue; } @@ -407,31 +448,13 @@ add_files_recursive (notmuch_database_t *notmuch, notmuch_filenames_move_to_next (db_subdirs); } - /* If we're looking at a symlink, we only want to add it if it - * links to a regular file, (and not to a directory, say). - * - * Similarly, if the file is of unknown type (due to filesystem - * limitations), then we also need to look closer. - * - * In either case, a stat does the trick. - */ - if (entry->d_type == DT_LNK || entry->d_type == DT_UNKNOWN) { - int err; - - next = talloc_asprintf (notmuch, "%s/%s", path, entry->d_name); - err = stat (next, &st); - talloc_free (next); - next = NULL; - - /* Don't emit an error for a link pointing nowhere, since - * the directory-traversal pass will have already done - * that. */ - if (err) - continue; - - if (! S_ISREG (st.st_mode)) - continue; - } else if (entry->d_type != DT_REG) { + /* Only add regular files (and symlinks to regular files). */ + entry_type = dirent_type (path, entry); + if (entry_type == -1) { + fprintf (stderr, "Error reading file %s/%s: %s\n", + path, entry->d_name, strerror (errno)); + return NOTMUCH_STATUS_FILE_ERROR; + } else if (entry_type != S_IFREG) { continue; } diff --git a/test/new b/test/new index a7bc146..cab7c01 100755 --- a/test/new +++ b/test/new @@ -140,7 +140,7 @@ test_begin_subtest "Broken symlink aborts" ln -s does-not-exist "${MAIL_DIR}/broken" output=$(NOTMUCH_NEW 2>&1) test_expect_equal "$output" \ -"Error reading directory ${MAIL_DIR}/broken: No such file or directory +"Error reading file ${MAIL_DIR}/broken: No such file or directory Note: A fatal error was encountered: Something went wrong trying to read or write a file No new mail." rm "${MAIL_DIR}/broken" -- cgit v1.2.3