notmuch: add maildir support

By associating the notmuch database with a maildir store, we can add the
Copy/Move/Delete operations on messages to the notmuch backend.
This change assumes that the notmuch database location is also the root
of the maildir store.

In a previous change, we added the ability to dynamically add and remove
message files to the notmuch DB. This change uses this facility to
synchronize the database with the filesystem operations on maildir
files.

While it's still possible to use the query-map file to create virtual
folders from notmuch search queries, the sidebar is now loaded with the
folders found in the maildir store.

With notmuch, two identical but distinct message files can be indexed in
the database with the same key. This change takes extra care of only
deleting or removing message files from the maildir corresponding to the
folder that is currently selected (if any).

Implements: https://todo.sr.ht/~rjarry/aerc/88
Fixes: https://todo.sr.ht/~rjarry/aerc/73
Signed-off-by: Julian Pidancet <julian.pidancet@oracle.com>
Acked-by: Robin Jarry <robin@jarry.cc>
Acked-by: Tim Culverhouse <tim@timculverhouse.com>

CHANGELOG.md

@@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 
 - View common email envelope headers with `:envelope`.
+- Notmuch accounts now support maildir operations: `:copy`, `:move`, `:mkdir`,
+  `:rmdir`, `:archive` and the `copy-to` option.
 
 ### Fixed
 

doc/aerc.1.scd

@@ -307,8 +307,6 @@ message list, the message in the message viewer, etc).
 *mkdir* <name>
 	Creates a new folder for this account and changes to that folder.
 
-	This is not supported on the 'notmuch' backend.
-
 *rmdir* [-f]
 	Removes the current folder.
 
@@ -317,8 +315,6 @@ message list, the message in the message viewer, etc).
 	*-f*
 		Remove the directory even if it contains messages.
 
-	This is not supported on the 'notmuch' backend.
-
 	Some programs that sync maildirs may recover deleted directories (e.g.
 	offlineimap).  These can either be specially configured to properly
 	handle directory deletion, or special commands need to be run to delete

worker/notmuch/eventhandlers.go

@@ -3,7 +3,12 @@
 
 package notmuch
 
-import "git.sr.ht/~rjarry/aerc/logging"
+import (
+	"fmt"
+	"strconv"
+
+	"git.sr.ht/~rjarry/aerc/logging"
+)
 
 func (w *worker) handleNotmuchEvent(et eventType) error {
 	switch ev := et.(type) {
@@ -15,6 +20,21 @@ func (w *worker) handleNotmuchEvent(et eventType) error {
 }
 
 func (w *worker) handleUpdateDirCounts(ev eventType) error {
+	folders, err := w.store.FolderMap()
+	if err != nil {
+		logging.Errorf("failed listing directories: %v", err)
+		return err
+	}
+	for name := range folders {
+		query := fmt.Sprintf("folder:%s", strconv.Quote(name))
+		info, err := w.buildDirInfo(name, query, true)
+		if err != nil {
+			logging.Errorf("could not gather DirectoryInfo: %v", err)
+			continue
+		}
+		w.w.PostMessage(info, nil)
+	}
+
 	for name, query := range w.nameQueryMap {
 		info, err := w.buildDirInfo(name, query, true)
 		if err != nil {

worker/notmuch/message.go

@@ -7,6 +7,10 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/emersion/go-maildir"
 
 	"git.sr.ht/~rjarry/aerc/models"
 	"git.sr.ht/~rjarry/aerc/worker/lib"
@@ -175,3 +179,108 @@ func (m *Message) RemoveTag(tag string) error {
 func (m *Message) ModifyTags(add, remove []string) error {
 	return m.db.MsgModifyTags(m.key, add, remove)
 }
+
+func (m *Message) Remove(dir maildir.Dir) error {
+	filenames, err := m.db.MsgFilenames(m.key)
+	if err != nil {
+		return err
+	}
+	for _, filename := range filenames {
+		if dirContains(dir, filename) {
+			err := m.db.DeleteMessage(filename)
+			if err != nil {
+				return err
+			}
+
+			if err := os.Remove(filename); err != nil {
+				return err
+			}
+
+			return nil
+		}
+	}
+
+	return fmt.Errorf("no matching message file found in %s", string(dir))
+}
+
+func (m *Message) Copy(target maildir.Dir) error {
+	filename, err := m.Filename()
+	if err != nil {
+		return err
+	}
+
+	source, key := parseFilename(filename)
+	if key == "" {
+		return fmt.Errorf("failed to parse message filename: %s", filename)
+	}
+
+	newKey, err := source.Copy(target, key)
+	if err != nil {
+		return err
+	}
+	newFilename, err := target.Filename(newKey)
+	if err != nil {
+		return err
+	}
+	_, err = m.db.IndexFile(newFilename)
+	return err
+}
+
+func (m *Message) Move(srcDir, destDir maildir.Dir) error {
+	var src string
+
+	filenames, err := m.db.MsgFilenames(m.key)
+	if err != nil {
+		return err
+	}
+	for _, filename := range filenames {
+		if dirContains(srcDir, filename) {
+			src = filename
+			break
+		}
+	}
+
+	if src == "" {
+		return fmt.Errorf("no matching message file found in %s", string(srcDir))
+	}
+
+	// Remove encoded UID information from the key to prevent sync issues
+	name := lib.StripUIDFromMessageFilename(filepath.Base(src))
+	dest := filepath.Join(string(destDir), "cur", name)
+
+	if err := m.db.DeleteMessage(src); err != nil {
+		return err
+	}
+
+	if err := os.Rename(src, dest); err != nil {
+		return err
+	}
+
+	_, err = m.db.IndexFile(dest)
+	return err
+}
+
+func parseFilename(filename string) (maildir.Dir, string) {
+	base := filepath.Base(filename)
+	dir := filepath.Dir(filename)
+	dir, curdir := filepath.Split(dir)
+	if curdir != "cur" {
+		return "", ""
+	}
+	split := strings.Split(base, ":")
+	if len(split) < 2 {
+		return maildir.Dir(dir), ""
+	}
+	key := split[0]
+	return maildir.Dir(dir), key
+}
+
+func dirContains(dir maildir.Dir, filename string) bool {
+	for _, sub := range []string{"cur", "new"} {
+		match, _ := filepath.Match(filepath.Join(string(dir), sub, "*"), filename)
+		if match {
+			return true
+		}
+	}
+	return false
+}

worker/notmuch/worker.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -42,6 +43,7 @@ type worker struct {
 	currentQueryName    string
 	queryMapOrder       []string
 	nameQueryMap        map[string]string
+	store               *lib.MaildirStore
 	db                  *notmuch.DB
 	setupErr            error
 	currentSortCriteria []*types.SortCriterion
@@ -135,13 +137,18 @@ func (w *worker) handleMessage(msg types.WorkerMessage) error {
 	case *types.CheckMail:
 		go w.handleCheckMail(msg)
 		return nil
-		// not implemented, they are generally not used
+	case *types.DeleteMessages:
-		// in a notmuch based workflow
+		return w.handleDeleteMessages(msg)
-		// case *types.DeleteMessages:
+	case *types.CopyMessages:
-		// case *types.CopyMessages:
+		return w.handleCopyMessages(msg)
-		// case *types.AppendMessage:
+	case *types.MoveMessages:
-		// case *types.CreateDirectory:
+		return w.handleMoveMessages(msg)
-		// case *types.RemoveDirectory:
+	case *types.AppendMessage:
+		return w.handleAppendMessage(msg)
+	case *types.CreateDirectory:
+		return w.handleCreateDirectory(msg)
+	case *types.RemoveDirectory:
+		return w.handleRemoveDirectory(msg)
 	}
 	return errUnsupported
 }
@@ -172,6 +179,12 @@ func (w *worker) handleConfigure(msg *types.Configure) error {
 	}
 	excludedTags := w.loadExcludeTags(msg.Config)
 	w.db = notmuch.NewDB(pathToDB, excludedTags)
+	store, err := lib.NewMaildirStore(pathToDB, false)
+	if err != nil {
+		return fmt.Errorf("Cannot initialize maildir store: %w", err)
+	}
+	w.store = store
+
 	return nil
 }
 
@@ -194,6 +207,21 @@ func (w *worker) handleConnect(msg *types.Connect) error {
 }
 
 func (w *worker) handleListDirectories(msg *types.ListDirectories) error {
+	folders, err := w.store.FolderMap()
+	if err != nil {
+		logging.Errorf("failed listing directories: %v", err)
+		return err
+	}
+	for name := range folders {
+		w.w.PostMessage(&types.Directory{
+			Message: types.RespondTo(msg),
+			Dir: &models.Directory{
+				Name:       name,
+				Attributes: []string{},
+			},
+		}, nil)
+	}
+
 	for _, name := range w.queryMapOrder {
 		w.w.PostMessage(&types.Directory{
 			Message: types.RespondTo(msg),
@@ -259,6 +287,10 @@ func (w *worker) queryFromName(name string) (string, bool) {
 	// try the friendly name first, if that fails assume it's a query
 	q, ok := w.nameQueryMap[name]
 	if !ok {
+		folders, _ := w.store.FolderMap()
+		if _, ok := folders[name]; ok {
+			return fmt.Sprintf("folder:%s", strconv.Quote(name)), true
+		}
 		return name, true
 	}
 	return q, false
@@ -679,3 +711,171 @@ func (w *worker) handleCheckMail(msg *types.CheckMail) {
 		}
 	}
 }
+
+func (w *worker) handleDeleteMessages(msg *types.DeleteMessages) error {
+	var deleted []uint32
+
+	// With notmuch, two identical files can be referenced under
+	// the same index key, even if they exist in two different
+	// folders. So in order to remove the message from the right
+	// maildir folder we need to pass a hint to Remove() so it
+	// can purge the right file.
+	folders, _ := w.store.FolderMap()
+	path, ok := folders[w.currentQueryName]
+	if !ok {
+		w.err(msg, fmt.Errorf("Can only delete file from a maildir folder"))
+		w.done(msg)
+		return nil
+	}
+
+	for _, uid := range msg.Uids {
+		m, err := w.msgFromUid(uid)
+		if err != nil {
+			logging.Errorf("could not get message: %v", err)
+			w.err(msg, err)
+			continue
+		}
+		if err := m.Remove(path); err != nil {
+			logging.Errorf("could not remove message: %v", err)
+			w.err(msg, err)
+			continue
+		}
+		deleted = append(deleted, uid)
+	}
+	if len(deleted) > 0 {
+		w.w.PostMessage(&types.MessagesDeleted{
+			Message: types.RespondTo(msg),
+			Uids:    deleted,
+		}, nil)
+	}
+	w.done(msg)
+	return nil
+}
+
+func (w *worker) handleCopyMessages(msg *types.CopyMessages) error {
+	// Only allow file to be copied to a maildir folder
+	folders, _ := w.store.FolderMap()
+	dest, ok := folders[msg.Destination]
+	if !ok {
+		return fmt.Errorf("Can only move file to a maildir folder")
+	}
+
+	for _, uid := range msg.Uids {
+		m, err := w.msgFromUid(uid)
+		if err != nil {
+			logging.Errorf("could not get message: %v", err)
+			return err
+		}
+		if err := m.Copy(dest); err != nil {
+			logging.Errorf("could not copy message: %v", err)
+			return err
+		}
+	}
+	w.w.PostMessage(&types.MessagesCopied{
+		Message:     types.RespondTo(msg),
+		Destination: msg.Destination,
+		Uids:        msg.Uids,
+	}, nil)
+	w.done(msg)
+	return nil
+}
+
+func (w *worker) handleMoveMessages(msg *types.MoveMessages) error {
+	var moved []uint32
+
+	// With notmuch, two identical files can be referenced under
+	// the same index key, even if they exist in two different
+	// folders. So in order to remove the message from the right
+	// maildir folder we need to pass a hint to Move() so it
+	// can act on the right file.
+	folders, _ := w.store.FolderMap()
+	source, ok := folders[w.currentQueryName]
+	if !ok {
+		return fmt.Errorf("Can only move file from a maildir folder")
+	}
+
+	// Only allow file to be moved to a maildir folder
+	dest, ok := folders[msg.Destination]
+	if !ok {
+		return fmt.Errorf("Can only move file to a maildir folder")
+	}
+
+	var err error
+	for _, uid := range msg.Uids {
+		m, err := w.msgFromUid(uid)
+		if err != nil {
+			logging.Errorf("could not get message: %v", err)
+			break
+		}
+		if err := m.Move(source, dest); err != nil {
+			logging.Errorf("could not copy message: %v", err)
+			break
+		}
+		moved = append(moved, uid)
+	}
+	w.w.PostMessage(&types.MessagesDeleted{
+		Message: types.RespondTo(msg),
+		Uids:    moved,
+	}, nil)
+	if err == nil {
+		w.done(msg)
+	}
+	return err
+}
+
+func (w *worker) handleAppendMessage(msg *types.AppendMessage) error {
+	// Only allow file to be created in a maildir folder
+	// since we are the "master" maildir process, we can modify the maildir directly
+	folders, _ := w.store.FolderMap()
+	dest, ok := folders[msg.Destination]
+	if !ok {
+		return fmt.Errorf("Can only create file in a maildir folder")
+	}
+	key, writer, err := dest.Create(lib.ToMaildirFlags(msg.Flags))
+	if err != nil {
+		logging.Errorf("could not create message at %s: %v", msg.Destination, err)
+		return err
+	}
+	filename, err := dest.Filename(key)
+	if err != nil {
+		writer.Close()
+		return err
+	}
+	if _, err := io.Copy(writer, msg.Reader); err != nil {
+		logging.Errorf("could not write message to destination: %v", err)
+		writer.Close()
+		os.Remove(filename)
+		return err
+	}
+	writer.Close()
+	if _, err := w.db.IndexFile(filename); err != nil {
+		return err
+	}
+	if err := w.emitDirectoryInfo(w.currentQueryName); err != nil {
+		logging.Errorf("could not emit directory info: %v", err)
+	}
+	w.done(msg)
+	return nil
+}
+
+func (w *worker) handleCreateDirectory(msg *types.CreateDirectory) error {
+	dir := w.store.Dir(msg.Directory)
+	if err := dir.Init(); err != nil {
+		logging.Errorf("could not create directory %s: %v",
+			msg.Directory, err)
+		return err
+	}
+	w.done(msg)
+	return nil
+}
+
+func (w *worker) handleRemoveDirectory(msg *types.RemoveDirectory) error {
+	dir := w.store.Dir(msg.Directory)
+	if err := os.RemoveAll(string(dir)); err != nil {
+		logging.Errorf("could not remove directory %s: %v",
+			msg.Directory, err)
+		return err
+	}
+	w.done(msg)
+	return nil
+}