feat: add background mail polling option for all workers

Check for new mail (recent, unseen, exists counts) with an external
command, or for imap with the STATUS command, at start or on
reconnection and every X time duration

IMAP:
The selected folder is skipped, per specification. Additional config
options are included for including/excluding folders explicitly.

Maildir/Notmuch:
An external command will be run in the background to check for new mail.
An optional timeout can be used with maildir/notmuch. Default is 10s

New account options:
check-mail
check-mail-cmd (maildir/notmuch only)
check-mail-timeout (maildir/notmuch only), default 10s
check-mail-include (IMAP only)
check-mail-exclude (IMAP only)

If unset, or set less than or equal to 0, check-mail will be ignored

Signed-off-by: Tim Culverhouse <tim@timculverhouse.com>
Tested-by: Moritz Poldrack <moritz@poldrack.dev>
Acked-by: Robin Jarry <robin@jarry.cc>

config/config.go

@@ -105,6 +105,13 @@ type AccountConfig struct {
 	EnableFoldersSort bool     `ini:"enable-folders-sort"`
 	FoldersSort       []string `ini:"folders-sort" delim:","`
 
+	// CheckMail
+	CheckMail        time.Duration `ini:"check-mail"`
+	CheckMailCmd     string        `ini:"check-mail-cmd"`
+	CheckMailTimeout time.Duration `ini:"check-mail-timeout"`
+	CheckMailInclude []string      `ini:"check-mail-include"`
+	CheckMailExclude []string      `ini:"check-mail-exclude"`
+
 	// PGP Config
 	PgpKeyId                string `ini:"pgp-key-id"`
 	PgpAutoSign             bool   `ini:"pgp-auto-sign"`
@@ -224,6 +231,7 @@ func loadAccountConfig(path string) ([]AccountConfig, error) {
 			Name:              _sec,
 			Params:            make(map[string]string),
 			EnableFoldersSort: true,
+			CheckMailTimeout:  10 * time.Second,
 		}
 		if err = sec.MapTo(&account); err != nil {
 			return nil, err

doc/aerc-config.5.scd

@@ -525,6 +525,20 @@ Note that many of these configuration options are written for you, such as
 
 	Default: Archive
 
+*check-mail*
+	Specifies an interval to check for new mail. Mail will be checked at
+	startup, and every interval. IMAP accounts will check for mail in all
+	unselected folders, and the selected folder will continue to receive PUSH
+	mail notifications. Maildir/Notmuch folders must use *check-mail-cmd* in
+	conjunction with this option. See *aerc-maildir* and *aerc-notmuch* for
+	more information.
+
+	Setting this option to 0 will disable check-mail
+
+	Example: 5m
+
+	Default: 0
+
 *copy-to*
 	Specifies a folder to copy sent mails to, usually "Sent".
 

doc/aerc-imap.5.scd

@@ -96,6 +96,21 @@ available:
 	This option is only supported on linux. On other platforms, it will be
 	ignored.
 
+*check-mail-include*
+	Specifies the comma separated list of folders to include when checking for
+	new mail with *check-mail*. Names prefixed with ~ are interpreted as regular
+	expressions.
+
+	Default: all folders
+
+*check-mail-exclude*
+	Specifies the comma separated list of folders to exclude when checking for
+	new mail with *check-mail*. Names prefixed with ~ are interpreted as regular
+	expressions.
+	Note that this overrides anything from *check-mail-include*.
+
+	Default: no folders
+
 # SEE ALSO
 
 *aerc*(1) *aerc-config*(5)

doc/aerc-maildir.5.scd

@@ -15,6 +15,21 @@ must be added manually to the *aerc-config*(5) file.
 
 The following maildir-specific options are available:
 
+*check-mail-cmd*
+	Command to run in conjunction with *check-mail* option.
+
+	Example:
+		mbsync -a
+
+	Default: none
+
+*check-mail-timeout*
+	Timeout for the *check-mail-cmd*. The command will be stopped if it does
+	not complete in this interval and an error will be displayed. Increase from
+	the default if repeated errors occur
+
+	Default: 10s
+
 *source*
 	maildir://path
 

doc/aerc-notmuch.5.scd

@@ -20,6 +20,21 @@ must be added manually.
 In accounts.conf (see *aerc-config*(5)), the following notmuch-specific
 options are available:
 
+*check-mail-cmd*
+	Command to run in conjunction with *check-mail* option.
+
+	Example:
+		mbsync -a
+
+	Default: none
+
+*check-mail-timeout*
+	Timeout for the *check-mail-cmd*. The command will be stopped if it does
+	not complete in this interval and an error will be displayed. Increase from
+	the default if repeated errors occur
+
+	Default: 10s
+
 *source*
 	notmuch://path
 

lib/msgstore.go

@@ -193,7 +193,9 @@ func (store *MessageStore) Update(msg types.WorkerMessage) {
 	switch msg := msg.(type) {
 	case *types.DirectoryInfo:
 		store.DirInfo = *msg.Info
+		if !msg.SkipSort {
 			store.Sort(store.sortCriteria, nil)
+		}
 		update = true
 	case *types.DirectoryContents:
 		newMap := make(map[uint32]*models.MessageInfo)

lib/statusline/state.go

@@ -64,9 +64,13 @@ func (s *State) SetWidth(w int) bool {
 	return changeState
 }
 
+func (s *State) Connected() bool {
+	return s.acct.Connected
+}
+
 type SetStateFunc func(s *State, folder string)
 
-func Connected(state bool) SetStateFunc {
+func SetConnected(state bool) SetStateFunc {
 	return func(s *State, folder string) {
 		s.acct.ConnActivity = ""
 		s.acct.Connected = state

widgets/account.go

@@ -33,6 +33,7 @@ type AccountView struct {
 	msglist *MessageList
 	worker  *types.Worker
 	state   *statusline.State
+	newConn bool // True if this is a first run after a new connection/reconnection
 }
 
 func (acct *AccountView) UiConfig() config.UIConfig {
@@ -100,6 +101,9 @@ func NewAccountView(aerc *Aerc, conf *config.AercConfig, acct *config.AccountCon
 	worker.PostAction(&types.Configure{Config: acct}, nil)
 	worker.PostAction(&types.Connect{}, nil)
 	view.SetStatus(statusline.ConnectionActivity("Connecting..."))
+	if acct.CheckMail.Minutes() > 0 {
+		view.CheckMailTimer(acct.CheckMail)
+	}
 
 	return view, nil
 }
@@ -258,13 +262,14 @@ func (acct *AccountView) onMessage(msg types.WorkerMessage) {
 				}
 				acct.msglist.SetInitDone()
 				acct.logger.Println("Connected.")
-				acct.SetStatus(statusline.Connected(true))
+				acct.SetStatus(statusline.SetConnected(true))
+				acct.newConn = true
 			})
 		case *types.Disconnect:
 			acct.dirlist.UpdateList(nil)
 			acct.msglist.SetStore(nil)
 			acct.logger.Println("Disconnected.")
-			acct.SetStatus(statusline.Connected(false))
+			acct.SetStatus(statusline.SetConnected(false))
 		case *types.OpenDirectory:
 			if store, ok := acct.dirlist.SelectedMsgStore(); ok {
 				// If we've opened this dir before, we can re-render it from
@@ -279,6 +284,11 @@ func (acct *AccountView) onMessage(msg types.WorkerMessage) {
 			acct.dirlist.UpdateList(nil)
 		case *types.RemoveDirectory:
 			acct.dirlist.UpdateList(nil)
+		case *types.FetchMessageHeaders:
+			if acct.newConn && acct.AccountConfig().CheckMail.Minutes() > 0 {
+				acct.newConn = false
+				acct.CheckMail()
+			}
 		}
 	case *types.DirectoryInfo:
 		if store, ok := acct.dirlist.MsgStore(msg.Info.Name); ok {
@@ -327,7 +337,7 @@ func (acct *AccountView) onMessage(msg types.WorkerMessage) {
 		acct.labels = msg.Labels
 	case *types.ConnError:
 		acct.logger.Printf("connection error: [%s] %v", acct.acct.Name, msg.Error)
-		acct.SetStatus(statusline.Connected(false))
+		acct.SetStatus(statusline.SetConnected(false))
 		acct.PushError(msg.Error)
 		acct.msglist.SetStore(nil)
 		acct.worker.PostAction(&types.Reconnect{}, nil)
@@ -349,3 +359,33 @@ func (acct *AccountView) GetSortCriteria() []*types.SortCriterion {
 	}
 	return criteria
 }
+
+func (acct *AccountView) CheckMail() {
+	// Exclude selected mailbox, per IMAP specification
+	exclude := append(acct.AccountConfig().CheckMailExclude, acct.dirlist.Selected())
+	dirs := acct.dirlist.List()
+	dirs = acct.dirlist.FilterDirs(dirs, acct.AccountConfig().CheckMailInclude, false)
+	dirs = acct.dirlist.FilterDirs(dirs, exclude, true)
+	acct.logger.Printf("Checking for new mail on account %s", acct.Name())
+	acct.SetStatus(statusline.ConnectionActivity("Checking for new mail..."))
+	msg := &types.CheckMail{
+		Directories: dirs,
+		Command:     acct.acct.CheckMailCmd,
+		Timeout:     acct.acct.CheckMailTimeout,
+	}
+	acct.worker.PostAction(msg, func(_ types.WorkerMessage) {
+		acct.SetStatus(statusline.ConnectionActivity(""))
+	})
+}
+
+func (acct *AccountView) CheckMailTimer(d time.Duration) {
+	ticker := time.NewTicker(d)
+	go func() {
+		for range ticker.C {
+			if !acct.state.Connected() {
+				continue
+			}
+			acct.CheckMail()
+		}
+	}()
+}

widgets/dirlist.go

@@ -40,6 +40,8 @@ type DirectoryLister interface {
 	SelectedMsgStore() (*lib.MessageStore, bool)
 	MsgStore(string) (*lib.MessageStore, bool)
 	SetMsgStore(string, *lib.MessageStore)
+
+	FilterDirs([]string, []string, bool) []string
 }
 
 type DirectoryList struct {
@@ -441,7 +443,21 @@ func (dirlist *DirectoryList) sortDirsByFoldersSortConfig() {
 // dirstore, based on AccountConfig.Folders (inclusion) and
 // AccountConfig.FoldersExclude (exclusion), in that order.
 func (dirlist *DirectoryList) filterDirsByFoldersConfig() {
-	filterDirs := func(orig, filters []string, exclude bool) []string {
+	dirlist.dirs = dirlist.store.List()
+
+	// 'folders' (if available) is used to make the initial list and
+	// 'folders-exclude' removes from that list.
+	configFolders := dirlist.acctConf.Folders
+	dirlist.dirs = dirlist.FilterDirs(dirlist.dirs, configFolders, false)
+
+	configFoldersExclude := dirlist.acctConf.FoldersExclude
+	dirlist.dirs = dirlist.FilterDirs(dirlist.dirs, configFoldersExclude, true)
+}
+
+// FilterDirs filters directories by the supplied filter. If exclude is false,
+// the filter will only include directories from orig which exist in filters.
+// If exclude is true, the directories in filters are removed from orig
+func (dirlist *DirectoryList) FilterDirs(orig, filters []string, exclude bool) []string {
 	if len(filters) == 0 {
 		return orig
 	}
@@ -464,17 +480,6 @@ func (dirlist *DirectoryList) filterDirsByFoldersConfig() {
 	return dest
 }
 
-	dirlist.dirs = dirlist.store.List()
-
-	// 'folders' (if available) is used to make the initial list and
-	// 'folders-exclude' removes from that list.
-	configFolders := dirlist.acctConf.Folders
-	dirlist.dirs = filterDirs(dirlist.dirs, configFolders, false)
-
-	configFoldersExclude := dirlist.acctConf.FoldersExclude
-	dirlist.dirs = filterDirs(dirlist.dirs, configFoldersExclude, true)
-}
-
 func (dirlist *DirectoryList) SelectedMsgStore() (*lib.MessageStore, bool) {
 	return dirlist.store.MessageStore(dirlist.selected)
 }

worker/imap/checkmail.go

@@ -0,0 +1,40 @@
+package imap
+
+import (
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-imap"
+)
+
+func (w *IMAPWorker) handleCheckMailMessage(msg *types.CheckMail) {
+	items := []imap.StatusItem{
+		imap.StatusMessages,
+		imap.StatusRecent,
+		imap.StatusUnseen,
+	}
+	for _, dir := range msg.Directories {
+		w.worker.Logger.Printf("Getting status of directory %s", dir)
+		status, err := w.client.Status(dir, items)
+		if err != nil {
+			w.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+		} else {
+			w.worker.PostMessage(&types.DirectoryInfo{
+				Info: &models.DirectoryInfo{
+					Flags:          status.Flags,
+					Name:           status.Name,
+					ReadOnly:       status.ReadOnly,
+					AccurateCounts: true,
+
+					Exists: int(status.Messages),
+					Recent: int(status.Recent),
+					Unseen: int(status.Unseen),
+				},
+				SkipSort: true,
+			}, nil)
+		}
+	}
+	w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+}

worker/imap/worker.go

@@ -190,6 +190,8 @@ func (w *IMAPWorker) handleMessage(msg types.WorkerMessage) error {
 		w.handleAppendMessage(msg)
 	case *types.SearchDirectory:
 		w.handleSearchDirectory(msg)
+	case *types.CheckMail:
+		w.handleCheckMailMessage(msg)
 	default:
 		reterr = errUnsupported
 	}

worker/maildir/worker.go

@@ -2,12 +2,14 @@ package maildir
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"sort"
 	"strings"
@@ -60,6 +62,13 @@ func (w *Worker) Run() {
 
 func (w *Worker) handleAction(action types.WorkerMessage) {
 	msg := w.worker.ProcessAction(action)
+	switch msg := msg.(type) {
+	// Explicitly handle all asynchronous actions. Async actions are
+	// responsible for posting their own Done message
+	case *types.CheckMail:
+		go w.handleCheckMail(msg)
+	default:
+		// Default handling, will be performed synchronously
 		if err := w.handleMessage(msg); err == errUnsupported {
 			w.worker.PostMessage(&types.Unsupported{
 				Message: types.RespondTo(msg),
@@ -73,6 +82,7 @@ func (w *Worker) handleAction(action types.WorkerMessage) {
 			w.done(msg)
 		}
 	}
+}
 
 func (w *Worker) handleFSEvent(ev fsnotify.Event) {
 	// we only care about files being created, removed or renamed
@@ -672,3 +682,24 @@ func (w *Worker) msgInfoFromUid(uid uint32) (*models.MessageInfo, error) {
 	}
 	return info, nil
 }
+
+func (w *Worker) handleCheckMail(msg *types.CheckMail) {
+	ctx, cancel := context.WithTimeout(context.Background(), msg.Timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "sh", "-c", msg.Command)
+	ch := make(chan error)
+	go func() {
+		err := cmd.Run()
+		ch <- err
+	}()
+	select {
+	case <-ctx.Done():
+		w.err(msg, fmt.Errorf("checkmail: timed out"))
+	case err := <-ch:
+		if err != nil {
+			w.err(msg, fmt.Errorf("checkmail: error running command: %v", err))
+		} else {
+			w.done(msg)
+		}
+	}
+}

worker/notmuch/worker.go

@@ -6,10 +6,12 @@ package notmuch
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -128,6 +130,9 @@ func (w *worker) handleMessage(msg types.WorkerMessage) error {
 		return w.handleSearchDirectory(msg)
 	case *types.ModifyLabels:
 		return w.handleModifyLabels(msg)
+	case *types.CheckMail:
+		go w.handleCheckMail(msg)
+		return nil
 
 		// not implemented, they are generally not used
 		// in a notmuch based workflow
@@ -616,3 +621,24 @@ func (w *worker) sort(uids []uint32,
 	}
 	return sortedUids, nil
 }
+
+func (w *worker) handleCheckMail(msg *types.CheckMail) {
+	ctx, cancel := context.WithTimeout(context.Background(), msg.Timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "sh", "-c", msg.Command)
+	ch := make(chan error)
+	go func() {
+		err := cmd.Run()
+		ch <- err
+	}()
+	select {
+	case <-ctx.Done():
+		w.err(msg, fmt.Errorf("checkmail: timed out"))
+	case err := <-ch:
+		if err != nil {
+			w.err(msg, fmt.Errorf("checkmail: error running command: %v", err))
+		} else {
+			w.done(msg)
+		}
+	}
+}

worker/types/messages.go

@@ -167,6 +167,13 @@ type AppendMessage struct {
 	Length      int
 }
 
+type CheckMail struct {
+	Message
+	Directories []string
+	Command     string
+	Timeout     time.Duration
+}
+
 // Messages
 
 type Directory struct {
@@ -177,6 +184,7 @@ type Directory struct {
 type DirectoryInfo struct {
 	Message
 	Info     *models.DirectoryInfo
+	SkipSort bool
 }
 
 type DirectoryContents struct {