diff --git a/worker/maildir/container.go b/worker/maildir/container.go
index 14815c9..1bdc4e7 100644
--- a/worker/maildir/container.go
+++ b/worker/maildir/container.go
@@ -15,9 +15,10 @@ import (
 // A Container is a directory which contains other directories which adhere to
 // the Maildir spec
 type Container struct {
-	dir  string
-	log  *log.Logger
-	uids *uidstore.Store
+	dir        string
+	log        *log.Logger
+	uids       *uidstore.Store
+	recentUIDS map[uint32]struct{} // used to set the recent flag
 }
 
 // NewContainer creates a new container at the specified directory
@@ -34,7 +35,8 @@ func NewContainer(dir string, l *log.Logger) (*Container, error) {
 	if !s.IsDir() {
 		return nil, fmt.Errorf("Given maildir '%s' not a directory", dir)
 	}
-	return &Container{dir: dir, uids: uidstore.NewStore(), log: l}, nil
+	return &Container{dir: dir, uids: uidstore.NewStore(), log: l,
+		recentUIDS: make(map[uint32]struct{})}, nil
 }
 
 // ListFolders returns a list of maildir folders in the container
@@ -72,17 +74,26 @@ func (c *Container) ListFolders() ([]string, error) {
 	return folders, err
 }
 
+// SyncNewMail adds emails from new to cur, tracking them
+func (c *Container) SyncNewMail(dir maildir.Dir) error {
+	keys, err := dir.Unseen()
+	if err != nil {
+		return err
+	}
+	for _, key := range keys {
+		uid := c.uids.GetOrInsert(key)
+		c.recentUIDS[uid] = struct{}{}
+	}
+	return nil
+}
+
 // OpenDirectory opens an existing maildir in the container by name, moves new
 // messages into cur, and registers the new keys in the UIDStore.
 func (c *Container) OpenDirectory(name string) (maildir.Dir, error) {
 	dir := c.Dir(name)
-	keys, err := dir.Unseen()
-	if err != nil {
+	if err := c.SyncNewMail(dir); err != nil {
 		return dir, err
 	}
-	for _, key := range keys {
-		c.uids.GetOrInsert(key)
-	}
 	return dir, nil
 }
 
@@ -91,6 +102,17 @@ func (c *Container) Dir(name string) maildir.Dir {
 	return maildir.Dir(filepath.Join(c.dir, name))
 }
 
+// IsRecent returns if a uid has the Recent flag set
+func (c *Container) IsRecent(uid uint32) bool {
+	_, ok := c.recentUIDS[uid]
+	return ok
+}
+
+// ClearRecentFlag removes the Recent flag from the message with the given uid
+func (c *Container) ClearRecentFlag(uid uint32) {
+	delete(c.recentUIDS, uid)
+}
+
 // UIDs fetches the unique message identifiers for the maildir
 func (c *Container) UIDs(d maildir.Dir) ([]uint32, error) {
 	keys, err := d.Keys()
diff --git a/worker/maildir/worker.go b/worker/maildir/worker.go
index 87ebc97..3f84e01 100644
--- a/worker/maildir/worker.go
+++ b/worker/maildir/worker.go
@@ -79,11 +79,12 @@ func (w *Worker) handleFSEvent(ev fsnotify.Event) {
 	if w.selected == nil {
 		return
 	}
-	newUnseen, err := w.selected.Unseen()
+	err := w.c.SyncNewMail(*w.selected)
 	if err != nil {
 		w.worker.Logger.Printf("could not move new to cur : %v", err)
 		return
 	}
+
 	uids, err := w.c.UIDs(*w.selected)
 	if err != nil {
 		w.worker.Logger.Printf("could not scan UIDs: %v", err)
@@ -98,7 +99,6 @@ func (w *Worker) handleFSEvent(ev fsnotify.Event) {
 		Uids: sortedUids,
 	}, nil)
 	dirInfo := w.getDirectoryInfo(w.selectedName)
-	dirInfo.Recent = len(newUnseen)
 	w.worker.PostMessage(&types.DirectoryInfo{
 		Info: dirInfo,
 	}, nil)
@@ -138,12 +138,6 @@ func (w *Worker) getDirectoryInfo(name string) *models.DirectoryInfo {
 		return dirInfo
 	}
 
-	recent, err := dir.UnseenCount()
-	if err != nil {
-		w.worker.Logger.Printf("could not get unseen count: %v", err)
-	}
-	dirInfo.Recent = recent
-
 	for _, uid := range uids {
 		message, err := w.c.Message(dir, uid)
 		if err != nil {
@@ -164,9 +158,12 @@ func (w *Worker) getDirectoryInfo(name string) *models.DirectoryInfo {
 		if !seen {
 			dirInfo.Unseen++
 		}
+		if w.c.IsRecent(uid) {
+			dirInfo.Recent++
+		}
 	}
 	dirInfo.Unseen += dirInfo.Recent
-	dirInfo.Exists = len(uids) + recent
+	dirInfo.Exists = len(uids) + dirInfo.Recent
 	return dirInfo
 }
 
@@ -332,12 +329,7 @@ func (w *Worker) sort(uids []uint32, criteria []*types.SortCriterion) ([]uint32,
 	}
 	var msgInfos []*models.MessageInfo
 	for _, uid := range uids {
-		m, err := w.c.Message(*w.selected, uid)
-		if err != nil {
-			w.worker.Logger.Printf("could not get message: %v", err)
-			continue
-		}
-		info, err := m.MessageInfo()
+		info, err := w.msgInfoFromUid(uid)
 		if err != nil {
 			w.worker.Logger.Printf("could not get message info: %v", err)
 			continue
@@ -375,13 +367,7 @@ func (w *Worker) handleRemoveDirectory(msg *types.RemoveDirectory) error {
 func (w *Worker) handleFetchMessageHeaders(
 	msg *types.FetchMessageHeaders) error {
 	for _, uid := range msg.Uids {
-		m, err := w.c.Message(*w.selected, uid)
-		if err != nil {
-			w.worker.Logger.Printf("could not get message: %v", err)
-			w.err(msg, err)
-			continue
-		}
-		info, err := m.MessageInfo()
+		info, err := w.msgInfoFromUid(uid)
 		if err != nil {
 			w.worker.Logger.Printf("could not get message info: %v", err)
 			w.err(msg, err)
@@ -391,6 +377,7 @@ func (w *Worker) handleFetchMessageHeaders(
 			Message: types.RespondTo(msg),
 			Info:    info,
 		}, nil)
+		w.c.ClearRecentFlag(uid)
 	}
 	return nil
 }
@@ -588,3 +575,18 @@ func (w *Worker) handleSearchDirectory(msg *types.SearchDirectory) error {
 	}, nil)
 	return nil
 }
+
+func (w *Worker) msgInfoFromUid(uid uint32) (*models.MessageInfo, error) {
+	m, err := w.c.Message(*w.selected, uid)
+	if err != nil {
+		return nil, err
+	}
+	info, err := m.MessageInfo()
+	if err != nil {
+		return nil, err
+	}
+	if w.c.IsRecent(uid) {
+		info.Flags = append(info.Flags, models.RecentFlag)
+	}
+	return info, nil
+}