diff --git a/commands/msg/reply.go b/commands/msg/reply.go
index 7a64d21..68c2089 100644
--- a/commands/msg/reply.go
+++ b/commands/msg/reply.go
@@ -9,12 +9,11 @@ import (
 	"strings"
 
 	"git.sr.ht/~sircmpwn/getopt"
-	"github.com/emersion/go-imap"
 	"github.com/emersion/go-message"
 	_ "github.com/emersion/go-message/charset"
 	"github.com/emersion/go-message/mail"
 
-	"git.sr.ht/~sircmpwn/aerc/lib"
+	"git.sr.ht/~sircmpwn/aerc/models"
 	"git.sr.ht/~sircmpwn/aerc/widgets"
 )
 
@@ -67,7 +66,7 @@ func (_ reply) Execute(aerc *widgets.Aerc, args []string) error {
 	var (
 		to     []string
 		cc     []string
-		toList []*imap.Address
+		toList []*models.Address
 	)
 	if args[0] == "reply" {
 		if len(msg.Envelope.ReplyTo) != 0 {
@@ -76,24 +75,23 @@ func (_ reply) Execute(aerc *widgets.Aerc, args []string) error {
 			toList = msg.Envelope.From
 		}
 		for _, addr := range toList {
-			if addr.PersonalName != "" {
+			if addr.Name != "" {
 				to = append(to, fmt.Sprintf("%s <%s@%s>",
-					addr.PersonalName, addr.MailboxName, addr.HostName))
+					addr.Name, addr.Mailbox, addr.Host))
 			} else {
-				to = append(to, fmt.Sprintf("<%s@%s>",
-					addr.MailboxName, addr.HostName))
+				to = append(to, fmt.Sprintf("<%s@%s>", addr.Mailbox, addr.Host))
 			}
 		}
 		if replyAll {
 			for _, addr := range msg.Envelope.Cc {
-				cc = append(cc, lib.FormatAddress(addr))
+				cc = append(cc, addr.Format())
 			}
 			for _, addr := range msg.Envelope.To {
-				address := fmt.Sprintf("%s@%s", addr.MailboxName, addr.HostName)
+				address := fmt.Sprintf("%s@%s", addr.Mailbox, addr.Host)
 				if address == us.Address {
 					continue
 				}
-				to = append(to, lib.FormatAddress(addr))
+				to = append(to, addr.Format())
 			}
 		}
 	}
@@ -163,7 +161,7 @@ func (_ reply) Execute(aerc *widgets.Aerc, args []string) error {
 			go composer.SetContents(pipeout)
 			// TODO: Let user customize the date format used here
 			io.WriteString(pipein, fmt.Sprintf("Forwarded message from %s on %s:\n\n",
-				msg.Envelope.From[0].PersonalName,
+				msg.Envelope.From[0].Name,
 				msg.Envelope.Date.Format("Mon Jan 2, 2006 at 3:04 PM")))
 			for scanner.Scan() {
 				io.WriteString(pipein, fmt.Sprintf("%s\n", scanner.Text()))
@@ -176,7 +174,7 @@ func (_ reply) Execute(aerc *widgets.Aerc, args []string) error {
 		if quote {
 			var (
 				path []int
-				part *imap.BodyStructure
+				part *models.BodyStructure
 			)
 			if len(msg.BodyStructure.Parts) != 0 {
 				part, path = findPlaintext(msg.BodyStructure, path)
@@ -212,7 +210,7 @@ func (_ reply) Execute(aerc *widgets.Aerc, args []string) error {
 				// TODO: Let user customize the date format used here
 				io.WriteString(pipein, fmt.Sprintf("On %s %s wrote:\n",
 					msg.Envelope.Date.Format("Mon Jan 2, 2006 at 3:04 PM"),
-					msg.Envelope.From[0].PersonalName))
+					msg.Envelope.From[0].Name))
 				for scanner.Scan() {
 					io.WriteString(pipein, fmt.Sprintf("> %s\n", scanner.Text()))
 				}
@@ -228,8 +226,8 @@ func (_ reply) Execute(aerc *widgets.Aerc, args []string) error {
 	return nil
 }
 
-func findPlaintext(bs *imap.BodyStructure,
-	path []int) (*imap.BodyStructure, []int) {
+func findPlaintext(bs *models.BodyStructure,
+	path []int) (*models.BodyStructure, []int) {
 
 	for i, part := range bs.Parts {
 		cur := append(path, i+1)
diff --git a/lib/address.go b/lib/address.go
deleted file mode 100644
index b557195..0000000
--- a/lib/address.go
+++ /dev/null
@@ -1,40 +0,0 @@
-package lib
-
-import (
-	"bytes"
-	"fmt"
-	"regexp"
-	"strings"
-
-	"github.com/emersion/go-imap"
-)
-
-var (
-	atom *regexp.Regexp = regexp.MustCompile("^[a-z0-9!#$%7'*+-/=?^_`{}|~ ]+$")
-)
-
-func FormatAddresses(addrs []*imap.Address) string {
-	val := bytes.Buffer{}
-	for i, addr := range addrs {
-		val.WriteString(FormatAddress(addr))
-		if i != len(addrs)-1 {
-			val.WriteString(", ")
-		}
-	}
-	return val.String()
-}
-
-func FormatAddress(addr *imap.Address) string {
-	if addr.PersonalName != "" {
-		if atom.MatchString(addr.PersonalName) {
-			return fmt.Sprintf("%s <%s@%s>",
-				addr.PersonalName, addr.MailboxName, addr.HostName)
-		} else {
-			return fmt.Sprintf("\"%s\" <%s@%s>",
-				strings.ReplaceAll(addr.PersonalName, "\"", "'"),
-				addr.MailboxName, addr.HostName)
-		}
-	} else {
-		return fmt.Sprintf("<%s@%s>", addr.MailboxName, addr.HostName)
-	}
-}
diff --git a/lib/indexformat.go b/lib/indexformat.go
index 43d2ef8..fa39909 100644
--- a/lib/indexformat.go
+++ b/lib/indexformat.go
@@ -6,8 +6,6 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/emersion/go-imap"
-
 	"git.sr.ht/~sircmpwn/aerc/config"
 	"git.sr.ht/~sircmpwn/aerc/models"
 )
@@ -70,10 +68,9 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 			}
 			addr := msg.Envelope.From[0]
 			retval = append(retval, 's')
-			args = append(args, fmt.Sprintf("%s@%s", addr.MailboxName,
-				addr.HostName))
+			args = append(args, fmt.Sprintf("%s@%s", addr.Mailbox, addr.Host))
 		case 'A':
-			var addr *imap.Address
+			var addr *models.Address
 			if len(msg.Envelope.ReplyTo) == 0 {
 				if len(msg.Envelope.From) == 0 {
 					return "", nil,
@@ -85,8 +82,7 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 				addr = msg.Envelope.ReplyTo[0]
 			}
 			retval = append(retval, 's')
-			args = append(args, fmt.Sprintf("%s@%s", addr.MailboxName,
-				addr.HostName))
+			args = append(args, fmt.Sprintf("%s@%s", addr.Mailbox, addr.Host))
 		case 'C':
 			retval = append(retval, 'd')
 			args = append(args, number)
@@ -100,7 +96,7 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 			if len(msg.Envelope.From) == 0 {
 				return "", nil, errors.New("found no address for sender")
 			}
-			addr := FormatAddress(msg.Envelope.From[0])
+			addr := msg.Envelope.From[0].Format()
 			retval = append(retval, 's')
 			args = append(args, addr)
 		case 'F':
@@ -111,11 +107,10 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 			// TODO: handle case when sender is current user. Then
 			// use recipient's name
 			var val string
-			if addr.PersonalName != "" {
-				val = addr.PersonalName
+			if addr.Name != "" {
+				val = addr.Name
 			} else {
-				val = fmt.Sprintf("%s@%s",
-					addr.MailboxName, addr.HostName)
+				val = fmt.Sprintf("%s@%s", addr.Mailbox, addr.Host)
 			}
 			retval = append(retval, 's')
 			args = append(args, val)
@@ -129,20 +124,19 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 			}
 			addr := msg.Envelope.From[0]
 			var val string
-			if addr.PersonalName != "" {
-				val = addr.PersonalName
+			if addr.Name != "" {
+				val = addr.Name
 			} else {
-				val = fmt.Sprintf("%s@%s",
-					addr.MailboxName, addr.HostName)
+				val = fmt.Sprintf("%s@%s", addr.Mailbox, addr.Host)
 			}
 			retval = append(retval, 's')
 			args = append(args, val)
 		case 'r':
-			addrs := FormatAddresses(msg.Envelope.To)
+			addrs := models.FormatAddresses(msg.Envelope.To)
 			retval = append(retval, 's')
 			args = append(args, addrs)
 		case 'R':
-			addrs := FormatAddresses(msg.Envelope.Cc)
+			addrs := models.FormatAddresses(msg.Envelope.Cc)
 			retval = append(retval, 's')
 			args = append(args, addrs)
 		case 's':
@@ -154,16 +148,16 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 			}
 			addr := msg.Envelope.From[0]
 			retval = append(retval, 's')
-			args = append(args, addr.MailboxName)
+			args = append(args, addr.Mailbox)
 		case 'v':
 			if len(msg.Envelope.From) == 0 {
 				return "", nil, errors.New("found no address for sender")
 			}
 			addr := msg.Envelope.From[0]
 			// check if message is from current user
-			if addr.PersonalName != "" {
+			if addr.Name != "" {
 				retval = append(retval, 's')
-				args = append(args, strings.Split(addr.PersonalName, " ")[0])
+				args = append(args, strings.Split(addr.Name, " ")[0])
 			}
 		case 'Z':
 			// calculate all flags
@@ -171,18 +165,18 @@ func ParseIndexFormat(conf *config.AercConfig, number int,
 			var delFlag = ""
 			var flaggedFlag = ""
 			for _, flag := range msg.Flags {
-				if flag == imap.SeenFlag {
+				if flag == models.SeenFlag {
 					readFlag = "O" // message is old
-				} else if flag == imap.RecentFlag {
+				} else if flag == models.RecentFlag {
 					readFlag = "N" // message is new
-				} else if flag == imap.AnsweredFlag {
+				} else if flag == models.AnsweredFlag {
 					readFlag = "r" // message has been replied to
 				}
-				if flag == imap.DeletedFlag {
+				if flag == models.DeletedFlag {
 					delFlag = "D"
 					// TODO: check if attachments
 				}
-				if flag == imap.FlaggedFlag {
+				if flag == models.FlaggedFlag {
 					flaggedFlag = "!"
 				}
 				// TODO: check gpg stuff
diff --git a/models/models.go b/models/models.go
index cff05b1..28d256c 100644
--- a/models/models.go
+++ b/models/models.go
@@ -1,13 +1,37 @@
 package models
 
 import (
+	"bytes"
+	"fmt"
 	"io"
+	"regexp"
+	"strings"
 	"time"
 
-	"github.com/emersion/go-imap"
 	"github.com/emersion/go-message/mail"
 )
 
+// Flag is an abstraction around the different flags which can be present in
+// different email backends and represents a flag that we use in the UI.
+type Flag int
+
+const (
+	// SeenFlag marks a message as having been seen previously
+	SeenFlag Flag = iota
+
+	// RecentFlag marks a message as being recent
+	RecentFlag
+
+	// AnsweredFlag marks a message as having been replied to
+	AnsweredFlag
+
+	// DeletedFlag marks a message as having been deleted
+	DeletedFlag
+
+	// FlaggedFlag marks a message with a user flag
+	FlaggedFlag
+)
+
 type Directory struct {
 	Name       string
 	Attributes []string
@@ -30,9 +54,9 @@ type DirectoryInfo struct {
 
 // A MessageInfo holds information about the structure of a message
 type MessageInfo struct {
-	BodyStructure *imap.BodyStructure
-	Envelope      *imap.Envelope
-	Flags         []string
+	BodyStructure *BodyStructure
+	Envelope      *Envelope
+	Flags         []Flag
 	InternalDate  time.Time
 	RFC822Headers *mail.Header
 	Size          uint32
@@ -50,3 +74,59 @@ type FullMessage struct {
 	Reader io.Reader
 	Uid    uint32
 }
+
+type BodyStructure struct {
+	MIMEType          string
+	MIMESubType       string
+	Params            map[string]string
+	Description       string
+	Encoding          string
+	Parts             []*BodyStructure
+	Disposition       string
+	DispositionParams map[string]string
+}
+
+type Envelope struct {
+	Date      time.Time
+	Subject   string
+	From      []*Address
+	ReplyTo   []*Address
+	To        []*Address
+	Cc        []*Address
+	Bcc       []*Address
+	MessageId string
+}
+
+type Address struct {
+	Name    string
+	Mailbox string
+	Host    string
+}
+
+var atom *regexp.Regexp = regexp.MustCompile("^[a-z0-9!#$%7'*+-/=?^_`{}|~ ]+$")
+
+func (a Address) Format() string {
+	if a.Name != "" {
+		if atom.MatchString(a.Name) {
+			return fmt.Sprintf("%s <%s@%s>", a.Name, a.Mailbox, a.Host)
+		} else {
+			return fmt.Sprintf("\"%s\" <%s@%s>",
+				strings.ReplaceAll(a.Name, "\"", "'"),
+				a.Mailbox, a.Host)
+		}
+	} else {
+		return fmt.Sprintf("<%s@%s>", a.Mailbox, a.Host)
+	}
+}
+
+// FormatAddresses formats a list of addresses, separating each by a comma
+func FormatAddresses(addrs []*Address) string {
+	val := bytes.Buffer{}
+	for i, addr := range addrs {
+		val.WriteString(addr.Format())
+		if i != len(addrs)-1 {
+			val.WriteString(", ")
+		}
+	}
+	return val.String()
+}
diff --git a/widgets/msglist.go b/widgets/msglist.go
index 7051478..8968653 100644
--- a/widgets/msglist.go
+++ b/widgets/msglist.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"log"
 
-	"github.com/emersion/go-imap"
 	"github.com/gdamore/tcell"
 	"github.com/mattn/go-runewidth"
 
@@ -86,7 +85,7 @@ func (ml *MessageList) Draw(ctx *ui.Context) {
 		// unread message
 		seen := false
 		for _, flag := range msg.Flags {
-			if flag == imap.SeenFlag {
+			if flag == models.SeenFlag {
 				seen = true
 			}
 		}
diff --git a/widgets/msgviewer.go b/widgets/msgviewer.go
index 6a645f9..f6bef2d 100644
--- a/widgets/msgviewer.go
+++ b/widgets/msgviewer.go
@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/danwakefield/fnmatch"
-	"github.com/emersion/go-imap"
 	"github.com/emersion/go-message"
 	_ "github.com/emersion/go-message/charset"
 	"github.com/emersion/go-message/mail"
@@ -66,12 +65,12 @@ func NewMessageViewer(acct *AccountView, conf *config.AercConfig,
 	headers.AddChild(
 		&HeaderView{
 			Name:  "From",
-			Value: lib.FormatAddresses(msg.Envelope.From),
+			Value: models.FormatAddresses(msg.Envelope.From),
 		}).At(0, 0)
 	headers.AddChild(
 		&HeaderView{
 			Name:  "To",
-			Value: lib.FormatAddresses(msg.Envelope.To),
+			Value: models.FormatAddresses(msg.Envelope.To),
 		}).At(0, 1)
 	headers.AddChild(
 		&HeaderView{
@@ -112,7 +111,7 @@ handle_error:
 }
 
 func enumerateParts(conf *config.AercConfig, store *lib.MessageStore,
-	msg *models.MessageInfo, body *imap.BodyStructure,
+	msg *models.MessageInfo, body *models.BodyStructure,
 	showHeaders bool, index []int) ([]*PartViewer, error) {
 
 	var parts []*PartViewer
@@ -324,7 +323,7 @@ type PartViewer struct {
 	msg         *models.MessageInfo
 	pager       *exec.Cmd
 	pagerin     io.WriteCloser
-	part        *imap.BodyStructure
+	part        *models.BodyStructure
 	showHeaders bool
 	sink        io.WriteCloser
 	source      io.Reader
@@ -334,7 +333,7 @@ type PartViewer struct {
 
 func NewPartViewer(conf *config.AercConfig,
 	store *lib.MessageStore, msg *models.MessageInfo,
-	part *imap.BodyStructure, showHeaders bool,
+	part *models.BodyStructure, showHeaders bool,
 	index []int) (*PartViewer, error) {
 
 	var (
@@ -365,11 +364,11 @@ func NewPartViewer(conf *config.AercConfig,
 			case "subject":
 				header = msg.Envelope.Subject
 			case "from":
-				header = lib.FormatAddresses(msg.Envelope.From)
+				header = models.FormatAddresses(msg.Envelope.From)
 			case "to":
-				header = lib.FormatAddresses(msg.Envelope.To)
+				header = models.FormatAddresses(msg.Envelope.To)
 			case "cc":
-				header = lib.FormatAddresses(msg.Envelope.Cc)
+				header = models.FormatAddresses(msg.Envelope.Cc)
 			}
 			if f.Regex.Match([]byte(header)) {
 				filter = exec.Command("sh", "-c", f.Command)
diff --git a/widgets/providesmessage.go b/widgets/providesmessage.go
index d8b1e77..a1cfaa8 100644
--- a/widgets/providesmessage.go
+++ b/widgets/providesmessage.go
@@ -1,8 +1,6 @@
 package widgets
 
 import (
-	"github.com/emersion/go-imap"
-
 	"git.sr.ht/~sircmpwn/aerc/lib"
 	"git.sr.ht/~sircmpwn/aerc/lib/ui"
 	"git.sr.ht/~sircmpwn/aerc/models"
@@ -10,8 +8,8 @@ import (
 
 type PartInfo struct {
 	Index []int
-	Msg   *types.MessageInfo
-	Part  *imap.BodyStructure
+	Msg   *models.MessageInfo
+	Part  *models.BodyStructure
 	Store *lib.MessageStore
 }
 
diff --git a/worker/imap/fetch.go b/worker/imap/fetch.go
index fe25977..1745ead 100644
--- a/worker/imap/fetch.go
+++ b/worker/imap/fetch.go
@@ -82,9 +82,9 @@ func (imapw *IMAPWorker) handleFetchMessages(
 				imapw.worker.PostMessage(&types.MessageInfo{
 					Message: types.RespondTo(msg),
 					Info: &models.MessageInfo{
-						BodyStructure: _msg.BodyStructure,
-						Envelope:      _msg.Envelope,
-						Flags:         _msg.Flags,
+						BodyStructure: translateBodyStructure(_msg.BodyStructure),
+						Envelope:      translateEnvelope(_msg.Envelope),
+						Flags:         translateFlags(_msg.Flags),
 						InternalDate:  _msg.InternalDate,
 						RFC822Headers: header,
 						Uid:           _msg.Uid,
@@ -103,7 +103,7 @@ func (imapw *IMAPWorker) handleFetchMessages(
 				imapw.worker.PostMessage(&types.MessageInfo{
 					Message: types.RespondTo(msg),
 					Info: &models.MessageInfo{
-						Flags: _msg.Flags,
+						Flags: translateFlags(_msg.Flags),
 						Uid:   _msg.Uid,
 					},
 				}, nil)
@@ -120,7 +120,7 @@ func (imapw *IMAPWorker) handleFetchMessages(
 				imapw.worker.PostMessage(&types.MessageInfo{
 					Message: types.RespondTo(msg),
 					Info: &models.MessageInfo{
-						Flags: _msg.Flags,
+						Flags: translateFlags(_msg.Flags),
 						Uid:   _msg.Uid,
 					},
 				}, nil)
diff --git a/worker/imap/imap.go b/worker/imap/imap.go
index 28bac93..06bcd00 100644
--- a/worker/imap/imap.go
+++ b/worker/imap/imap.go
@@ -2,6 +2,8 @@ package imap
 
 import (
 	"github.com/emersion/go-imap"
+
+	"git.sr.ht/~sircmpwn/aerc/models"
 )
 
 func toSeqSet(uids []uint32) *imap.SeqSet {
@@ -11,3 +13,76 @@ func toSeqSet(uids []uint32) *imap.SeqSet {
 	}
 	return &set
 }
+
+func translateBodyStructure(bs *imap.BodyStructure) *models.BodyStructure {
+	if bs == nil {
+		return nil
+	}
+	var parts []*models.BodyStructure
+	for _, part := range bs.Parts {
+		parts = append(parts, translateBodyStructure(part))
+	}
+	return &models.BodyStructure{
+		MIMEType:          bs.MIMEType,
+		MIMESubType:       bs.MIMESubType,
+		Params:            bs.Params,
+		Description:       bs.Description,
+		Encoding:          bs.Encoding,
+		Parts:             parts,
+		Disposition:       bs.Disposition,
+		DispositionParams: bs.DispositionParams,
+	}
+}
+
+func translateEnvelope(e *imap.Envelope) *models.Envelope {
+	if e == nil {
+		return nil
+	}
+	return &models.Envelope{
+		Date:      e.Date,
+		Subject:   e.Subject,
+		From:      translateAddresses(e.From),
+		ReplyTo:   translateAddresses(e.ReplyTo),
+		To:        translateAddresses(e.To),
+		Cc:        translateAddresses(e.Cc),
+		Bcc:       translateAddresses(e.Bcc),
+		MessageId: e.MessageId,
+	}
+}
+
+func translateAddress(a *imap.Address) *models.Address {
+	if a == nil {
+		return nil
+	}
+	return &models.Address{
+		Name:    a.PersonalName,
+		Mailbox: a.MailboxName,
+		Host:    a.HostName,
+	}
+}
+
+func translateAddresses(addrs []*imap.Address) []*models.Address {
+	var converted []*models.Address
+	for _, addr := range addrs {
+		converted = append(converted, translateAddress(addr))
+	}
+	return converted
+}
+
+var flagMap = map[string]models.Flag{
+	imap.SeenFlag:     models.SeenFlag,
+	imap.RecentFlag:   models.RecentFlag,
+	imap.AnsweredFlag: models.AnsweredFlag,
+	imap.DeletedFlag:  models.DeletedFlag,
+	imap.FlaggedFlag:  models.FlaggedFlag,
+}
+
+func translateFlags(imapFlags []string) []models.Flag {
+	var flags []models.Flag
+	for _, imapFlag := range imapFlags {
+		if flag, ok := flagMap[imapFlag]; ok {
+			flags = append(flags, flag)
+		}
+	}
+	return flags
+}
diff --git a/worker/imap/worker.go b/worker/imap/worker.go
index 9ddaa47..88f8b37 100644
--- a/worker/imap/worker.go
+++ b/worker/imap/worker.go
@@ -187,9 +187,9 @@ func (w *IMAPWorker) handleImapUpdate(update client.Update) {
 		}
 		w.worker.PostMessage(&types.MessageInfo{
 			Info: &models.MessageInfo{
-				BodyStructure: msg.BodyStructure,
-				Envelope:      msg.Envelope,
-				Flags:         msg.Flags,
+				BodyStructure: translateBodyStructure(msg.BodyStructure),
+				Envelope:      translateEnvelope(msg.Envelope),
+				Flags:         translateFlags(msg.Flags),
 				InternalDate:  msg.InternalDate,
 				Uid:           msg.Uid,
 			},