diff --git a/worker/imap/seqmap.go b/worker/imap/seqmap.go
index 2752cc8..093bbc5 100644
--- a/worker/imap/seqmap.go
+++ b/worker/imap/seqmap.go
@@ -1,6 +1,8 @@
 package imap
 
-import "sync"
+import (
+	"sync"
+)
 
 type SeqMap struct {
 	lock sync.Mutex
@@ -35,7 +37,17 @@ func (s *SeqMap) Pop(seqnum uint32) (uint32, bool) {
 	s.lock.Lock()
 	uid, found := s.m[seqnum]
 	if found {
-		delete(s.m, seqnum)
+		m := make(map[uint32]uint32)
+		for s, u := range s.m {
+			if s > seqnum {
+				// All sequence numbers greater than the removed one must be decremented by one
+				// https://datatracker.ietf.org/doc/html/rfc3501#section-7.4.1
+				m[s-1] = u
+			} else if s < seqnum {
+				m[s] = u
+			}
+		}
+		s.m = m
 	}
 	s.lock.Unlock()
 	return uid, found
diff --git a/worker/imap/seqmap_test.go b/worker/imap/seqmap_test.go
index 0ee0738..0fa8711 100644
--- a/worker/imap/seqmap_test.go
+++ b/worker/imap/seqmap_test.go
@@ -39,12 +39,15 @@ func TestSeqMap(t *testing.T) {
 	assert.Equal(found, true)
 	assert.Equal(seqmap.Size(), 2)
 
+	// Repop the same seqnum should work because of the syncing
 	_, found = seqmap.Pop(1)
-	assert.Equal(found, false)
-	assert.Equal(seqmap.Size(), 2)
+	assert.Equal(found, true)
+	assert.Equal(seqmap.Size(), 1)
 
+	// sync means we already have a 1. This is replacing that UID so the size
+	// shouldn't increase
 	seqmap.Put(1, 7331)
-	assert.Equal(seqmap.Size(), 3)
+	assert.Equal(seqmap.Size(), 1)
 
 	seqmap.Clear()
 	assert.Equal(seqmap.Size(), 0)