diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index ccabab0..4f4355c 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -38,8 +38,7 @@ type TorrentManager struct {
 	RepairKillSwitch  chan struct{}
 	RemountTrigger    chan struct{}
 
-	latestState   *LibraryState
-	allAccessKeys mapset.Set[string]
+	latestState *LibraryState
 
 	fixers          cmap.ConcurrentMap[string, string] // trigger -> [command, id]
 	repairTrigger   chan *Torrent
@@ -69,8 +68,7 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
 		RepairKillSwitch:  make(chan struct{}, 1),
 		RemountTrigger:    make(chan struct{}, 1),
 
-		latestState:   &LibraryState{},
-		allAccessKeys: mapset.NewSet[string](), // this is for tracking what is deleted
+		latestState: &LibraryState{},
 	}
 
 	t.fixers = t.readFixersFromFile()
diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go
index 70af538..162cde4 100644
--- a/internal/torrent/refresh.go
+++ b/internal/torrent/refresh.go
@@ -80,12 +80,11 @@ func (t *TorrentManager) refreshTorrents(isInitialRun bool) []string {
 		}
 	}
 	// removed torrents
-	t.allAccessKeys.Difference(freshAccessKeys).Each(func(accessKey string) bool {
+	allAccessKeys := mapset.NewSet[string](allTorrents.Keys()...)
+	allAccessKeys.Difference(freshAccessKeys).Each(func(accessKey string) bool {
 		t.Delete(accessKey, false)
-		t.allAccessKeys.Remove(accessKey)
 		return false
 	})
-	t.allAccessKeys.Append(freshAccessKeys.ToSlice()...)
 
 	t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)