Refactor torrent manager

internal/config/v1.go

@@ -5,7 +5,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"gopkg.in/yaml.v3"
 )
 
@@ -33,8 +32,6 @@ func (z *ZurgConfigV1) GetDirectories() []string {
 }
 
 func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
-	log := logutil.NewLogger().Named("config")
-
 	var groupMap = make(map[string][]string)
 	var groupOrderMap = make(map[string]int) // To store GroupOrder for each directory
 
@@ -42,7 +39,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
 	for directory, val := range z.Directories {
 		groupMap[val.Group] = append(groupMap[val.Group], directory)
 		groupOrderMap[directory] = val.GroupOrder
-		log.Debugf("Added directory to group: %s, group: %s, order: %d", directory, val.Group, val.GroupOrder)
 	}
 
 	// Sort the slice based on GroupOrder and then directory name for deterministic order
@@ -54,7 +50,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
 			return groupOrderMap[dirs[i]] < groupOrderMap[dirs[j]]
 		})
 		groupMap[group] = dirs
-		log.Debugf("Sorted directories within a group: %s %v", group, dirs)
 	}
 
 	// Return a deep copy of the map

internal/dav/listing.go

@@ -11,22 +11,14 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/dav"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
+func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
 	log := logutil.NewLogger().Named("dav")
 
 	requestPath := path.Clean(r.URL.Path)
 	requestPath = strings.Trim(requestPath, "/")
 
-	if data, exists := cache.Get(requestPath); exists {
-		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
-		w.WriteHeader(http.StatusMultiStatus)
-		fmt.Fprint(w, data)
-		return
-	}
-
 	var output []byte
 	var err error
 
@@ -53,8 +45,6 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 
 	if output != nil {
 		respBody := fmt.Sprintf("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n%s\n", output)
-		cache.Add(requestPath, respBody)
-
 		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
 		w.WriteHeader(http.StatusMultiStatus)
 		fmt.Fprint(w, respBody)

internal/dav/response.go

@@ -15,7 +15,7 @@ func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*d
 	seen := make(map[string]bool)
 
 	for _, item := range torrents {
-		if item.Progress != 100 {
+		if item.InProgress {
 			continue
 		}
 		if _, exists := seen[item.AccessKey]; exists {
@@ -64,7 +64,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*
 			torrentResponses = append(torrentResponses, dav.File(
 				filePath,
 				file.Bytes,
-				convertRFC3339toRFC1123(torrent.Added),
+				convertRFC3339toRFC1123(torrent.LatestAdded),
 				file.Link,
 			))
 		}

internal/http/listing.go

@@ -5,26 +5,19 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
+func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
 	log := logutil.NewLogger().Named("http")
 
 	requestPath := path.Clean(r.URL.Path)
 
-	if data, exists := cache.Get(requestPath); exists {
-		w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprint(w, data)
-		return
-	}
-
 	var output *string
 	var err error
 
@@ -49,8 +42,6 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
 	}
 
 	if output != nil {
-		cache.Add(requestPath, *output)
-
 		w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
 		w.WriteHeader(http.StatusOK)
 		fmt.Fprint(w, *output)
@@ -73,12 +64,19 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 
 	for _, directory := range c.GetDirectories() {
 		if basePath == directory {
-			torrents := t.GetByDirectory(basePath)
-			resp, err := createMultiTorrentResponse(requestPath, torrents)
-			if err != nil {
-				return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+			htmlDoc := "<ol>"
+			for name, torrent := range t.TorrentMap {
+				if len(torrent.SelectedFiles) == 0 {
+					continue
+				}
+				for _, dir := range torrent.Directories {
+					if dir == basePath {
+						htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(name)), name)
+						break
+					}
+				}
 			}
-			return &resp, nil
+			return &htmlDoc, nil
 		}
 	}
 
@@ -86,18 +84,17 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 }
 
 func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) {
-	fullDir := path.Dir(requestPath)
-	directory := path.Base(fullDir)
 	torrentName := path.Base(requestPath)
-
-	sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
-	if len(sameNameTorrents) == 0 {
-		return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
+	htmlDoc := "<ol>"
+	for _, file := range t.TorrentMap[torrentName].SelectedFiles {
+		if file.Link == "" {
+			// TODO: fix the file?
+			fmt.Printf("File %s has no link, skipping\n", file.Path)
+			continue
+		}
+		filename := filepath.Base(file.Path)
+		filePath := filepath.Join(requestPath, url.PathEscape(filename))
+		htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
 	}
-
-	resp, err := createSingleTorrentResponse(requestPath, sameNameTorrents)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
-	}
-	return &resp, nil
+	return &htmlDoc, nil
 }

internal/http/response.go

@@ -1,60 +0,0 @@
-package http
-
-import (
-	"fmt"
-	"net/url"
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
-	htmlDoc := "<ul>"
-
-	seen := make(map[string]bool)
-
-	for _, item := range torrents {
-		if item.Progress != 100 {
-			continue
-		}
-		if _, exists := seen[item.AccessKey]; exists {
-			continue
-		}
-		seen[item.AccessKey] = true
-
-		path := filepath.Join(basePath, url.PathEscape(item.AccessKey))
-		htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", path, item.AccessKey)
-	}
-
-	return htmlDoc, nil
-}
-
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
-	htmlDoc := "<ul>"
-
-	finalName := make(map[string]bool)
-
-	currentPath := filepath.Join(basePath)
-
-	for _, torrent := range torrents {
-		for _, file := range torrent.SelectedFiles {
-			if file.Link == "" {
-				// TODO: fix the file?
-				// log.Println("File has no link, skipping", file.Path)
-				continue
-			}
-
-			filename := filepath.Base(file.Path)
-			if finalName[filename] {
-				continue
-			}
-			finalName[filename] = true
-
-			filePath := filepath.Join(currentPath, url.PathEscape(filename))
-			htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
-		}
-	}
-
-	return htmlDoc, nil
-}

internal/net/router.go

@@ -25,7 +25,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
 			if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 {
 				universal.HandleGetRequest(w, r, t, c, cache)
 			} else {
-				intHttp.HandleDirectoryListing(w, r, t, c, cache)
+				intHttp.HandleDirectoryListing(w, r, t, c)
 			}
 
 		case http.MethodHead:
@@ -40,7 +40,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
 		case "PROPFIND":
-			dav.HandlePropfindRequest(w, r, t, c, cache)
+			dav.HandlePropfindRequest(w, r, t, c)
 
 		case http.MethodGet:
 			universal.HandleGetRequest(w, r, t, c, cache)

internal/torrent/manager.go

@@ -3,7 +3,6 @@ package torrent
 import (
 	"encoding/gob"
 	"fmt"
-	"math"
 	"os"
 	"strings"
 	"sync"
@@ -13,18 +12,18 @@ import (
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
+	"github.com/nutsdb/nutsdb"
 	"go.uber.org/zap"
 )
 
 type TorrentManager struct {
+	TorrentMap        map[string]*Torrent
 	requiredVersion   string
 	rd                *realdebrid.RealDebrid
-	torrents          []Torrent
-	torrentMap        map[string]*Torrent
-	inProgress        []string
 	checksum          string
 	config            config.ConfigInterface
 	cache             *expirable.LRU[string, string]
+	db                *nutsdb.DB
 	workerPool        chan bool
 	directoryMap      map[string][]string
 	processedTorrents map[string][]string
@@ -34,14 +33,15 @@ type TorrentManager struct {
 
 // NewTorrentManager creates a new torrent manager
 // it will fetch all torrents and their info in the background
-// and store them in-memory; it is called only once at startup
-func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
+// and store them in-memory and cached in files
+func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string], db *nutsdb.DB) *TorrentManager {
 	t := &TorrentManager{
+		TorrentMap:        make(map[string]*Torrent),
 		requiredVersion:   fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
 		rd:                realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
-		torrentMap:        make(map[string]*Torrent),
 		config:            config,
 		cache:             cache,
+		db:                db,
 		workerPool:        make(chan bool, config.GetNumOfWorkers()),
 		directoryMap:      make(map[string][]string),
 		processedTorrents: make(map[string][]string),
@@ -51,8 +51,6 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 
 	// start with a clean slate
 	t.mu.Lock()
-	t.cache.Purge()
-	t.torrents = nil
 
 	newTorrents, _, err := t.rd.GetTorrents(0)
 	if err != nil {
@@ -76,46 +74,74 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 		if newTorrent == nil {
 			continue
 		}
-		t.torrents = append(t.torrents, *newTorrent)
-		if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
-			t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
-			t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
-			t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
-			t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+		if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists {
+			t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent)
 		} else {
-			t.torrentMap[newTorrent.AccessKey] = newTorrent
+			t.TorrentMap[newTorrent.AccessKey] = newTorrent
 		}
 	}
 	t.checksum = t.getChecksum()
 	t.mu.Unlock()
 
-	if t.config.EnableRepair() {
-		go t.repairAll()
-	}
+	// if t.config.EnableRepair() {
+	// 	go t.repairAll()
+	// }
 	go t.startRefreshJob()
 
 	return t
 }
 
+func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
+	merged := t1
+
+	// Merge SelectedFiles
+	fileMap := make(map[int]File)
+	for _, f := range append(t1.SelectedFiles, t2.SelectedFiles...) {
+		if _, exists := fileMap[f.ID]; !exists {
+			fileMap[f.ID] = f
+		}
+	}
+	for _, f := range fileMap {
+		merged.SelectedFiles = append(merged.SelectedFiles, f)
+	}
+
+	// Merge Instances
+	merged.Instances = append(t1.Instances, t2.Instances...)
+
+	// LatestAdded
+	if t1.LatestAdded < t2.LatestAdded {
+		merged.LatestAdded = t2.LatestAdded
+	}
+
+	// InProgress
+	for _, instance := range merged.Instances {
+		if instance.Progress != 100 {
+			merged.InProgress = true
+			break
+		}
+	}
+
+	return merged
+}
+
 // GetByDirectory returns all torrents that have a file in the specified directory
 func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
 	var torrents []Torrent
-	for i := range t.torrents {
-		for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
+	for k, v := range t.TorrentMap {
+		found := false
+		for _, dir := range v.Directories {
 			if dir == directory {
-				torrents = append(torrents, t.torrents[i])
+				found = true
+				break
 			}
 		}
+		if found {
+			torrents = append(torrents, *t.TorrentMap[k])
+		}
 	}
 	return torrents
 }
 
-// HideTheFile marks a file as deleted
-func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
-	file.Unavailable = true
-	t.repair(torrent.ID, torrent.SelectedFiles, false)
-}
-
 // FindAllTorrentsWithName finds all torrents in a given directory with a given name
 func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string) []Torrent {
 	var matchingTorrents []Torrent
@@ -133,26 +159,12 @@ func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictRe
 	return t.rd.UnrestrictUntilOk(link)
 }
 
-// findAllDownloadedFilesFromHash finds all files that were  with a given hash
-func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File {
-	var files []File
-	for _, torrent := range t.torrents {
-		if torrent.Hash == hash {
-			for _, file := range torrent.SelectedFiles {
-				if file.Link != "" {
-					files = append(files, file)
-				}
-			}
-		}
-	}
-	return files
-}
-
 type torrentsResponse struct {
 	torrents   []realdebrid.Torrent
 	totalCount int
 }
 
+// generates a checksum based on the number of torrents, the first torrent id and the number of active torrents
 func (t *TorrentManager) getChecksum() string {
 	torrentsChan := make(chan torrentsResponse)
 	countChan := make(chan int)
@@ -214,8 +226,6 @@ func (t *TorrentManager) startRefreshJob() {
 		}
 
 		t.mu.Lock()
-		t.cache.Purge()
-		t.torrents = nil
 
 		newTorrents, _, err := t.rd.GetTorrents(0)
 		if err != nil {
@@ -229,7 +239,6 @@ func (t *TorrentManager) startRefreshJob() {
 			wg.Add(1)
 			go func(idx int) {
 				defer wg.Done()
-				t.log.Debug(newTorrents[idx].ID)
 				t.workerPool <- true
 				torrentsChan <- t.getMoreInfo(newTorrents[idx])
 				<-t.workerPool
@@ -241,73 +250,58 @@ func (t *TorrentManager) startRefreshJob() {
 			if newTorrent == nil {
 				continue
 			}
-			t.torrents = append(t.torrents, *newTorrent)
-			if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
-				t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
-				t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
-				t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
-				t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+			if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists {
+				t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent)
 			} else {
-				t.torrentMap[newTorrent.AccessKey] = newTorrent
+				t.TorrentMap[newTorrent.AccessKey] = newTorrent
 			}
 		}
 		t.checksum = t.getChecksum()
 		t.mu.Unlock()
 
-		if t.config.EnableRepair() {
-			go t.repairAll()
-		}
+		// if t.config.EnableRepair() {
+		// 	go t.repairAll()
+		// }
 		go OnLibraryUpdateHook(t.config)
 	}
 }
 
-// getMoreInfo updates the selected files for a torrent
+// getMoreInfo gets original name, size and files for a torrent
 func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
-	t.log.Info("Getting more info for", rdTorrent.ID)
+	var info *realdebrid.TorrentInfo
+	var err error
 	// file cache
 	torrentFromFile := t.readFromFile(rdTorrent.ID)
-	if torrentFromFile != nil {
+	if torrentFromFile != nil && len(torrentFromFile.ID) > 0 && len(torrentFromFile.Links) == len(rdTorrent.Links) {
 		// see if api data and file data still match
 		// then it means data is still usable
-		if len(torrentFromFile.Links) == len(rdTorrent.Links) {
-			return torrentFromFile
+		info = torrentFromFile
+	}
+	if info == nil {
+		info, err = t.rd.GetTorrentInfo(rdTorrent.ID)
+		if err != nil {
+			t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
+			return nil
 		}
 	}
-	t.log.Debug("Getting info for", rdTorrent.ID)
-	info, err := t.rd.GetTorrentInfo(rdTorrent.ID)
-	if err != nil {
-		t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
-		return nil
-	}
-
-	torrent := Torrent{
-		Version:       t.requiredVersion,
-		Torrent:       *info,
-		SelectedFiles: nil,
-		ForRepair:     false,
-	}
-
 	// SelectedFiles is a subset of Files with only the selected ones
 	// it also has a Link field, which can be empty
 	// if it is empty, it means the file is no longer available
 	// Files+Links together are the same as SelectedFiles
 	var selectedFiles []File
-	var streamableFiles []File
+	streamableCount := 0
 	// if some Links are empty, we need to repair it
 	forRepair := false
 	for _, file := range info.Files {
 		if isStreamable(file.Path) {
-			streamableFiles = append(streamableFiles, File{
-				File: file,
-				Link: "",
-			})
+			streamableCount++
 		}
 		if file.Selected == 0 {
 			continue
 		}
 		selectedFiles = append(selectedFiles, File{
 			File: file,
-			Link: "",
+			Link: "", // no link yet
 		})
 	}
 	if len(selectedFiles) > len(info.Links) && info.Progress == 100 {
@@ -315,12 +309,12 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 		// chaotic file means RD will not output the desired file selection
 		// e.g. even if we select just a single mkv, it will output a rar
 		var isChaotic bool
-		selectedFiles, isChaotic = t.organizeChaos(info, selectedFiles)
+		selectedFiles, isChaotic = t.organizeChaos(&rdTorrent, selectedFiles)
 		if isChaotic {
 			t.log.Infof("Torrent %s %s is unfixable, it's always returning an unstreamable link, ignoring", info.ID, info.Name)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 		} else {
-			if len(streamableFiles) > 1 {
+			if streamableCount > 1 {
 				t.log.Infof("Torrent %s %s marked for repair", info.ID, info.Name)
 				forRepair = true
 			} else {
@@ -328,22 +322,25 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 				t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 			}
 		}
-	} else {
+	} else if len(selectedFiles) > 0 {
 		// all links are still intact! good!
 		for i, link := range info.Links {
 			selectedFiles[i].Link = link
 		}
 	}
 
-	torrent.ForRepair = forRepair
-	torrent.SelectedFiles = selectedFiles
-	torrent.AccessKey = t.getName(info.Name, info.OriginalName)
-
-	// update file cache
-	if len(selectedFiles) > 0 {
-		t.writeToFile(&torrent)
+	info.ForRepair = forRepair
+	torrent := Torrent{
+		AccessKey:     t.getName(info.Name, info.OriginalName),
+		SelectedFiles: selectedFiles,
+		Directories:   t.getDirectories(info),
+		LatestAdded:   info.Added,
+		InProgress:    info.Progress != 100,
+		Instances:     []realdebrid.TorrentInfo{*info},
+	}
+	if len(selectedFiles) > 0 && torrentFromFile == nil {
+		t.writeToFile(info) // only when there are selected files, else it's useless
 	}
-	t.log.Debugf("Got info for %s %s", torrent.ID, torrent.AccessKey)
 	return &torrent
 }
 
@@ -358,72 +355,7 @@ func (t *TorrentManager) getName(name, originalName string) string {
 	}
 }
 
-// mapToDirectories maps torrents to directories
-func (t *TorrentManager) mapToDirectories() {
-	// Map torrents to directories
-	switch t.config.GetVersion() {
-	case "v1":
-		configV1 := t.config.(*config.ZurgConfigV1)
-		groupMap := configV1.GetGroupMap()
-		// for every group, iterate over every torrent
-		// and then sprinkle/distribute the torrents to the directories of the group
-		for group, directories := range groupMap {
-			counter := make(map[string]int)
-			for i := range t.torrents {
-				// don't process torrents that are already mapped if it is not the first run
-				alreadyMappedToGroup := false
-				for _, mappedGroup := range t.processedTorrents[t.torrents[i].AccessKey] {
-					if mappedGroup == group {
-						alreadyMappedToGroup = true
-					}
-				}
-				if alreadyMappedToGroup {
-					continue
-				}
-
-				for _, directory := range directories {
-					var filenames []string
-					for _, file := range t.torrents[i].SelectedFiles {
-						filenames = append(filenames, file.Path)
-					}
-					if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].AccessKey, filenames) {
-						found := false
-						// check if it is already mapped to this directory
-						for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
-							if dir == directory {
-								found = true
-								break // it is already mapped to this directory
-							}
-						}
-						if !found {
-							counter[directory]++
-							t.mu.Lock()
-							t.directoryMap[t.torrents[i].AccessKey] = append(t.directoryMap[t.torrents[i].AccessKey], directory)
-							t.mu.Unlock()
-							break // we found a directory for this torrent, so we can stop looking for more
-						}
-					}
-				}
-				t.mu.Lock()
-				t.processedTorrents[t.torrents[i].AccessKey] = append(t.processedTorrents[t.torrents[i].AccessKey], group)
-				t.mu.Unlock()
-			}
-			sum := 0
-			for _, count := range counter {
-				sum += count
-			}
-			if sum > 0 {
-				t.log.Infof("Group processing completed: %s %v total: %d", group, counter, sum)
-			} else {
-				t.log.Infof("No new additions to directory group %s", group)
-			}
-		}
-	default:
-		t.log.Error("Unknown config version")
-	}
-}
-
-func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
+func (t *TorrentManager) getDirectories(torrent *realdebrid.TorrentInfo) []string {
 	var ret []string
 	// Map torrents to directories
 	switch t.config.GetVersion() {
@@ -435,10 +367,14 @@ func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
 		for _, directories := range groupMap {
 			for _, directory := range directories {
 				var filenames []string
-				for _, file := range torrent.SelectedFiles {
+				for _, file := range torrent.Files {
+					if file.Selected == 0 {
+						continue
+					}
 					filenames = append(filenames, file.Path)
 				}
-				if configV1.MeetsConditions(directory, torrent.ID, torrent.AccessKey, filenames) {
+				accessKey := t.getName(torrent.Name, torrent.OriginalName)
+				if configV1.MeetsConditions(directory, torrent.ID, accessKey, filenames) {
 					ret = append(ret, directory)
 					break // we found a directory for this torrent for this group, so we can stop looking for more
 				}
@@ -450,18 +386,7 @@ func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
 	return ret
 }
 
-// getByID returns a torrent by its ID
-func (t *TorrentManager) getByID(torrentID string) *Torrent {
-	for i := range t.torrents {
-		if t.torrents[i].ID == torrentID {
-			return &t.torrents[i]
-		}
-	}
-	return nil
-}
-
-// writeToFile writes a torrent to a file
-func (t *TorrentManager) writeToFile(torrent *Torrent) {
+func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) {
 	filePath := fmt.Sprintf("data/%s.bin", torrent.ID)
 	file, err := os.Create(filePath)
 	if err != nil {
@@ -475,8 +400,7 @@ func (t *TorrentManager) writeToFile(torrent *Torrent) {
 	dataEncoder.Encode(torrent)
 }
 
-// readFromFile reads a torrent from a file
-func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
+func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo {
 	filePath := fmt.Sprintf("data/%s.bin", torrentID)
 	fileInfo, err := os.Stat(filePath)
 	if err != nil {
@@ -493,7 +417,7 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 	}
 	defer file.Close()
 
-	var torrent Torrent
+	var torrent realdebrid.TorrentInfo
 	dataDecoder := gob.NewDecoder(file)
 	err = dataDecoder.Decode(&torrent)
 	if err != nil {
@@ -505,185 +429,6 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 	return &torrent
 }
 
-func (t *TorrentManager) repairAll() {
-	for _, torrent := range t.torrents {
-		if torrent.ForRepair {
-			t.log.Infof("There were less links than was expected on %s %s; fixing...", torrent.ID, torrent.AccessKey)
-			t.repair(torrent.ID, torrent.SelectedFiles, true)
-		}
-		if len(torrent.Links) == 0 && torrent.Progress == 100 {
-			// If the torrent has no links
-			// and already processing repair
-			// delete it!
-			t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.AccessKey)
-			t.rd.DeleteTorrent(torrent.ID)
-		}
-	}
-}
-
-func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReinsertionFirst bool) {
-	torrent := t.getByID(torrentID)
-	if torrent == nil {
-		return
-	}
-
-	// check if it is already "being" repaired
-	found := false
-	for _, hash := range t.inProgress {
-		if hash == torrent.Hash {
-			found = true
-			break
-		}
-	}
-	if found {
-		t.log.Infof("Repair in progress, skipping %s", torrentID)
-		return
-	}
-
-	// check if it is already repaired
-	foundFiles := t.findAllDownloadedFilesFromHash(torrent.Hash)
-	var missingFiles []File
-	for _, sFile := range selectedFiles {
-		if sFile.Link == "" || sFile.Unavailable {
-			found := false
-			for _, fFile := range foundFiles {
-				// same file but different link, then yes it has been repaired
-				if sFile.Path == fFile.Path && sFile.Link != fFile.Link {
-					found = true
-					break
-				}
-			}
-			if !found {
-				missingFiles = append(missingFiles, sFile)
-			}
-		}
-	}
-	if len(missingFiles) == 0 {
-		t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.AccessKey)
-		return
-	}
-
-	// then we repair it!
-	t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.AccessKey)
-	// check if we can still add more downloads
-	proceed := t.canCapacityHandle()
-	if !proceed {
-		t.log.Error("Cannot add more torrents, exiting")
-		return
-	}
-
-	// first solution: add the same selection, maybe it can be fixed by reinsertion?
-	success := false
-	if tryReinsertionFirst {
-		success = t.reinsertTorrent(torrent, "", true)
-	}
-	if !success {
-		// if all the selected files are missing but there are other streamable files
-		var otherStreamableFileIDs []int
-		for _, file := range torrent.Files {
-			found := false
-			for _, selectedFile := range selectedFiles {
-				if selectedFile.ID == file.ID {
-					found = true
-					break
-				}
-			}
-			if !found && isStreamable(file.Path) {
-				otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID)
-			}
-		}
-		if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 {
-			// we will download 1 extra streamable file to force a redownload of the missing files
-			// or if there's only 1 missing file, we will download 1 more to prevent a rename
-			missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",")
-			t.log.Infof("Redownloading %d missing files", len(missingFiles))
-			t.reinsertTorrent(torrent, missingFilesPlus1, false)
-		} else if len(selectedFiles) > 1 {
-			// if not, last resort: add only the missing files but do it in 2 batches
-			half := len(missingFiles) / 2
-			missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
-			missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
-			if missingFiles1 != "" {
-				t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1))
-				t.reinsertTorrent(torrent, missingFiles1, false)
-			}
-			if missingFiles2 != "" {
-				t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2))
-				t.reinsertTorrent(torrent, missingFiles2, false)
-			} else {
-				t.log.Info("No other missing files left to reinsert")
-			}
-		} else {
-			t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.AccessKey)
-			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash)
-			return
-		}
-		t.log.Info("Waiting for downloads to finish")
-	}
-}
-
-func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool {
-	// if missingFiles is not provided, look for missing files
-	if missingFiles == "" {
-		t.log.Info("Redownloading whole torrent", torrent.AccessKey)
-		var selection string
-		for _, file := range torrent.SelectedFiles {
-			selection += fmt.Sprintf("%d,", file.ID)
-		}
-		if selection == "" {
-			return false
-		}
-		if len(selection) > 0 {
-			missingFiles = selection[:len(selection)-1]
-		}
-	}
-
-	// redownload torrent
-	resp, err := t.rd.AddMagnetHash(torrent.Hash)
-	if err != nil {
-		t.log.Errorf("Cannot redownload torrent: %v", err)
-		return false
-	}
-	newTorrentID := resp.ID
-	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
-	if err != nil {
-		t.log.Errorf("Cannot start redownloading: %v", err)
-	}
-
-	if deleteIfFailed {
-		if err != nil {
-			t.rd.DeleteTorrent(newTorrentID)
-			return false
-		}
-		time.Sleep(1 * time.Second)
-		// see if the torrent is ready
-		info, err := t.rd.GetTorrentInfo(newTorrentID)
-		if err != nil {
-			t.log.Errorf("Cannot get info on redownloaded torrent: %v", err)
-			if deleteIfFailed {
-				t.rd.DeleteTorrent(newTorrentID)
-			}
-			return false
-		}
-		time.Sleep(1 * time.Second)
-
-		if info.Progress != 100 {
-			t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress)
-			t.rd.DeleteTorrent(newTorrentID)
-			return false
-		}
-
-		if len(info.Links) != len(torrent.SelectedFiles) {
-			t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles))
-			t.rd.DeleteTorrent(newTorrentID)
-			return false
-		}
-		t.log.Info("Redownload successful, deleting old torrent")
-		t.rd.DeleteTorrent(torrent.ID)
-	}
-	return true
-}
-
 func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) ([]File, bool) {
 	type Result struct {
 		Response *realdebrid.UnrestrictResponse
@@ -744,43 +489,235 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 	return selectedFiles, isChaotic
 }
 
-func (t *TorrentManager) canCapacityHandle() bool {
-	// max waiting time is 45 minutes
-	const maxRetries = 50
-	const baseDelay = 1 * time.Second
-	const maxDelay = 60 * time.Second
-	retryCount := 0
-	for {
-		count, err := t.rd.GetActiveTorrentCount()
-		if err != nil {
-			t.log.Errorf("Cannot get active downloads count: %v", err)
-			if retryCount >= maxRetries {
-				t.log.Error("Max retries reached. Exiting.")
-				return false
-			}
-			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
-			if delay > maxDelay {
-				delay = maxDelay
-			}
-			time.Sleep(delay)
-			retryCount++
-			continue
-		}
+// HideTheFile marks a file as deleted
+// func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
+// 	file.Unavailable = true
+// 	t.repair(torrent, false)
+// }
 
-		if count.DownloadingCount < count.MaxNumberOfTorrents {
-			t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
-			return true
-		}
+// func (t *TorrentManager) repairAll() {
+// 	for _, torrent := range t.torrentMap {
+// 		// do not repair if:
+// 		// in progress
+// 		hasInProgress := false
+// 		for _, info := range torrent.Instances {
+// 			if info.Progress != 100 {
+// 				hasInProgress = true
+// 				break
+// 			}
+// 		}
+// 		if hasInProgress {
+// 			continue
+// 		}
+// 		// already repaired based on other instances
+// 		var missingFiles []File
+// 		for _, file := range torrent.SelectedFiles {
+// 			if file.Link == "" || file.Unavailable {
+// 				missingFiles = append(missingFiles, file)
+// 			}
+// 		}
+// 		for _, sFile := range selectedFiles {
+// 			if sFile.Link == "" || sFile.Unavailable {
+// 				found := false
+// 				for _, fFile := range foundFiles {
+// 					// same file but different link, then yes it has been repaired
+// 					if sFile.Path == fFile.Path && sFile.Link != fFile.Link {
+// 						found = true
+// 						break
+// 					}
+// 				}
+// 				if !found {
+// 					missingFiles = append(missingFiles, sFile)
+// 				}
+// 			}
+// 		}
+// 		if len(missingFiles) == 0 {
+// 			t.log.Infof("Torrent id=%s is already repaired", info.ID)
+// 			return
+// 		}
 
-		if retryCount >= maxRetries {
-			t.log.Error("Max retries reached, exiting")
-			return false
-		}
-		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
-		if delay > maxDelay {
-			delay = maxDelay
-		}
-		time.Sleep(delay)
-		retryCount++
-	}
-}
+// 		for _, info := range torrent.Instances {
+// 			if info.Progress != 100 {
+// 				continue
+// 			}
+// 			if info.ForRepair {
+// 				t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name)
+// 				t.repair(&info, true)
+// 				break // only repair the first one for repair and then move on
+// 			}
+// 			if len(info.Links) == 0 && info.Progress == 100 {
+// 				// If the torrent has no links
+// 				// and already processing repair
+// 				// delete it!
+// 				t.log.Infof("Deleting broken torrent id=%s as it doesn't contain any files", info.ID)
+// 				t.rd.DeleteTorrent(info.ID)
+// 			}
+// 		}
+// 	}
+// }
+
+// func (t *TorrentManager) repair(info *realdebrid.TorrentInfo, tryReinsertionFirst bool) {
+// 	// then we repair it!
+// 	t.log.Infof("Repairing torrent id=%s", info.ID)
+// 	// check if we can still add more downloads
+// 	proceed := t.canCapacityHandle()
+// 	if !proceed {
+// 		t.log.Error("Cannot add more torrents, exiting")
+// 		return
+// 	}
+
+// 	// first solution: add the same selection, maybe it can be fixed by reinsertion?
+// 	success := false
+// 	if tryReinsertionFirst {
+// 		success = t.reinsertTorrent(info, "", true)
+// 	}
+// 	if !success {
+// 		// if all the selected files are missing but there are other streamable files
+// 		var otherStreamableFileIDs []int
+// 		for _, file := range info.Files {
+// 			found := false
+// 			for _, selectedFile := range selectedFiles {
+// 				if selectedFile.ID == file.ID {
+// 					found = true
+// 					break
+// 				}
+// 			}
+// 			if !found && isStreamable(file.Path) {
+// 				otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID)
+// 			}
+// 		}
+// 		if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 {
+// 			// we will download 1 extra streamable file to force a redownload of the missing files
+// 			// or if there's only 1 missing file, we will download 1 more to prevent a rename
+// 			missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",")
+// 			t.log.Infof("Redownloading %d missing files", len(missingFiles))
+// 			t.reinsertTorrent(info, missingFilesPlus1, false)
+// 		} else if len(selectedFiles) > 1 {
+// 			// if not, last resort: add only the missing files but do it in 2 batches
+// 			half := len(missingFiles) / 2
+// 			missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
+// 			missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
+// 			if missingFiles1 != "" {
+// 				t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1))
+// 				t.reinsertTorrent(info, missingFiles1, false)
+// 			}
+// 			if missingFiles2 != "" {
+// 				t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2))
+// 				t.reinsertTorrent(info, missingFiles2, false)
+// 			} else {
+// 				t.log.Info("No other missing files left to reinsert")
+// 			}
+// 		} else {
+// 			t.log.Infof("Torrent id=%s is unfixable as the only link cached in RD is already broken", info.ID)
+// 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+// 			return
+// 		}
+// 		t.log.Info("Waiting for downloads to finish")
+// 	}
+// }
+
+// func (t *TorrentManager) reinsertTorrent(torrent *realdebrid.TorrentInfo, missingFiles string, deleteIfFailed bool) bool {
+// 	// if missingFiles is not provided, look for missing files
+// 	if missingFiles == "" {
+// 		var tmpSelection string
+// 		for _, file := range torrent.Files {
+// 			if file.Selected == 0 {
+// 				continue
+// 			}
+// 			tmpSelection += fmt.Sprintf("%d,", file.ID)
+// 		}
+// 		if tmpSelection == "" {
+// 			return false
+// 		}
+// 		if len(tmpSelection) > 0 {
+// 			missingFiles = tmpSelection[:len(tmpSelection)-1]
+// 		}
+// 	}
+
+// 	// redownload torrent
+// 	resp, err := t.rd.AddMagnetHash(torrent.Hash)
+// 	if err != nil {
+// 		t.log.Errorf("Cannot redownload torrent: %v", err)
+// 		return false
+// 	}
+// 	newTorrentID := resp.ID
+// 	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
+// 	if err != nil {
+// 		t.log.Errorf("Cannot start redownloading: %v", err)
+// 	}
+
+// 	if deleteIfFailed {
+// 		if err != nil {
+// 			t.rd.DeleteTorrent(newTorrentID)
+// 			return false
+// 		}
+// 		time.Sleep(1 * time.Second)
+// 		// see if the torrent is ready
+// 		info, err := t.rd.GetTorrentInfo(newTorrentID)
+// 		if err != nil {
+// 			t.log.Errorf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
+// 			if deleteIfFailed {
+// 				t.rd.DeleteTorrent(newTorrentID)
+// 			}
+// 			return false
+// 		}
+// 		time.Sleep(1 * time.Second)
+
+// 		if info.Progress != 100 {
+// 			t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion, currently %d%%", info.ID, info.Progress)
+// 			t.rd.DeleteTorrent(newTorrentID)
+// 			return false
+// 		}
+
+// 		missingCount := len(strings.Split(missingFiles, ","))
+// 		if len(info.Links) != missingCount {
+// 			t.log.Infof("It didn't fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount)
+// 			t.rd.DeleteTorrent(newTorrentID)
+// 			return false
+// 		}
+// 		t.log.Infof("Redownload successful id=%s, deleting old torrent id=%s", newTorrentID, torrent.ID)
+// 		t.rd.DeleteTorrent(torrent.ID)
+// 	}
+// 	return true
+// }
+
+// func (t *TorrentManager) canCapacityHandle() bool {
+// 	// max waiting time is 45 minutes
+// 	const maxRetries = 50
+// 	const baseDelay = 1 * time.Second
+// 	const maxDelay = 60 * time.Second
+// 	retryCount := 0
+// 	for {
+// 		count, err := t.rd.GetActiveTorrentCount()
+// 		if err != nil {
+// 			t.log.Errorf("Cannot get active downloads count: %v", err)
+// 			if retryCount >= maxRetries {
+// 				t.log.Error("Max retries reached. Exiting.")
+// 				return false
+// 			}
+// 			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+// 			if delay > maxDelay {
+// 				delay = maxDelay
+// 			}
+// 			time.Sleep(delay)
+// 			retryCount++
+// 			continue
+// 		}
+
+// 		if count.DownloadingCount < count.MaxNumberOfTorrents {
+// 			t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
+// 			return true
+// 		}
+
+// 		if retryCount >= maxRetries {
+// 			t.log.Error("Max retries reached, exiting")
+// 			return false
+// 		}
+// 		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+// 		if delay > maxDelay {
+// 			delay = maxDelay
+// 		}
+// 		time.Sleep(delay)
+// 		retryCount++
+// 	}
+// }

internal/torrent/types.go

@@ -5,11 +5,13 @@ import (
 )
 
 type Torrent struct {
-	AccessKey string
-	Version   string
-	realdebrid.Torrent
+	AccessKey     string
 	SelectedFiles []File
-	ForRepair     bool
+	Directories   []string
+	LatestAdded   string
+	InProgress    bool
+
+	Instances []realdebrid.TorrentInfo
 }
 
 type File struct {

internal/universal/get.go

@@ -35,9 +35,9 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	// If there are less than 3 segments, return an error or adjust as needed
 	if len(segments) <= 3 {
 		if isDav {
-			dav.HandlePropfindRequest(w, r, t, c, cache)
+			dav.HandlePropfindRequest(w, r, t, c)
 		} else {
-			intHttp.HandleDirectoryListing(w, r, t, c, cache)
+			intHttp.HandleDirectoryListing(w, r, t, c)
 		}
 		return
 	}
@@ -57,7 +57,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 		return
 	}
 
-	torrent, file := getFile(torrents, filename)
+	_, file := getFile(torrents, filename)
 	if file == nil {
 		log.Errorf("Cannot find file from path %s", requestPath)
 		http.Error(w, "File not found", http.StatusNotFound)
@@ -75,7 +75,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	if resp == nil {
 		if !file.Unavailable {
 			log.Errorf("Cannot unrestrict file %s %s", filename, link)
-			t.HideTheFile(torrent, file)
+			// t.HideTheFile(torrent, file)
 		}
 		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
 		return