Create a torrentMap

internal/torrent/hooks.go

@@ -30,8 +30,7 @@ func (se *ScriptExecutor) Execute() (string, error) {
 }
 
 func OnLibraryUpdateHook(config config.ConfigInterface) {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("hooks")
+	log := logutil.NewLogger().Named("hooks")
 
 	executor := &ScriptExecutor{
 		Script: config.GetOnLibraryUpdate(),

internal/torrent/manager.go

@@ -18,7 +18,9 @@ import (
 
 type TorrentManager struct {
 	requiredVersion   string
+	rd                *realdebrid.RealDebrid
 	torrents          []Torrent
+	torrentMap        map[string]*Torrent
 	inProgress        []string
 	checksum          string
 	config            config.ConfigInterface
@@ -35,7 +37,9 @@ type TorrentManager struct {
 // and store them in-memory; it is called only once at startup
 func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
 	t := &TorrentManager{
-		requiredVersion:   fmt.Sprintf("4.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
+		requiredVersion:   fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
+		rd:                realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
+		torrentMap:        make(map[string]*Torrent),
 		config:            config,
 		cache:             cache,
 		workerPool:        make(chan bool, config.GetNumOfWorkers()),
@@ -45,32 +49,49 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 		log:               logutil.NewLogger().Named("manager"),
 	}
 
-	// Initialize torrents for the first time
+	// start with a clean slate
 	t.mu.Lock()
-	t.torrents = t.getFreshListFromAPI()
-	t.checksum = t.getChecksum()
-	t.mu.Unlock()
+	t.cache.Purge()
+	t.torrents = nil
 
-	// log.Println("First checksum", t.checksum)
+	newTorrents, _, err := t.rd.GetTorrents(0)
+	if err != nil {
+		t.log.Fatalf("Cannot get torrents: %v\n", err)
+	}
 
+	torrentsChan := make(chan *Torrent, len(newTorrents))
 	var wg sync.WaitGroup
-
-	for i := range t.torrents {
+	for i := range newTorrents {
 		wg.Add(1)
 		go func(idx int) {
 			defer wg.Done()
 			t.workerPool <- true
-			t.addMoreInfo(&t.torrents[idx])
+			torrentsChan <- t.getMoreInfo(newTorrents[idx])
 			<-t.workerPool
 		}(i)
 	}
+	wg.Wait()
+	close(torrentsChan)
+	for newTorrent := range torrentsChan {
+		if newTorrent == nil {
+			continue
+		}
+		t.torrents = append(t.torrents, *newTorrent)
+		if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
+			t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
+			t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
+			t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
+			t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+		} else {
+			t.torrentMap[newTorrent.AccessKey] = newTorrent
+		}
+	}
+	t.checksum = t.getChecksum()
+	t.mu.Unlock()
 
 	if t.config.EnableRepair() {
-		go t.repairAll(&wg)
+		go t.repairAll()
 	}
-
-	wg.Wait()
-	t.mapToDirectories()
 	go t.startRefreshJob()
 
 	return t
@@ -80,7 +101,7 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
 func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
 	var torrents []Torrent
 	for i := range t.torrents {
-		for _, dir := range t.directoryMap[t.torrents[i].Name] {
+		for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
 			if dir == directory {
 				torrents = append(torrents, t.torrents[i])
 			}
@@ -100,13 +121,18 @@ func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string)
 	var matchingTorrents []Torrent
 	torrents := t.GetByDirectory(directory)
 	for i := range torrents {
-		if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) {
+		if torrents[i].AccessKey == torrentName || strings.Contains(torrents[i].AccessKey, torrentName) {
 			matchingTorrents = append(matchingTorrents, torrents[i])
 		}
 	}
 	return matchingTorrents
 }
 
+// proxy
+func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictResponse {
+	return t.rd.UnrestrictUntilOk(link)
+}
+
 // findAllDownloadedFilesFromHash finds all files that were  with a given hash
 func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File {
 	var files []File
@@ -134,7 +160,7 @@ func (t *TorrentManager) getChecksum() string {
 
 	// GetTorrents request
 	go func() {
-		torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1)
+		torrents, totalCount, err := t.rd.GetTorrents(1)
 		if err != nil {
 			errChan <- err
 			return
@@ -144,7 +170,7 @@ func (t *TorrentManager) getChecksum() string {
 
 	// GetActiveTorrentCount request
 	go func() {
-		count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+		count, err := t.rd.GetActiveTorrentCount()
 		if err != nil {
 			errChan <- err
 			return
@@ -186,92 +212,79 @@ func (t *TorrentManager) startRefreshJob() {
 		if checksum == t.checksum {
 			continue
 		}
+
+		t.mu.Lock()
 		t.cache.Purge()
+		t.torrents = nil
 
-		newTorrents := t.getFreshListFromAPI()
+		newTorrents, _, err := t.rd.GetTorrents(0)
+		if err != nil {
+			t.log.Errorf("Cannot get torrents: %v\n", err)
+			continue
+		}
+
+		torrentsChan := make(chan *Torrent)
 		var wg sync.WaitGroup
-
 		for i := range newTorrents {
 			wg.Add(1)
 			go func(idx int) {
 				defer wg.Done()
+				t.log.Debug(newTorrents[idx].ID)
 				t.workerPool <- true
-				t.addMoreInfo(&newTorrents[idx])
+				torrentsChan <- t.getMoreInfo(newTorrents[idx])
 				<-t.workerPool
 			}(i)
 		}
 		wg.Wait()
-
-		// apply side effects
-		t.mu.Lock()
-		t.torrents = newTorrents
+		close(torrentsChan)
+		for newTorrent := range torrentsChan {
+			if newTorrent == nil {
+				continue
+			}
+			t.torrents = append(t.torrents, *newTorrent)
+			if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
+				t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
+				t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
+				t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
+				t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
+			} else {
+				t.torrentMap[newTorrent.AccessKey] = newTorrent
+			}
+		}
 		t.checksum = t.getChecksum()
 		t.mu.Unlock()
 
-		// log.Println("Checksum changed", t.checksum)
 		if t.config.EnableRepair() {
-			go t.repairAll(&wg)
+			go t.repairAll()
 		}
-		go t.mapToDirectories()
 		go OnLibraryUpdateHook(t.config)
 	}
 }
 
-// getFreshListFromAPI returns all torrents
-func (t *TorrentManager) getFreshListFromAPI() []Torrent {
-	torrents, _, err := realdebrid.GetTorrents(t.config.GetToken(), 0)
-	if err != nil {
-		t.log.Errorf("Cannot get torrents: %v\n", err)
-		return nil
-	}
-
-	// convert to own internal type without SelectedFiles yet
-	// populate inProgress
-	var torrentsV2 []Torrent
-	t.inProgress = t.inProgress[:0] // reset
-	for _, torrent := range torrents {
-		torrent.Name = strings.TrimSuffix(torrent.Name, "/")
-		torrentV2 := Torrent{
-			Torrent:       torrent,
-			SelectedFiles: nil,
-			ForRepair:     false,
-			lock:          &sync.Mutex{},
-		}
-		torrentsV2 = append(torrentsV2, torrentV2)
-
-		if torrent.Progress != 100 {
-			t.inProgress = append(t.inProgress, torrent.Hash)
-		}
-	}
-
-	t.log.Infof("Fetched %d torrents", len(torrentsV2))
-	return torrentsV2
-}
-
-// addMoreInfo updates the selected files for a torrent
-func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
+// getMoreInfo updates the selected files for a torrent
+func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
+	t.log.Info("Getting more info for", rdTorrent.ID)
 	// file cache
-	torrentFromFile := t.readFromFile(torrent.ID)
+	torrentFromFile := t.readFromFile(rdTorrent.ID)
 	if torrentFromFile != nil {
 		// see if api data and file data still match
 		// then it means data is still usable
-		if len(torrentFromFile.Links) == len(torrent.Links) {
-			torrent.Name = t.getName(torrentFromFile)
-			torrent.ForRepair = torrentFromFile.ForRepair
-			torrent.SelectedFiles = torrentFromFile.SelectedFiles[:]
-			return
+		if len(torrentFromFile.Links) == len(rdTorrent.Links) {
+			return torrentFromFile
 		}
 	}
-	// no file data yet as it is still downloading
-	if torrent.Progress != 100 {
-		return
+	t.log.Debug("Getting info for", rdTorrent.ID)
+	info, err := t.rd.GetTorrentInfo(rdTorrent.ID)
+	if err != nil {
+		t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
+		return nil
 	}
 
-	// t.log.Println("Getting info for", torrent.ID)
-	info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrent.ID)
-	if err != nil {
-		t.log.Errorf("Cannot get info: %v\n", err)
-		return
+	torrent := Torrent{
+		Version:       t.requiredVersion,
+		Torrent:       *info,
+		SelectedFiles: nil,
+		ForRepair:     false,
 	}
 
 	// SelectedFiles is a subset of Files with only the selected ones
@@ -321,23 +334,25 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 			selectedFiles[i].Link = link
 		}
 	}
+
+	torrent.ForRepair = forRepair
+	torrent.SelectedFiles = selectedFiles
+	torrent.AccessKey = t.getName(info.Name, info.OriginalName)
+
 	// update file cache
-	torrent.OriginalName = info.OriginalName
-	torrent.Name = t.getName(torrent)
 	if len(selectedFiles) > 0 {
-		// update the torrent with more data!
-		torrent.SelectedFiles = selectedFiles
-		torrent.ForRepair = forRepair
-		t.writeToFile(torrent)
+		t.writeToFile(&torrent)
 	}
+	t.log.Debugf("Got info for %s %s", torrent.ID, torrent.AccessKey)
+	return &torrent
 }
 
-func (t *TorrentManager) getName(torrent *Torrent) string {
+func (t *TorrentManager) getName(name, originalName string) string {
 	// drop the extension from the name
-	if t.config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
-		return torrent.Name
+	if t.config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
+		return name
 	} else {
-		ret := strings.TrimSuffix(torrent.OriginalName, ".mp4")
+		ret := strings.TrimSuffix(originalName, ".mp4")
 		ret = strings.TrimSuffix(ret, ".mkv")
 		return ret
 	}
@@ -357,7 +372,7 @@ func (t *TorrentManager) mapToDirectories() {
 			for i := range t.torrents {
 				// don't process torrents that are already mapped if it is not the first run
 				alreadyMappedToGroup := false
-				for _, mappedGroup := range t.processedTorrents[t.torrents[i].Name] {
+				for _, mappedGroup := range t.processedTorrents[t.torrents[i].AccessKey] {
 					if mappedGroup == group {
 						alreadyMappedToGroup = true
 					}
@@ -371,10 +386,10 @@ func (t *TorrentManager) mapToDirectories() {
 					for _, file := range t.torrents[i].SelectedFiles {
 						filenames = append(filenames, file.Path)
 					}
-					if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].Name, filenames) {
+					if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].AccessKey, filenames) {
 						found := false
 						// check if it is already mapped to this directory
-						for _, dir := range t.directoryMap[t.torrents[i].Name] {
+						for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
 							if dir == directory {
 								found = true
 								break // it is already mapped to this directory
@@ -383,14 +398,14 @@ func (t *TorrentManager) mapToDirectories() {
 						if !found {
 							counter[directory]++
 							t.mu.Lock()
-							t.directoryMap[t.torrents[i].Name] = append(t.directoryMap[t.torrents[i].Name], directory)
+							t.directoryMap[t.torrents[i].AccessKey] = append(t.directoryMap[t.torrents[i].AccessKey], directory)
 							t.mu.Unlock()
 							break // we found a directory for this torrent, so we can stop looking for more
 						}
 					}
 				}
 				t.mu.Lock()
-				t.processedTorrents[t.torrents[i].Name] = append(t.processedTorrents[t.torrents[i].Name], group)
+				t.processedTorrents[t.torrents[i].AccessKey] = append(t.processedTorrents[t.torrents[i].AccessKey], group)
 				t.mu.Unlock()
 			}
 			sum := 0
@@ -408,6 +423,33 @@ func (t *TorrentManager) mapToDirectories() {
 	}
 }
 
+func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
+	var ret []string
+	// Map torrents to directories
+	switch t.config.GetVersion() {
+	case "v1":
+		configV1 := t.config.(*config.ZurgConfigV1)
+		groupMap := configV1.GetGroupMap()
+		// for every group, iterate over every torrent
+		// and then sprinkle/distribute the torrents to the directories of the group
+		for _, directories := range groupMap {
+			for _, directory := range directories {
+				var filenames []string
+				for _, file := range torrent.SelectedFiles {
+					filenames = append(filenames, file.Path)
+				}
+				if configV1.MeetsConditions(directory, torrent.ID, torrent.AccessKey, filenames) {
+					ret = append(ret, directory)
+					break // we found a directory for this torrent for this group, so we can stop looking for more
+				}
+			}
+		}
+	default:
+		t.log.Error("Unknown config version")
+	}
+	return ret
+}
+
 // getByID returns a torrent by its ID
 func (t *TorrentManager) getByID(torrentID string) *Torrent {
 	for i := range t.torrents {
@@ -463,19 +505,18 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 	return &torrent
 }
 
-func (t *TorrentManager) repairAll(wg *sync.WaitGroup) {
-	wg.Wait()
+func (t *TorrentManager) repairAll() {
 	for _, torrent := range t.torrents {
 		if torrent.ForRepair {
-			t.log.Infof("Issues were detected on %s %s; fixing...", torrent.ID, torrent.Name)
+			t.log.Infof("There were less links than was expected on %s %s; fixing...", torrent.ID, torrent.AccessKey)
 			t.repair(torrent.ID, torrent.SelectedFiles, true)
 		}
 		if len(torrent.Links) == 0 && torrent.Progress == 100 {
 			// If the torrent has no links
 			// and already processing repair
 			// delete it!
-			t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.Name)
-			realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
+			t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.AccessKey)
+			t.rd.DeleteTorrent(torrent.ID)
 		}
 	}
 }
@@ -518,12 +559,12 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
 		}
 	}
 	if len(missingFiles) == 0 {
-		t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.Name)
+		t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.AccessKey)
 		return
 	}
 
 	// then we repair it!
-	t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.Name)
+	t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.AccessKey)
 	// check if we can still add more downloads
 	proceed := t.canCapacityHandle()
 	if !proceed {
@@ -573,7 +614,7 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
 				t.log.Info("No other missing files left to reinsert")
 			}
 		} else {
-			t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.Name)
+			t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.AccessKey)
 			t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash)
 			return
 		}
@@ -584,7 +625,7 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
 func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool {
 	// if missingFiles is not provided, look for missing files
 	if missingFiles == "" {
-		t.log.Info("Redownloading whole torrent", torrent.Name)
+		t.log.Info("Redownloading whole torrent", torrent.AccessKey)
 		var selection string
 		for _, file := range torrent.SelectedFiles {
 			selection += fmt.Sprintf("%d,", file.ID)
@@ -598,29 +639,29 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string,
 	}
 
 	// redownload torrent
-	resp, err := realdebrid.AddMagnetHash(t.config.GetToken(), torrent.Hash)
+	resp, err := t.rd.AddMagnetHash(torrent.Hash)
 	if err != nil {
 		t.log.Errorf("Cannot redownload torrent: %v", err)
 		return false
 	}
 	newTorrentID := resp.ID
-	err = realdebrid.SelectTorrentFiles(t.config.GetToken(), newTorrentID, missingFiles)
+	err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
 	if err != nil {
 		t.log.Errorf("Cannot start redownloading: %v", err)
 	}
 
 	if deleteIfFailed {
 		if err != nil {
-			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			t.rd.DeleteTorrent(newTorrentID)
 			return false
 		}
 		time.Sleep(1 * time.Second)
 		// see if the torrent is ready
-		info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), newTorrentID)
+		info, err := t.rd.GetTorrentInfo(newTorrentID)
 		if err != nil {
 			t.log.Errorf("Cannot get info on redownloaded torrent: %v", err)
 			if deleteIfFailed {
-				realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+				t.rd.DeleteTorrent(newTorrentID)
 			}
 			return false
 		}
@@ -628,17 +669,17 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string,
 
 		if info.Progress != 100 {
 			t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress)
-			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			t.rd.DeleteTorrent(newTorrentID)
 			return false
 		}
 
 		if len(info.Links) != len(torrent.SelectedFiles) {
 			t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles))
-			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			t.rd.DeleteTorrent(newTorrentID)
 			return false
 		}
 		t.log.Info("Redownload successful, deleting old torrent")
-		realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
+		t.rd.DeleteTorrent(torrent.ID)
 	}
 	return true
 }
@@ -652,43 +693,43 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 	var wg sync.WaitGroup
 
 	// Limit concurrency
-	sem := make(chan struct{}, t.config.GetNumOfWorkers())
+	sem := make(chan bool, t.config.GetNumOfWorkers())
 
 	for _, link := range info.Links {
 		wg.Add(1)
-		sem <- struct{}{}
+		sem <- true
 		go func(lnk string) {
 			defer wg.Done()
 			defer func() { <-sem }()
-
-			unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
-				return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk)
-			}
-			resp := realdebrid.RetryUntilOk(unrestrictFn)
-			if resp != nil {
-				resultsChan <- Result{Response: resp}
-			}
+			resp := t.rd.UnrestrictUntilOk(lnk)
+			resultsChan <- Result{Response: resp}
 		}(link)
 	}
 
 	go func() {
+		t.log.Debugf("Checking %d link(s) for problematic torrent id=%s", len(info.Links), info.ID)
 		wg.Wait()
 		close(sem)
 		close(resultsChan)
+		t.log.Debugf("Closing results channel for torrent id=%s, checking...", info.ID)
 	}()
 
 	isChaotic := false
 	for result := range resultsChan {
+		if result.Response == nil {
+			continue
+		}
 		found := false
 		for i := range selectedFiles {
-			if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) {
+			if strings.Contains(selectedFiles[i].Path, result.Response.Filename) {
+				t.log.Debugf("Found a file that is in the selection for torrent id=%s: %s", info.ID, result.Response.Filename)
 				selectedFiles[i].Link = result.Response.Link
 				found = true
 			}
 		}
 		if !found {
-			// "chaos" file, we don't know where it belongs
-			isChaotic = !isStreamable(result.Response.Filename)
+			isChaotic = result.Response.Streamable == 0
+			t.log.Debugf("Found a file that is not in the selection for torrent id=%s: %s %v", info.ID, result.Response.Filename, result.Response.Streamable)
 			selectedFiles = append(selectedFiles, File{
 				File: realdebrid.File{
 					Path:     result.Response.Filename,
@@ -710,7 +751,7 @@ func (t *TorrentManager) canCapacityHandle() bool {
 	const maxDelay = 60 * time.Second
 	retryCount := 0
 	for {
-		count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+		count, err := t.rd.GetActiveTorrentCount()
 		if err != nil {
 			t.log.Errorf("Cannot get active downloads count: %v", err)
 			if retryCount >= maxRetries {

internal/torrent/types.go

@@ -1,17 +1,15 @@
 package torrent
 
 import (
-	"sync"
-
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 )
 
 type Torrent struct {
-	Version string
+	AccessKey string
+	Version   string
 	realdebrid.Torrent
 	SelectedFiles []File
 	ForRepair     bool
-	lock          *sync.Mutex
 }
 
 type File struct {