Implement autoheal feature

internal/dav/getfile.go

@@ -64,23 +64,19 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	}
 	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
-		// TODO: Readd the file
-		// when unrestricting fails, it means the file is not available anymore, but still in their database
-		// if it's the only file, tough luck
-		// if it's the only file, try to readd it
-		// delete the old one, add a new one
 		log.Println("Cannot unrestrict link", link, filenameV2)
 		t.MarkFileAsDeleted(torrent, file)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 	if resp.Filename != filenameV2 {
-		// TODO: Redo the logic to handle mismatch
+		actualExt := filepath.Ext(resp.Filename)
-		// [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar
+		expectedExt := filepath.Ext(filenameV2)
-		// Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv
+		if actualExt != expectedExt {
-		// Action: schedule a "cleanup" job for the parent torrent
+			log.Println("File extension mismatch", resp.Filename, filenameV2)
-		// If the file extension changed, that means it's a different file
+		} else {
-		log.Println("Filename mismatch", resp.Filename, filenameV2)
+			log.Println("Filename mismatch", resp.Filename, filenameV2)
+		}
 	}
 	cache.Add(requestPath, resp.Download)
 	http.Redirect(w, r, resp.Download, http.StatusFound)

internal/dav/response.go

@@ -52,8 +52,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*
 	for _, torrent := range torrents {
 		for _, file := range torrent.SelectedFiles {
 			if file.Link == "" {
-				// TODO: Fix this file
+				log.Println("File has no link, skipping (repairing links take time)", file.Path)
-				log.Println("File has no link, skipping", file.Path)
 				continue
 			}
 

internal/http/get.go

@@ -141,23 +141,19 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	}
 	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
-		// TODO: Readd the file
-		// when unrestricting fails, it means the file is not available anymore, but still in their database
-		// if it's the only file, tough luck
-		// if it's the only file, try to readd it
-		// delete the old one, add a new one
 		log.Println("Cannot unrestrict link", link, filenameV2)
 		t.MarkFileAsDeleted(torrent, file)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 	if resp.Filename != filenameV2 {
-		// TODO: Redo the logic to handle mismatch
+		actualExt := filepath.Ext(resp.Filename)
-		// [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar
+		expectedExt := filepath.Ext(filenameV2)
-		// Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv
+		if actualExt != expectedExt {
-		// Action: schedule a "cleanup" job for the parent torrent
+			log.Println("File extension mismatch", resp.Filename, filenameV2)
-		// If the file extension changed, that means it's a different file
+		} else {
-		log.Println("Filename mismatch", resp.Filename, filenameV2)
+			log.Println("Filename mismatch", resp.Filename, filenameV2)
+		}
 	}
 	cache.Add(requestPath, resp.Download)
 	http.Redirect(w, r, resp.Download, http.StatusFound)

internal/torrent/manager.go

@@ -4,6 +4,7 @@ import (
 	"encoding/gob"
 	"fmt"
 	"log"
+	"math"
 	"os"
 	"strings"
 	"sync"
@@ -91,6 +92,8 @@ func (t *TorrentManager) MarkFileAsDeleted(torrent *Torrent, file *File) {
 	log.Println("Marking file as deleted", file.Path)
 	file.Link = ""
 	t.writeToFile(torrent.ID, torrent)
+	log.Println("Healing a single file in the torrent", torrent.Name)
+	t.heal(torrent.ID, []File{*file})
 }
 
 // GetInfo returns the info for a torrent
@@ -114,7 +117,7 @@ func (t *TorrentManager) getChecksum() string {
 		log.Println("Huh, no torrents returned")
 		return t.checksum
 	}
-	return fmt.Sprintf("%d-%s", totalCount, torrents[0].ID)
+	return fmt.Sprintf("%d-%s-%v", totalCount, torrents[0].ID, torrents[0].Progress == 100)
 }
 
 // refreshTorrents periodically refreshes the torrents
@@ -221,9 +224,11 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
 	if torrentFromFile != nil {
 		torrent := t.getByID(torrentID)
 		if torrent != nil {
-			torrent.SelectedFiles = torrentFromFile.SelectedFiles
+			if len(torrentFromFile.SelectedFiles) == len(torrent.Links) {
+				torrent.SelectedFiles = torrentFromFile.SelectedFiles
+				return torrent
+			}
 		}
-		return torrent
 	}
 	log.Println("Getting info for", torrentID)
 	info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrentID)
@@ -242,62 +247,9 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
 		})
 	}
 	if len(selectedFiles) != len(info.Links) {
-		// TODO: This means some files have expired
-		// we need to 'fix' this torrent then, at least the missing selected files
 		log.Println("Some links has expired for", info.Name)
-
+		selectedFiles = t.organizeChaos(info, selectedFiles)
-		type Result struct {
+		t.heal(torrentID, selectedFiles)
-			Response *realdebrid.UnrestrictResponse
-		}
-
-		resultsChan := make(chan Result, len(info.Links))
-		var wg sync.WaitGroup
-
-		// Limit concurrency
-		sem := make(chan struct{}, t.config.GetNumOfWorkers())
-
-		for _, link := range info.Links {
-			wg.Add(1)
-			sem <- struct{}{} // Acquire semaphore
-			go func(lnk string) {
-				defer wg.Done()
-				defer func() { <-sem }() // Release semaphore
-
-				unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
-					return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk)
-				}
-				resp := realdebrid.RetryUntilOk(unrestrictFn)
-				if resp != nil {
-					resultsChan <- Result{Response: resp}
-				}
-			}(link)
-		}
-
-		go func() {
-			wg.Wait()
-			close(sem)
-			close(resultsChan)
-		}()
-
-		for result := range resultsChan {
-			found := false
-			for i := range selectedFiles {
-				if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) {
-					selectedFiles[i].Link = result.Response.Link
-					found = true
-				}
-			}
-			if !found {
-				selectedFiles = append(selectedFiles, File{
-					File: realdebrid.File{
-						Path:     result.Response.Filename,
-						Bytes:    result.Response.Filesize,
-						Selected: 1,
-					},
-					Link: result.Response.Link,
-				})
-			}
-		}
 	} else {
 		for i, link := range info.Links {
 			selectedFiles[i].Link = link
@@ -365,3 +317,191 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 
 	return &torrent
 }
+
+func (t *TorrentManager) reinsertTorrent(oldTorrentID string, missingFiles string, deleteIfFailed bool) bool {
+	torrent := t.GetInfo(oldTorrentID)
+	if torrent == nil {
+		return false
+	}
+
+	if missingFiles == "" {
+		var selection string
+		for _, file := range torrent.SelectedFiles {
+			if file.Link == "" {
+				selection += fmt.Sprintf("%d,", file.ID)
+			}
+		}
+		if selection == "" {
+			return false
+		}
+		missingFiles = selection[:len(selection)-1]
+	}
+
+	// reinsert torrent
+	resp, err := realdebrid.AddMagnetHash(t.config.GetToken(), torrent.Hash)
+	if err != nil {
+		log.Printf("Cannot reinsert torrent: %v\n", err)
+		return false
+	}
+	newTorrentID := resp.ID
+	err = realdebrid.SelectTorrentFiles(t.config.GetToken(), newTorrentID, missingFiles)
+	if err != nil {
+		log.Printf("Cannot select files on reinserted torrent: %v\n", err)
+	}
+
+	if deleteIfFailed {
+		if err != nil {
+			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			return false
+		}
+		time.Sleep(1 * time.Second)
+		// see if the torrent is ready
+		info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), newTorrentID)
+		if err != nil {
+			log.Printf("Cannot get info on reinserted torrent: %v\n", err)
+			if deleteIfFailed {
+				realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			}
+			return false
+		}
+		time.Sleep(1 * time.Second)
+
+		if info.Progress != 100 {
+			log.Printf("Torrent is not cached anymore, %d%%\n", info.Progress)
+			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			return false
+		}
+		if len(info.Links) != len(torrent.SelectedFiles) {
+			log.Printf("It doesn't fix the problem, got %d but we need %d\n", len(info.Links), len(torrent.SelectedFiles))
+			realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
+			return false
+		}
+		log.Println("Reinsertion successful, deleting old torrent")
+		realdebrid.DeleteTorrent(t.config.GetToken(), oldTorrentID)
+	}
+	return true
+}
+
+func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) []File {
+	type Result struct {
+		Response *realdebrid.UnrestrictResponse
+	}
+
+	resultsChan := make(chan Result, len(info.Links))
+	var wg sync.WaitGroup
+
+	// Limit concurrency
+	sem := make(chan struct{}, t.config.GetNumOfWorkers())
+
+	for _, link := range info.Links {
+		wg.Add(1)
+		sem <- struct{}{} // Acquire semaphore
+		go func(lnk string) {
+			defer wg.Done()
+			defer func() { <-sem }() // Release semaphore
+
+			unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
+				return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk)
+			}
+			resp := realdebrid.RetryUntilOk(unrestrictFn)
+			if resp != nil {
+				resultsChan <- Result{Response: resp}
+			}
+		}(link)
+	}
+
+	go func() {
+		wg.Wait()
+		close(sem)
+		close(resultsChan)
+	}()
+
+	for result := range resultsChan {
+		found := false
+		for i := range selectedFiles {
+			if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) {
+				selectedFiles[i].Link = result.Response.Link
+				found = true
+			}
+		}
+		if !found {
+			selectedFiles = append(selectedFiles, File{
+				File: realdebrid.File{
+					Path:     result.Response.Filename,
+					Bytes:    result.Response.Filesize,
+					Selected: 1,
+				},
+				Link: result.Response.Link,
+			})
+		}
+	}
+
+	return selectedFiles
+}
+
+func (t *TorrentManager) heal(torrentID string, selectedFiles []File) {
+	// max waiting time is 45 minutes
+	const maxRetries = 50
+	const baseDelay = 1 * time.Second
+	const maxDelay = 60 * time.Second
+	retryCount := 0
+	for {
+		count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+		if err != nil {
+			log.Printf("Cannot get active torrent count: %v\n", err)
+			if retryCount >= maxRetries {
+				log.Println("Max retries reached. Exiting.")
+				return
+			}
+			delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+			if delay > maxDelay {
+				delay = maxDelay
+			}
+			time.Sleep(delay)
+			retryCount++
+			continue
+		}
+
+		if count.DownloadingCount < count.MaxNumberOfTorrents {
+			log.Printf("We can still add a new torrent, %d/%d\n", count.DownloadingCount, count.MaxNumberOfTorrents)
+			break
+		}
+
+		if retryCount >= maxRetries {
+			log.Println("Max retries reached. Exiting.")
+			return
+		}
+		delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
+		if delay > maxDelay {
+			delay = maxDelay
+		}
+		time.Sleep(delay)
+		retryCount++
+	}
+
+	// now we can get the missing files
+	half := len(selectedFiles) / 2
+	missingFiles1 := getMissingFiles(0, half, selectedFiles)
+	missingFiles2 := getMissingFiles(half, len(selectedFiles), selectedFiles)
+
+	// first solution: add the same selection, maybe it can be fixed by reinsertion?
+	success := t.reinsertTorrent(torrentID, "", true)
+	if !success {
+		// if not, last resort: add only the missing files and do it in 2 batches
+		t.reinsertTorrent(torrentID, missingFiles1, false)
+		t.reinsertTorrent(torrentID, missingFiles2, false)
+	}
+}
+
+func getMissingFiles(start, end int, files []File) string {
+	var missingFiles string
+	for i := start; i < end; i++ {
+		if files[i].File.Selected == 1 && files[i].ID != 0 && files[i].Link == "" {
+			missingFiles += fmt.Sprintf("%d,", files[i].ID)
+		}
+	}
+	if len(missingFiles) > 0 {
+		missingFiles = missingFiles[:len(missingFiles)-1]
+	}
+	return missingFiles
+}

pkg/realdebrid/api.go

@@ -104,6 +104,7 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
 		params := url.Values{}
 		params.Set("page", fmt.Sprintf("%d", page))
 		params.Set("limit", fmt.Sprintf("%d", limit))
+		params.Set("filter", "active")
 
 		reqURL := baseURL + "?" + params.Encode()
 
@@ -264,12 +265,11 @@ func DeleteTorrent(accessToken string, id string) error {
 	}
 }
 
-// AddMagnet adds a magnet link to download.
+// AddMagnetHash adds a magnet link to download.
-func AddMagnet(accessToken, magnet, host string) (*MagnetResponse, error) {
+func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
 	// Prepare request data
 	data := url.Values{}
-	data.Set("magnet", magnet)
+	data.Set("magnet", fmt.Sprintf("magnet:?xt=urn:btih:%s", magnet))
-	data.Set("host", host)
 
 	// Construct request URL
 	reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet"

pkg/realdebrid/types.go

@@ -25,6 +25,7 @@ type Torrent struct {
 }
 
 type File struct {
+	ID       int    `json:"id"`
 	Path     string `json:"path"`
 	Bytes    int64  `json:"bytes"`
 	Selected int    `json:"selected"`