Big commit

2023-10-24 22:14:04 +02:00
parent 91472334b9
commit 2ce8273779
8 changed files with 105 additions and 71 deletions
--- a/README.md
+++ b/README.md
@@ -35,7 +35,10 @@ concurrent_workers: 10 # the higher the number the faster zurg runs through your
 check_for_changes_every_secs: 15 # zurg polls real-debrid for changes in your library
 info_cache_time_hours: 12 # how long do we want to check if a torrent is still alive or dead? 12 to 24 hours is good enough

-# repair fixes broken links, but it doesn't mean it will appear on the same location (especially if there's only 1 episode missing)
+# zurg can repair broken links, but it doesn't mean it will appear on the same location (especially if there's only 1 episode missing)
+# e.g. i was missing e06 of Better.Call.Saul.S03.2160p.NF.WEBRip.DD5.1.x264-ViSUM
+# after zurg re-added the file, it appeared on a different directory:
+# Better.Call.Saul.S03E06.2160p.NF.WEBRip.DD5.1.x264-ViSUM.mkv
 enable_repair: false # BEWARE! THERE CAN ONLY BE 1 INSTANCE OF ZURG THAT SHOULD REPAIR YOUR TORRENTS

 # List of directory definitions and their filtering rules
--- a/internal/dav/getfile.go
+++ b/internal/dav/getfile.go
@@ -1,7 +1,6 @@
 package dav

 import (
-	"fmt"
 	"log"
 	"net/http"
 	"path"
@@ -40,15 +39,15 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent

 	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
 	if torrents == nil {
-		log.Println("Cannot find torrent", torrentName)
+		log.Println("Cannot find torrent", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}

 	filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
-	torrent, file := getFile(torrents, filenameV2, linkFragment)
+	_, file := getFile(torrents, filenameV2, linkFragment)
 	if file == nil {
-		log.Println("Cannot find file", filename)
+		log.Println("Cannot find file", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
@@ -65,8 +64,8 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
 		log.Println("Cannot unrestrict link", link, filenameV2)
-		t.MarkFileAsDeleted(torrent, file)
-		http.Error(w, "Cannot find file", http.StatusNotFound)
+		// t.HideTheFile(torrent, file)
+		http.Redirect(w, r, "https://send.nukes.wtf/tDeTd0", http.StatusFound)
 		return
 	}
 	if resp.Filename != filenameV2 {
@@ -87,7 +86,7 @@ func getFile(torrents []torrent.Torrent, filename, fragment string) (*torrent.To
 	for t := range torrents {
 		for f, file := range torrents[t].SelectedFiles {
 			fname := filepath.Base(file.Path)
-			if filename == fname && strings.HasPrefix(file.Link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) {
+			if filename == fname && strings.Contains(file.Link, fragment) {
 				return &torrents[t], &torrents[t].SelectedFiles[f]
 			}
 		}
--- a/internal/http/get.go
+++ b/internal/http/get.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"net/url"
 	"path"
 	"path/filepath"
 	"strings"
@@ -17,7 +18,7 @@ import (

 func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
 	requestPath := path.Clean(r.URL.Path)
-	requestPath = strings.Replace(requestPath, "/http", "/", 1)
+	requestPath = strings.Replace(requestPath, "/http", "", 1)
 	if requestPath == "/favicon.ico" {
 		return
 	}
@@ -46,7 +47,7 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren

 	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
 	if torrents == nil {
-		log.Println("Cannot find torrent", torrentName, segments)
+		log.Println("Cannot find torrent", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
@@ -54,12 +55,12 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
 	filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
 	_, file := getFile(torrents, filenameV2, linkFragment)
 	if file == nil {
-		log.Println("Cannot find file", filename, segments)
+		log.Println("Cannot find file (head)", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 	if file.Link == "" {
-		log.Println("Link not found", filename)
+		log.Println("Link not found 222", filename)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
@@ -117,20 +118,20 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent

 	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
 	if torrents == nil {
-		log.Println("Cannot find torrent", torrentName)
+		log.Println("Cannot find torrent", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}

 	filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
-	torrent, file := getFile(torrents, filenameV2, linkFragment)
+	_, file := getFile(torrents, filenameV2, linkFragment)
 	if file == nil {
-		log.Println("Cannot find file", filename)
+		log.Println("Cannot find file (get)", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 	if file.Link == "" {
-		log.Println("Link not found", filename)
+		log.Println("Link not found 33", filename)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
@@ -142,8 +143,9 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
 		log.Println("Cannot unrestrict link", link, filenameV2)
-		t.MarkFileAsDeleted(torrent, file)
-		http.Error(w, "Cannot find file", http.StatusNotFound)
+		// t.HideTheFile(torrent, file)
+		// http.Error(w, "Cannot find file", http.StatusNotFound)
+		http.Redirect(w, r, "https://send.nukes.wtf/tDeTd0", http.StatusFound)
 		return
 	}
 	if resp.Filename != filenameV2 {
@@ -163,8 +165,12 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 func getFile(torrents []torrent.Torrent, filename, fragment string) (*torrent.Torrent, *torrent.File) {
 	for t := range torrents {
 		for f, file := range torrents[t].SelectedFiles {
+			// fmt.Println("~~~~~~~~~~~~~~", torrents[t].Version)
+			if torrents[t].ID == "ABUNEKZP3UPMU" || torrents[t].ID == "TAA5WUJ6BVEAE" {
+				fmt.Println("~~~~~~~~~~~~~~", torrents[t].ID, torrents[t].Version, len(torrents[t].Links), len(file.Link))
+			}
 			fname := filepath.Base(file.Path)
-			if filename == fname && strings.HasPrefix(file.Link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) {
+			if filename == fname && strings.Contains(file.Link, fragment) {
 				return &torrents[t], &torrents[t].SelectedFiles[f]
 			}
 		}
@@ -219,7 +225,8 @@ func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface
 	htmlDoc := "<ul>"

 	for _, directory := range c.GetDirectories() {
-		htmlDoc += fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", directory, directory)
+		directoryPath := url.PathEscape(directory)
+		htmlDoc += fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", directoryPath, directory)
 	}

 	return &htmlDoc, nil
--- a/internal/http/response.go
+++ b/internal/http/response.go
@@ -2,6 +2,7 @@ package http

 import (
 	"fmt"
+	"net/url"
 	"path/filepath"

 	"github.com/debridmediamanager.com/zurg/internal/torrent"
@@ -23,7 +24,7 @@ func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (st
 		}
 		seen[item.Name] = true

-		path := filepath.Join(basePath, item.Name)
+		path := filepath.Join(basePath, url.PathEscape(item.Name))
 		htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", path, item.Name)
 	}

@@ -59,7 +60,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (s
 			}
 			finalName[filename] = true

-			filePath := filepath.Join(currentPath, filename)
+			filePath := filepath.Join(currentPath, url.PathEscape(filename))
 			htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
 		}
 	}
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -16,12 +16,13 @@ import (
 )

 type TorrentManager struct {
-	torrents   []Torrent
-	inProgress []string
-	checksum   string
-	config     config.ConfigInterface
-	cache      *expirable.LRU[string, string]
-	workerPool chan bool
+	requiredVersion string
+	torrents        []Torrent
+	inProgress      []string
+	checksum        string
+	config          config.ConfigInterface
+	cache           *expirable.LRU[string, string]
+	workerPool      chan bool
 }

 // NewTorrentManager creates a new torrent manager
@@ -29,9 +30,10 @@ type TorrentManager struct {
 // and store them in-memory
 func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
 	t := &TorrentManager{
-		config:     config,
-		cache:      cache,
-		workerPool: make(chan bool, config.GetNumOfWorkers()),
+		requiredVersion: "v2",
+		config:          config,
+		cache:           cache,
+		workerPool:      make(chan bool, config.GetNumOfWorkers()),
 	}

 	// Initialize torrents for the first time
@@ -67,12 +69,13 @@ func (t *TorrentManager) repairAll(wg *sync.WaitGroup) {
 	for _, torrent := range t.torrents {
 		if torrent.ForRepair {
 			log.Println("Issues detected on", torrent.Name, "; fixing...")
-			t.repair(torrent.ID, torrent.SelectedFiles)
+			t.repair(torrent.ID, torrent.SelectedFiles, true)
 		}
 		if len(torrent.Links) == 0 {
 			// If the torrent has no links
 			// and already processing repair
 			// delete it!
+			log.Println("Deleting", torrent.Name, "as it has no links")
 			realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
 		}
 	}
@@ -91,13 +94,11 @@ func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
 	return torrents
 }

-// MarkFileAsDeleted marks a file as deleted
-func (t *TorrentManager) MarkFileAsDeleted(torrent *Torrent, file *File) {
-	log.Println("Marking file as deleted", file.Path)
+// HideTheFile marks a file as deleted
+func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
 	file.Link = ""
 	t.writeToFile(torrent)
-	log.Println("Healing a single file in the torrent", torrent.Name)
-	t.repair(torrent.ID, []File{*file})
+	t.repair(torrent.ID, []File{*file}, false)
 }

 // FindAllTorrentsWithName finds all torrents in a given directory with a given name
@@ -252,11 +253,16 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 	// file cache
 	torrentFromFile := t.readFromFile(torrent.ID)
 	if torrentFromFile != nil {
+		fmt.Println("from file!")
 		// see if api data and file data still match
 		// then it means data is still usable
 		if len(torrentFromFile.Links) == len(torrent.Links) {
 			torrent.ForRepair = torrentFromFile.ForRepair
-			torrent.SelectedFiles = torrentFromFile.SelectedFiles
+			torrent.SelectedFiles = torrentFromFile.SelectedFiles[:]
+			if torrentFromFile.ID == "ABUNEKZP3UPMU" || torrentFromFile.ID == "TAA5WUJ6BVEAE" {
+				fmt.Println(">>>>>>>>>>>>>>>>", torrentFromFile.ID, len(torrentFromFile.Links), len(torrent.Links), len(torrent.SelectedFiles[0].Link))
+				torrent.Version = "v2"
+			}
 			return
 		}
 	}
@@ -306,11 +312,13 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
 			selectedFiles[i].Link = link
 		}
 	}
-	// update the torrent with more data!
-	torrent.SelectedFiles = selectedFiles
-	torrent.ForRepair = forRepair
 	// update file cache
-	t.writeToFile(torrent)
+	if len(selectedFiles) > 0 {
+		// update the torrent with more data!
+		torrent.SelectedFiles = selectedFiles
+		torrent.ForRepair = forRepair
+		t.writeToFile(torrent)
+	}
 }

 // getByID returns a torrent by its ID
@@ -333,6 +341,7 @@ func (t *TorrentManager) writeToFile(torrent *Torrent) {
 	}
 	defer file.Close()

+	torrent.Version = t.requiredVersion
 	dataEncoder := gob.NewEncoder(file)
 	dataEncoder.Encode(torrent)
 }
@@ -362,6 +371,9 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
 		log.Fatalf("Failed decoding file: %s", err)
 		return nil
 	}
+	if torrent.Version != t.requiredVersion {
+		return nil
+	}
 	return &torrent
 }

@@ -489,7 +501,7 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
 	return selectedFiles, isChaotic
 }

-func (t *TorrentManager) repair(torrentID string, selectedFiles []File) {
+func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReinsertionFirst bool) {
 	torrent := t.getByID(torrentID)
 	if torrent == nil {
 		return
@@ -540,7 +552,10 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File) {
 	}

 	// first solution: add the same selection, maybe it can be fixed by reinsertion?
-	success := t.reinsertTorrent(torrent, "", true)
+	success := false
+	if tryReinsertionFirst {
+		success = t.reinsertTorrent(torrent, "", true)
+	}
 	if !success {
 		// if not, last resort: add only the missing files and do it in 2 batches
 		half := len(missingFiles) / 2
--- a/internal/torrent/types.go
+++ b/internal/torrent/types.go
@@ -3,6 +3,7 @@ package torrent
 import "github.com/debridmediamanager.com/zurg/pkg/realdebrid"

 type Torrent struct {
+	Version string
 	realdebrid.Torrent
 	Directories   []string
 	SelectedFiles []File
--- a/pkg/realdebrid/util.go
+++ b/pkg/realdebrid/util.go
@@ -21,34 +21,42 @@ func RetryUntilOk[T any](fn func() (T, error)) T {
 }

 func canFetchFirstByte(url string) bool {
-	// Create a new HTTP request
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return false
-	}
+	const maxAttempts = 3

-	// Set the Range header to request only the first byte
-	req.Header.Set("Range", "bytes=0-0")
+	for i := 0; i < maxAttempts; i++ {
+		// Create a new HTTP request
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			continue
+		}

-	// Execute the request
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return false
-	}
-	defer resp.Body.Close()
+		// Set the Range header to request only the first byte
+		req.Header.Set("Range", "bytes=0-0")

-	// If server supports partial content
-	if resp.StatusCode == http.StatusPartialContent {
-		buffer := make([]byte, 1)
-		_, err := resp.Body.Read(buffer)
-		return err == nil
+		// Execute the request
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			time.Sleep(1 * time.Second) // Add a delay before the next retry
+			continue
+		}
+		defer resp.Body.Close()
+
+		// If server supports partial content
+		if resp.StatusCode == http.StatusPartialContent {
+			buffer := make([]byte, 1)
+			_, err := resp.Body.Read(buffer)
+			if err == nil {
+				return true
+			}
+		} else if resp.StatusCode == http.StatusOK {
+			// If server doesn't support partial content, try reading the first byte and immediately close
+			buffer := make([]byte, 1)
+			_, err = resp.Body.Read(buffer)
+			if err == nil {
+				return true
+			}
+		}
+		time.Sleep(1 * time.Second) // Add a delay before the next retry
 	}
-	if resp.StatusCode != http.StatusOK {
-		return false
-	}
-	// If server doesn't support partial content, try reading the first byte and immediately close
-	buffer := make([]byte, 1)
-	_, err = resp.Body.Read(buffer)
-	resp.Body.Close() // Close immediately after reading
-	return err == nil
+	return false
 }
--- a/rclone.conf
+++ b/rclone.conf
@@ -2,4 +2,4 @@
 type = http
 url = http://zurg:9999/http
 no_head = false
-no_slash = true
+no_slash = false