Proper caching

.gitignore

@@ -19,3 +19,6 @@
 
 # Go workspace file
 go.work
+
+data/
+rclone

internal/dav/getfile.go

@@ -4,14 +4,13 @@ import (
 	"fmt"
 	"log"
 	"net/http"
-	"net/url"
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
+	"github.com/debridmediamanager.com/zurg/pkg/davextra"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 )
 
@@ -24,21 +23,27 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	if len(segments) < 3 {
 		log.Println("Invalid url", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
+		return
 	}
 
 	// Get the last two segments
 	torrentName := segments[len(segments)-2]
+	filename := segments[len(segments)-1]
+
 	torrents := findAllTorrentsWithName(t, torrentName)
 	if torrents == nil {
-		log.Println("Cannot find directory", requestPath)
+		log.Println("Cannot find torrent", torrentName)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 
-	filename := segments[len(segments)-1]
+	filenameV2, linkFragment := davextra.ExtractLinkFragment(filename)
-
+	link := findLinkByFragment(torrents, filenameV2, linkFragment)
-	filenameV2, linkFragment := extractIDFromFilename(filename)
+	if link == "" {
-	link := findLinkByFragment(torrents, linkFragment)
+		log.Println("Link not found")
+		http.Error(w, "Cannot find file", http.StatusNotFound)
+		return
+	}
 
 	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
 		return realdebrid.UnrestrictLink(os.Getenv("RD_TOKEN"), link)
@@ -52,38 +57,25 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	}
 	if resp.Filename != filenameV2 {
 		// TODO: Redo the logic to handle mismatch
+		// Filename mismatch
+		// [SRS] Pokemon S22E01-35 1080p WEBRip AAC 2.0 x264 CC.rar
+		// Pokemon.S22E24.The.Secret.Princess.DUBBED.1080p.WEBRip.AAC.2.0.x264-SRS.mkv
+		// Filename mismatch
+		// Adventure.Time.S06E10.Something.Big.1080p.HMAX.WEBRip.DD.2.0.H.265.-EDGE2020.mkv
+		// Adventure.Time.S06E03.James.II.1080p.HMAX.WEBRip.DD.2.0.H.265.-EDGE2020.mkv
+		// Action: schedule a "cleanup" job for the parent torrent
 		log.Println("Filename mismatch", resp.Filename, filenameV2)
 	}
 	http.Redirect(w, r, resp.Download, http.StatusFound)
 }
 
-// extractIDFromFilename extracts the link ID from a filename
-func extractIDFromFilename(filename string) (string, string) {
-	filenameV2, err := url.PathUnescape(filename)
-	if err != nil {
-		filenameV2 = filename
-	}
-	ext := filepath.Ext(filenameV2)
-	name := strings.TrimSuffix(filenameV2, ext)
-
-	r := regexp.MustCompile(`\sDMM(\w+)`)
-	matches := r.FindStringSubmatch(name)
-	if len(matches) < 2 {
-		// No ID found
-		return filenameV2, ""
-	}
-
-	// Remove ID from filename
-	originalName := strings.Replace(name, matches[0], "", 1)
-	return originalName + ext, matches[1]
-}
-
 // findLinkByFragment finds a link by a fragment, it might be wrong
-func findLinkByFragment(torrents []torrent.Torrent, fragment string) string {
+func findLinkByFragment(torrents []torrent.Torrent, filename, fragment string) string {
 	for _, torrent := range torrents {
-		for _, link := range torrent.Links {
+		for _, file := range torrent.SelectedFiles {
-			if strings.HasPrefix(link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) {
+			fname := filepath.Base(file.Path)
-				return link
+			if filename == fname && strings.HasPrefix(file.Link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) {
+				return file.Link
 			}
 		}
 	}

internal/dav/propfind.go

@@ -26,7 +26,7 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 	}
 	if err != nil {
 		log.Printf("Cannot marshal xml: %v\n", err.Error())
-		http.Error(w, fmt.Sprintf("Cannot marshal xml: %v", err.Error()), http.StatusInternalServerError)
+		http.Error(w, "Cannot read directory", http.StatusInternalServerError)
 		return
 	}
 	if output != nil {
@@ -54,8 +54,8 @@ func handleListOfTorrents(w http.ResponseWriter, r *http.Request, t *torrent.Tor
 	allTorrents := t.GetAll()
 	allTorrentsResponse, err := createMultiTorrentResponse(allTorrents)
 	if err != nil {
-		log.Printf("Cannot read directory: %v\n", err.Error())
+		log.Printf("Cannot read directory (/torrents): %v\n", err.Error())
-		http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError)
+		http.Error(w, "Cannot read directory", http.StatusInternalServerError)
 		return nil, nil
 	}
 	return xml.MarshalIndent(allTorrentsResponse, "", "  ")
@@ -67,15 +67,15 @@ func handleSingleTorrent(w http.ResponseWriter, r *http.Request, t *torrent.Torr
 	torrentName := path.Base(requestPath)
 	foundTorrents := findAllTorrentsWithName(t, torrentName)
 	if len(foundTorrents) == 0 {
-		log.Println("Cannot find directory")
+		log.Println("Cannot find directory", requestPath)
 		http.Error(w, "Cannot find directory", http.StatusNotFound)
 		return nil, nil
 	}
 	var torrentResponse *dav.MultiStatus
 	torrentResponse, err := createCombinedTorrentResponse(foundTorrents, t)
 	if err != nil {
-		log.Printf("Cannot read directory: %v\n", err.Error())
+		log.Printf("Cannot read directory (%s): %v\n", requestPath, err.Error())
-		http.Error(w, fmt.Sprintf("Cannot read directory: %v", err.Error()), http.StatusInternalServerError)
+		http.Error(w, "Cannot read directory", http.StatusInternalServerError)
 		return nil, nil
 	}
 	return xml.MarshalIndent(torrentResponse, "", "  ")

internal/dav/response.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/dav"
+	"github.com/debridmediamanager.com/zurg/pkg/davextra"
 )
 
 // createMultiTorrentResponse creates a WebDAV response for a list of torrents
@@ -33,7 +34,8 @@ func createMultiTorrentResponse(torrents []torrent.Torrent) (*dav.MultiStatus, e
 	}, nil
 }
 
-// createTorrentResponse creates a WebDAV response for torrents with the same name
+// createTorrentResponse creates a WebDAV response for a single torrent
+// but it also handles the case where there are many torrents with the same name
 func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.TorrentManager) (*dav.MultiStatus, error) {
 	var responses []dav.Response
 	// initial response is the directory itself
@@ -41,16 +43,13 @@ func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.Torren
 	responses = append(responses, dav.Directory(currentPath))
 
 	seen := make(map[string]bool)
-	idx := 0
 
 	var torrentResponses []dav.Response
 	for _, torrent := range torrents {
-		info := t.GetInfo(torrent.ID)
+		for _, file := range torrent.SelectedFiles {
-		if info == nil {
-			continue
-		}
-		for _, file := range info.SelectedFiles {
 			filename := filepath.Base(file.Path)
+			fragment := davextra.GetLinkFragment(file.Link)
+			filename = davextra.InsertLinkFragment(filename, fragment)
 			if _, exists := seen[filename]; exists {
 				continue
 			}
@@ -59,10 +58,9 @@ func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.Torren
 			torrentResponses = append(torrentResponses, dav.File(
 				filePath,
 				file.Bytes,
-				convertDate(info.Added),
+				convertDate(torrent.Added),
-				info.Links[idx],
+				file.Link,
 			))
-			idx++
 		}
 	}
 	responses = append(responses, torrentResponses...)

internal/dav/router.go

@@ -4,7 +4,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path"
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 )
@@ -14,9 +13,8 @@ func Router(mux *http.ServeMux) {
 	t := torrent.NewTorrentManager(os.Getenv("RD_TOKEN"))
 
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		requestPath := path.Clean(r.URL.Path)
+		// requestPath := path.Clean(r.URL.Path)
-
+		// log.Println(r.Method, requestPath)
-		log.Println(r.Method, requestPath)
 
 		switch r.Method {
 		case "PROPFIND":
@@ -24,6 +22,7 @@ func Router(mux *http.ServeMux) {
 
 		case http.MethodGet:
 			HandleGetRequest(w, r, t)
+			// default return
 
 		case http.MethodOptions:
 			w.WriteHeader(http.StatusOK)

internal/dav/util.go

@@ -2,6 +2,7 @@ package dav
 
 import (
 	"log"
+	"strings"
 	"time"
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
@@ -23,7 +24,7 @@ func findAllTorrentsWithName(t *torrent.TorrentManager, filename string) []torre
 
 	torrents := t.GetAll()
 	for _, torrent := range torrents {
-		if torrent.Filename == filename {
+		if torrent.Filename == filename || strings.HasPrefix(torrent.Filename, filename) {
 			matchingTorrents = append(matchingTorrents, torrent)
 		}
 	}

internal/torrent/manager.go

@@ -1,7 +1,9 @@
 package torrent
 
 import (
+	"encoding/gob"
 	"log"
+	"os"
 	"strings"
 	"sync"
 
@@ -58,6 +60,14 @@ func (t *TorrentManager) GetAll() []Torrent {
 }
 
 func (t *TorrentManager) getInfo(torrentID string) *Torrent {
+	torrentFromFile := t.readFromFile(torrentID)
+	if torrentFromFile != nil {
+		torrent := t.getByID(torrentID)
+		if torrent != nil {
+			torrent.SelectedFiles = torrentFromFile.SelectedFiles
+		}
+		return torrent
+	}
 	info, err := realdebrid.GetTorrentInfo(t.token, torrentID)
 	if err != nil {
 		log.Printf("Cannot get info: %v\n", err.Error())
@@ -123,26 +133,58 @@ func (t *TorrentManager) getInfo(torrentID string) *Torrent {
 	if torrent != nil {
 		torrent.SelectedFiles = selectedFiles
 	}
-	log.Println("Fetched info for", info.Filename)
+	if len(torrent.SelectedFiles) > 0 {
+		t.writeToFile(torrentID, torrent)
+	}
 	return torrent
 }
 
 func (t *TorrentManager) GetInfo(torrentID string) *Torrent {
 	for _, torrent := range t.torrents {
 		if torrent.ID == torrentID {
-			if torrent.SelectedFiles != nil {
+			return &torrent
+		}
+	}
 	return t.getInfo(torrentID)
 }
+
+func (t *TorrentManager) getByID(torrentID string) *Torrent {
+	for i, torrent := range t.torrents {
+		if torrent.ID == torrentID {
+			return &t.torrents[i]
 		}
 	}
 	return nil
 }
 
-func (t *TorrentManager) getByID(torrentID string) *Torrent {
+func (t *TorrentManager) writeToFile(torrentID string, torrent *Torrent) {
-	for _, torrent := range t.torrents {
+	filePath := "data/" + torrentID + ".bin"
-		if torrent.ID == torrentID {
+	file, err := os.Create(filePath)
-			return &torrent
+	if err != nil {
+		log.Fatalf("Failed creating file: %s", err)
+		return
 	}
+	defer file.Close()
+
+	dataEncoder := gob.NewEncoder(file)
+	dataEncoder.Encode(torrent)
 }
+
+func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
+	filePath := "data/" + torrentID + ".bin"
+	file, err := os.Open(filePath)
+	if err != nil {
 		return nil
 	}
+	defer file.Close()
+
+	var torrent Torrent
+	dataDecoder := gob.NewDecoder(file)
+	err = dataDecoder.Decode(&torrent)
+	if err != nil {
+		log.Fatalf("Failed decoding file: %s", err)
+		return nil
+	}
+
+	return &torrent
+}

pkg/davextra/util.go

@@ -2,11 +2,13 @@ package davextra
 
 import (
 	"fmt"
+	"net/url"
 	"path/filepath"
 	"regexp"
 	"strings"
 )
 
+// GetLinkFragment returns the longest alphanumeric string from a link
 func GetLinkFragment(link string) string {
 	re := regexp.MustCompile(`\w+`)
 	matches := re.FindAllString(link, -1) // Returns all matches
@@ -19,8 +21,32 @@ func GetLinkFragment(link string) string {
 	return longestMatch[:4]
 }
 
+// InsertLinkFragment inserts the link ID into a filename
+// returns the new filename
 func InsertLinkFragment(filename, dupeID string) string {
 	ext := filepath.Ext(filename)
 	name := strings.TrimSuffix(filename, ext)
 	return fmt.Sprintf("%s DMM%s%s", name, dupeID, ext)
 }
+
+// ExtractLinkFragment extracts the link ID from a filename
+// returns the original filename and the link ID
+func ExtractLinkFragment(filename string) (string, string) {
+	filenameV2, err := url.PathUnescape(filename)
+	if err != nil {
+		filenameV2 = filename
+	}
+	ext := filepath.Ext(filenameV2)
+	name := strings.TrimSuffix(filenameV2, ext)
+
+	r := regexp.MustCompile(`\sDMM(\w+)`)
+	matches := r.FindStringSubmatch(name)
+	if len(matches) < 2 {
+		// No ID found
+		return filenameV2, ""
+	}
+
+	// Remove ID from filename
+	originalName := strings.Replace(name, matches[0], "", 1)
+	return originalName + ext, matches[1]
+}

pkg/realdebrid/api.go

@@ -88,7 +88,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) {
 	baseURL := "https://api.real-debrid.com/rest/1.0/torrents"
 	var allTorrents []Torrent
 	page := 1
-	limit := 2500
+	limit := 100
 
 	for {
 		params := url.Values{}
@@ -124,7 +124,7 @@ func GetTorrents(accessToken string) ([]Torrent, error) {
 
 		allTorrents = append(allTorrents, torrents...)
 
-		totalCountHeader := resp.Header.Get("x-total-count")
+		totalCountHeader := "100" // resp.Header.Get("x-total-count")
 		totalCount, err := strconv.Atoi(totalCountHeader)
 		if err != nil {
 			break