fuse integreated

.github/workflows/docker-build.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository }}-testing
 
 jobs:
   build-and-push-image:

Dockerfile

@@ -9,11 +9,11 @@ COPY . .
 RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags="-s -w" -o zurg cmd/zurg/main.go
 
 # Obfuscation stage
-# FROM alpine:3 AS obfuscator
+FROM alpine:3 AS obfuscator
-# WORKDIR /app
+WORKDIR /app
-# COPY --from=builder /app/zurg .
+COPY --from=builder /app/zurg .
-# RUN apk add --no-cache upx
+RUN apk add --no-cache upx
-# RUN upx --brute zurg
+RUN upx --brute zurg
 
 # Final stage
 FROM alpine:3

cmd/zurg/main.go

@@ -16,12 +16,17 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/zfs"
 	"github.com/debridmediamanager.com/zurg/pkg/chunk"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
+	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
 func main() {
-	rlog := logutil.NewLogger()
+	if len(os.Args) > 1 && os.Args[1] == "networktest" {
-	log := rlog.Named("zurg")
+		realdebrid.RunTest()
+		return
+	}
+
+	log := logutil.NewLogger().Named("zurg")
 
 	config, configErr := config.LoadZurgConfig("./config.yml")
 	if configErr != nil {
@@ -30,7 +35,9 @@ func main() {
 
 	cache := expirable.NewLRU[string, string](1e4, nil, time.Hour)
 
-	torrentMgr := torrent.NewTorrentManager(config, cache)
+	rd := realdebrid.NewRealDebrid(config.GetToken(), config, logutil.NewLogger().Named("realdebrid"))
+
+	torrentMgr := torrent.NewTorrentManager(config, rd)
 
 	mux := http.NewServeMux()
 	net.Router(mux, config, torrentMgr, cache)
@@ -53,7 +60,8 @@ func main() {
 		1,                          // 1 chunk - load ahead (1MB total)
 		max(runtime.NumCPU()/2, 1), // check threads
 		max(runtime.NumCPU()/2, 1), // load threads
-		runtime.NumCPU()*2,         // max chunks
+		runtime.NumCPU()*2,
+		torrentMgr, // max chunks
 		config)
 	if nil != err {
 		log.Panicf("Failed to initialize chunk manager: %v", err)

go.mod

@@ -4,9 +4,10 @@ go 1.21.3
 
 require (
 	bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
+	github.com/elliotchance/orderedmap/v2 v2.2.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	go.uber.org/zap v1.26.0
-	golang.org/x/sys v0.4.0
+	golang.org/x/sys v0.14.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -2,6 +2,8 @@ bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 h1:A0NsYy4lDBZAC6QiYeJ4N+XuHIK
 bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5/go.mod h1:gG3RZAMXCa/OTes6rr9EwusmR1OH1tDDy+cg9c5YliY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
+github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -16,8 +18,8 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
 go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
-golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

hosts.txt

@@ -0,0 +1,76 @@
+20.download.real-debrid.cloud
+21.download.real-debrid.cloud
+22.download.real-debrid.cloud
+23.download.real-debrid.cloud
+30.download.real-debrid.cloud
+31.download.real-debrid.cloud
+32.download.real-debrid.cloud
+34.download.real-debrid.cloud
+40.download.real-debrid.cloud
+41.download.real-debrid.cloud
+42.download.real-debrid.cloud
+43.download.real-debrid.cloud
+44.download.real-debrid.cloud
+45.download.real-debrid.cloud
+50.download.real-debrid.cloud
+51.download.real-debrid.cloud
+52.download.real-debrid.cloud
+53.download.real-debrid.cloud
+54.download.real-debrid.cloud
+55.download.real-debrid.cloud
+56.download.real-debrid.cloud
+57.download.real-debrid.cloud
+58.download.real-debrid.cloud
+59.download.real-debrid.cloud
+60.download.real-debrid.cloud
+61.download.real-debrid.cloud
+62.download.real-debrid.cloud
+63.download.real-debrid.cloud
+64.download.real-debrid.cloud
+65.download.real-debrid.cloud
+66.download.real-debrid.cloud
+67.download.real-debrid.cloud
+68.download.real-debrid.cloud
+69.download.real-debrid.cloud
+20.download.real-debrid.com
+21.download.real-debrid.com
+22.download.real-debrid.com
+23.download.real-debrid.com
+30.download.real-debrid.com
+31.download.real-debrid.com
+32.download.real-debrid.com
+34.download.real-debrid.com
+40.download.real-debrid.com
+41.download.real-debrid.com
+42.download.real-debrid.com
+43.download.real-debrid.com
+44.download.real-debrid.com
+45.download.real-debrid.com
+50.download.real-debrid.com
+51.download.real-debrid.com
+52.download.real-debrid.com
+53.download.real-debrid.com
+54.download.real-debrid.com
+55.download.real-debrid.com
+56.download.real-debrid.com
+57.download.real-debrid.com
+58.download.real-debrid.com
+59.download.real-debrid.com
+60.download.real-debrid.com
+61.download.real-debrid.com
+62.download.real-debrid.com
+63.download.real-debrid.com
+64.download.real-debrid.com
+65.download.real-debrid.com
+66.download.real-debrid.com
+67.download.real-debrid.com
+68.download.real-debrid.com
+69.download.real-debrid.com
+hkg1.download.real-debrid.com
+lax1.download.real-debrid.com
+lon1.download.real-debrid.com
+mum1.download.real-debrid.com
+rbx.download.real-debrid.com
+sgp1.download.real-debrid.com
+tlv1.download.real-debrid.com
+tyo1.download.real-debrid.com

internal/config/load.go

@@ -23,11 +23,11 @@ type ConfigInterface interface {
 	GetNetworkBufferSize() int
 	GetMountPoint() string
 	EnableRetainFolderNameExtension() bool
+	GetRandomPreferredHost() string
 }
 
 func LoadZurgConfig(filename string) (ConfigInterface, error) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("config")
-	log := rlog.Named("config")
 
 	log.Debug("Loading config file ", filename)
 	content, err := os.ReadFile(filename)

internal/config/types.go

@@ -1,5 +1,7 @@
 package config
 
+import "math/rand"
+
 type ZurgConfig struct {
 	Version                   string   `yaml:"zurg"`
 	Token                     string   `yaml:"token"`
@@ -13,6 +15,7 @@ type ZurgConfig struct {
 	NetworkBufferSize         int      `yaml:"network_buffer_size"`
 	MountPoint                string   `yaml:"mount_point"`
 	RetainFolderNameExtension bool     `yaml:"retain_folder_name_extension"`
+	PreferredHosts            []string `yaml:"preferred_hosts"`
 }
 
 func (z *ZurgConfig) GetToken() string {
@@ -73,3 +76,11 @@ func (z *ZurgConfig) GetMountPoint() string {
 func (z *ZurgConfig) EnableRetainFolderNameExtension() bool {
 	return z.RetainFolderNameExtension
 }
+
+func (z *ZurgConfig) GetRandomPreferredHost() string {
+	if len(z.PreferredHosts) == 0 {
+		return ""
+	}
+	randomIndex := rand.Intn(len(z.PreferredHosts))
+	return z.PreferredHosts[randomIndex]
+}

internal/config/v1.go

@@ -5,7 +5,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/debridmediamanager.com/zurg/pkg/logutil"
 	"gopkg.in/yaml.v3"
 )
 
@@ -33,9 +32,6 @@ func (z *ZurgConfigV1) GetDirectories() []string {
 }
 
 func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
-	rlog := logutil.NewLogger()
-	log := rlog.Named("config")
-
 	var groupMap = make(map[string][]string)
 	var groupOrderMap = make(map[string]int) // To store GroupOrder for each directory
 
@@ -43,7 +39,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
 	for directory, val := range z.Directories {
 		groupMap[val.Group] = append(groupMap[val.Group], directory)
 		groupOrderMap[directory] = val.GroupOrder
-		log.Debugf("Added directory to group: %s, group: %s, order: %d", directory, val.Group, val.GroupOrder)
 	}
 
 	// Sort the slice based on GroupOrder and then directory name for deterministic order
@@ -55,7 +50,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
 			return groupOrderMap[dirs[i]] < groupOrderMap[dirs[j]]
 		})
 		groupMap[group] = dirs
-		log.Debugf("Sorted directories within a group: %s %v", group, dirs)
 	}
 
 	// Return a deep copy of the map

internal/dav/listing.go

@@ -5,29 +5,21 @@ import (
 	"fmt"
 	"net/http"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/dav"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
+func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("dav")
-	log := rlog.Named("dav")
 
 	requestPath := path.Clean(r.URL.Path)
 	requestPath = strings.Trim(requestPath, "/")
 
-	if data, exists := cache.Get(requestPath); exists {
-		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
-		w.WriteHeader(http.StatusMultiStatus)
-		fmt.Fprint(w, data)
-		return
-	}
-
 	var output []byte
 	var err error
 
@@ -41,12 +33,16 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 	case len(filteredSegments) == 2:
 		output, err = handleSingleTorrent(requestPath, w, r, t)
 	default:
-		log.Errorf("Request %s %s not found", r.Method, requestPath)
+		log.Warnf("Request %s %s not found", r.Method, requestPath)
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
 
 	if err != nil {
+		if strings.Contains(err.Error(), "cannot find") {
+			http.Error(w, "Not Found", http.StatusNotFound)
+			return
+		}
 		log.Errorf("Error processing request: %v", err)
 		http.Error(w, "Server error", http.StatusInternalServerError)
 		return
@@ -54,8 +50,6 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 
 	if output != nil {
 		respBody := fmt.Sprintf("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n%s\n", output)
-		cache.Add(requestPath, respBody)
-
 		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
 		w.WriteHeader(http.StatusMultiStatus)
 		fmt.Fprint(w, respBody)
@@ -64,9 +58,9 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 
 func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface) ([]byte, error) {
 	var responses []dav.Response
-	responses = append(responses, dav.Directory("/"))
+	responses = append(responses, dav.Directory(""))
 	for _, directory := range c.GetDirectories() {
-		responses = append(responses, dav.Directory("/"+directory))
+		responses = append(responses, dav.Directory(directory))
 	}
 	rootResponse := dav.MultiStatus{
 		XMLNS:    "DAV:",
@@ -80,10 +74,28 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 
 	for _, directory := range c.GetDirectories() {
 		if basePath == directory {
-			torrents := t.GetByDirectory(basePath)
+			var responses []dav.Response
-			resp, err := createMultiTorrentResponse("/"+basePath, torrents)
+
-			if err != nil {
+			responses = append(responses, dav.Directory(basePath))
-				return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+
+			for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+				accessKey := el.Key
+				torrent := el.Value
+				if torrent.InProgress {
+					continue
+				}
+				for _, dir := range torrent.Directories {
+					if dir == basePath {
+						path := filepath.Join(basePath, accessKey)
+						responses = append(responses, dav.Directory(path))
+						break
+					}
+				}
+			}
+
+			resp := &dav.MultiStatus{
+				XMLNS:    "DAV:",
+				Response: responses,
 			}
 			return xml.Marshal(resp)
 		}
@@ -93,17 +105,37 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 }
 
 func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) {
-	directory := path.Dir(requestPath)
+	accessKey := path.Base(requestPath)
-	torrentName := path.Base(requestPath)
+	torrent, exists := t.TorrentMap.Get(accessKey)
-
+	if !exists {
-	sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
+		return nil, fmt.Errorf("cannot find torrent %s", accessKey)
-	if len(sameNameTorrents) == 0 {
-		return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
 	}
 
-	resp, err := createSingleTorrentResponse("/"+directory, sameNameTorrents)
+	var responses []dav.Response
-	if err != nil {
+
-		return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
+	// initial response is the directory itself
+	responses = append(responses, dav.Directory(requestPath))
+
+	for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+		file := el.Value
+		if file.Link == "" {
+			// will be caught by torrent manager's repairAll
+			// just skip it for now
+			continue
+		}
+		filename := filepath.Base(file.Path)
+		filePath := filepath.Join(requestPath, filename)
+		responses = append(responses, dav.File(
+			filePath,
+			file.Bytes,
+			convertRFC3339toRFC1123(torrent.LatestAdded),
+			file.Link,
+		))
+	}
+
+	resp := &dav.MultiStatus{
+		XMLNS:    "DAV:",
+		Response: responses,
 	}
 	return xml.Marshal(resp)
 }

internal/dav/response.go

@@ -1,79 +0,0 @@
-package dav
-
-import (
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-	"github.com/debridmediamanager.com/zurg/pkg/dav"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*dav.MultiStatus, error) {
-	var responses []dav.Response
-	responses = append(responses, dav.Directory(basePath))
-
-	seen := make(map[string]bool)
-
-	for _, item := range torrents {
-		if item.Progress != 100 {
-			continue
-		}
-		if _, exists := seen[item.Name]; exists {
-			continue
-		}
-		seen[item.Name] = true
-
-		path := filepath.Join(basePath, item.Name)
-		responses = append(responses, dav.Directory(path))
-	}
-
-	return &dav.MultiStatus{
-		XMLNS:    "DAV:",
-		Response: responses,
-	}, nil
-}
-
-// createTorrentResponse creates a WebDAV response for a single torrent
-// but it also handles the case where there are many torrents with the same name
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*dav.MultiStatus, error) {
-	var responses []dav.Response
-
-	// initial response is the directory itself
-	currentPath := filepath.Join(basePath, torrents[0].Name)
-	responses = append(responses, dav.Directory(currentPath))
-
-	finalName := make(map[string]bool)
-
-	var torrentResponses []dav.Response
-
-	for _, torrent := range torrents {
-		for _, file := range torrent.SelectedFiles {
-			if file.Link == "" {
-				// TODO: fix the file?
-				// log.Println("File has no link, skipping (repairing links take time)", file.Path)
-				continue
-			}
-
-			filename := filepath.Base(file.Path)
-			if finalName[filename] {
-				continue
-			}
-			finalName[filename] = true
-
-			filePath := filepath.Join(currentPath, filename)
-			torrentResponses = append(torrentResponses, dav.File(
-				filePath,
-				file.Bytes,
-				convertRFC3339toRFC1123(torrent.Added),
-				file.Link,
-			))
-		}
-	}
-
-	responses = append(responses, torrentResponses...)
-
-	return &dav.MultiStatus{
-		XMLNS:    "DAV:",
-		Response: responses,
-	}, nil
-}

internal/http/listing.go

@@ -5,27 +5,19 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 )
 
-func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
+func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("http")
-	log := rlog.Named("http")
 
 	requestPath := path.Clean(r.URL.Path)
 
-	if data, exists := cache.Get(requestPath); exists {
-		w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
-		w.WriteHeader(http.StatusOK)
-		fmt.Fprint(w, data)
-		return
-	}
-
 	var output *string
 	var err error
 
@@ -38,20 +30,21 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
 	case len(filteredSegments) == 3:
 		output, err = handleSingleTorrent(requestPath, w, r, t)
 	default:
-		log.Errorf("Request %s %s not found", r.Method, requestPath)
+		log.Warnf("Request %s %s not found", r.Method, requestPath)
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
-
 	if err != nil {
+		if strings.Contains(err.Error(), "cannot find") {
+			http.Error(w, "Not Found", http.StatusNotFound)
+			return
+		}
 		log.Errorf("Error processing request: %v", err)
 		http.Error(w, "Server error", http.StatusInternalServerError)
 		return
 	}
 
 	if output != nil {
-		cache.Add(requestPath, *output)
-
 		w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
 		w.WriteHeader(http.StatusOK)
 		fmt.Fprint(w, *output)
@@ -74,12 +67,21 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 
 	for _, directory := range c.GetDirectories() {
 		if basePath == directory {
-			torrents := t.GetByDirectory(basePath)
+			htmlDoc := "<ol>"
-			resp, err := createMultiTorrentResponse(requestPath, torrents)
+			for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
-			if err != nil {
+				accessKey := el.Key
-				return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+				torrent := el.Value
+				if torrent.InProgress {
+					continue
 				}
-			return &resp, nil
+				for _, dir := range torrent.Directories {
+					if dir == basePath {
+						htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(accessKey)), accessKey)
+						break
+					}
+				}
+			}
+			return &htmlDoc, nil
 		}
 	}
 
@@ -87,18 +89,23 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
 }
 
 func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) {
-	fullDir := path.Dir(requestPath)
+	accessKey := path.Base(requestPath)
-	directory := path.Base(fullDir)
+	torrent, _ := t.TorrentMap.Get(accessKey)
-	torrentName := path.Base(requestPath)
+	if torrent == nil {
-
+		return nil, fmt.Errorf("cannot find torrent %s", accessKey)
-	sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
-	if len(sameNameTorrents) == 0 {
-		return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
 	}
 
-	resp, err := createSingleTorrentResponse(requestPath, sameNameTorrents)
+	htmlDoc := "<ol>"
-	if err != nil {
+	for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
-		return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
+		file := el.Value
+		if file.Link == "" {
+			// will be caught by torrent manager's repairAll
+			// just skip it for now
+			continue
 		}
-	return &resp, nil
+		filename := filepath.Base(file.Path)
+		filePath := filepath.Join(requestPath, url.PathEscape(filename))
+		htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
+	}
+	return &htmlDoc, nil
 }

internal/http/response.go

@@ -1,60 +0,0 @@
-package http
-
-import (
-	"fmt"
-	"net/url"
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
-	htmlDoc := "<ul>"
-
-	seen := make(map[string]bool)
-
-	for _, item := range torrents {
-		if item.Progress != 100 {
-			continue
-		}
-		if _, exists := seen[item.Name]; exists {
-			continue
-		}
-		seen[item.Name] = true
-
-		path := filepath.Join(basePath, url.PathEscape(item.Name))
-		htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", path, item.Name)
-	}
-
-	return htmlDoc, nil
-}
-
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
-	htmlDoc := "<ul>"
-
-	finalName := make(map[string]bool)
-
-	currentPath := filepath.Join(basePath)
-
-	for _, torrent := range torrents {
-		for _, file := range torrent.SelectedFiles {
-			if file.Link == "" {
-				// TODO: fix the file?
-				// log.Println("File has no link, skipping", file.Path)
-				continue
-			}
-
-			filename := filepath.Base(file.Path)
-			if finalName[filename] {
-				continue
-			}
-			finalName[filename] = true
-
-			filePath := filepath.Join(currentPath, url.PathEscape(filename))
-			htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
-		}
-	}
-
-	return htmlDoc, nil
-}

internal/net/router.go

@@ -16,8 +16,7 @@ import (
 
 // Router creates a WebDAV router
 func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentManager, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("net")
-	log := rlog.Named("net")
 
 	mux.HandleFunc("/http/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
@@ -26,7 +25,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
 			if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 {
 				universal.HandleGetRequest(w, r, t, c, cache)
 			} else {
-				intHttp.HandleDirectoryListing(w, r, t, c, cache)
+				intHttp.HandleDirectoryListing(w, r, t, c)
 			}
 
 		case http.MethodHead:
@@ -41,7 +40,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
 		case "PROPFIND":
-			dav.HandlePropfindRequest(w, r, t, c, cache)
+			dav.HandlePropfindRequest(w, r, t, c)
 
 		case http.MethodGet:
 			universal.HandleGetRequest(w, r, t, c, cache)

internal/torrent/hooks.go

@@ -30,8 +30,7 @@ func (se *ScriptExecutor) Execute() (string, error) {
 }
 
 func OnLibraryUpdateHook(config config.ConfigInterface) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("hooks")
-	log := rlog.Named("hooks")
 
 	executor := &ScriptExecutor{
 		Script: config.GetOnLibraryUpdate(),

internal/torrent/types.go

@@ -1,21 +1,22 @@
 package torrent
 
 import (
-	"sync"
-
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
+	"github.com/elliotchance/orderedmap/v2"
 )
 
 type Torrent struct {
-	Version string
+	AccessKey     string
-	realdebrid.Torrent
+	SelectedFiles *orderedmap.OrderedMap[string, *File]
-	SelectedFiles []File
+	Directories   []string
+	LatestAdded   string
+	InProgress    bool
 	ForRepair     bool
-	lock          *sync.Mutex
+
+	Instances []realdebrid.TorrentInfo
 }
 
 type File struct {
 	realdebrid.File
 	Link string
-	Unavailable bool
 }

internal/universal/get.go

@@ -11,16 +11,15 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/dav"
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
+	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/hashicorp/golang-lru/v2/expirable"
 	"go.uber.org/zap"
 )
 
 // HandleGetRequest handles a GET request universally for both WebDAV and HTTP
 func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("uniget")
-	log := rlog.Named("uniget")
 
 	requestPath := path.Clean(r.URL.Path)
 	isDav := true
@@ -35,93 +34,99 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
 	// If there are less than 3 segments, return an error or adjust as needed
 	if len(segments) <= 3 {
 		if isDav {
-			dav.HandlePropfindRequest(w, r, t, c, cache)
+			dav.HandlePropfindRequest(w, r, t, c)
 		} else {
-			intHttp.HandleDirectoryListing(w, r, t, c, cache)
+			intHttp.HandleDirectoryListing(w, r, t, c)
 		}
 		return
 	}
-	if data, exists := cache.Get(requestPath); exists {
-		streamFileToResponse(data, w, r, c, log)
-		return
-	}
 
 	baseDirectory := segments[len(segments)-3]
-	torrentName := segments[len(segments)-2]
+	accessKey := segments[len(segments)-2]
 	filename := segments[len(segments)-1]
 
-	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
+	torrent, _ := t.TorrentMap.Get(accessKey)
-	if torrents == nil {
+	if torrent == nil {
-		log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
+		log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
-	torrent, file := getFile(torrents, filename)
+	file, _ := torrent.SelectedFiles.Get(filename)
 	if file == nil {
-		log.Errorf("Cannot find file from path %s", requestPath)
+		log.Warnf("Cannot find file from path %s", requestPath)
 		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
+
+	if data, exists := cache.Get(requestPath); exists {
+		streamFileToResponse(torrent, data, w, r, t, c, log)
+		return
+	}
+
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
-		log.Errorf("File %s is no longer available", filename)
+		log.Warnf("File %s is not yet available, zurg is repairing the torrent", filename)
-		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, c, log)
+		streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, t, c, log)
 		return
 	}
 	link := file.Link
 
-	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
+	resp := t.UnrestrictUntilOk(link)
-		return realdebrid.UnrestrictLink(c.GetToken(), link)
-	}
-	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
-		if !file.Unavailable {
+		go t.Repair(torrent.AccessKey)
-			log.Errorf("Cannot unrestrict file %s %s", filename, link)
+		log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path)
-			t.HideTheFile(torrent, file)
+		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
-		}
-		streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, c, log)
 		return
 	} else if resp.Filename != filename {
 		actualExt := filepath.Ext(resp.Filename)
 		expectedExt := filepath.Ext(filename)
 		if actualExt != expectedExt && resp.Streamable != 1 {
-			log.Errorf("File extension mismatch: %s and %s", filename, resp.Filename)
+			log.Warnf("File was changed and is not streamable: %s and %s", filename, resp.Filename)
-			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, c, log)
+			streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, t, c, log)
 			return
 		} else {
-			log.Errorf("Filename mismatch: %s and %s", filename, resp.Filename)
+			log.Warnf("Filename mismatch: %s and %s", filename, resp.Filename)
 		}
 	}
 	cache.Add(requestPath, resp.Download)
-	streamFileToResponse(resp.Download, w, r, c, log)
+	streamFileToResponse(torrent, resp.Download, w, r, t, c, log)
 }
 
-func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
+func streamFileToResponse(torrent *torrent.Torrent, url string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
+	// Create a new request for the file download.
 	req, err := http.NewRequest(http.MethodGet, url, nil)
 	if err != nil {
-		log.Errorf("Error creating new request %v", err)
+		log.Errorf("Error creating new request: %v", err)
-		streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, c, log)
+		streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, t, c, log)
 		return
 	}
 
-	for k, values := range r.Header {
+	// copy range header if it exists
-		for _, v := range values {
+	if r.Header.Get("Range") != "" {
-			req.Header.Add(k, v)
+		req.Header.Add("Range", r.Header.Get("Range"))
-		}
 	}
 
-	resp, err := http.DefaultClient.Do(req)
+	// Create a custom HTTP client
+	client := zurghttp.NewHTTPClient(c.GetToken(), 10, c)
+
+	resp, err := client.Do(req)
 	if err != nil {
-		log.Errorf("Error downloading file %v", err)
+		log.Warnf("Cannot download file %v ; torrent is marked for repair", err)
-		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, c, log)
+		if torrent != nil {
+			go t.Repair(torrent.AccessKey)
+		}
+		streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, t, c, log)
 		return
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		log.Errorf("Received a nonOK status code %d", resp.StatusCode)
+		log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status)
-		streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, c, log)
+		if torrent != nil {
+			go t.Repair(torrent.AccessKey)
+		}
+		streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, t, c, log)
 		return
 	}
 
@@ -135,14 +140,11 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c
 	io.CopyBuffer(w, resp.Body, buf)
 }
 
-func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
+func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
-	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
+	resp := t.UnrestrictUntilOk(link)
-		return realdebrid.UnrestrictLink(c.GetToken(), link)
-	}
-	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
 		http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError)
 		return
 	}
-	streamFileToResponse(resp.Download, w, r, c, log)
+	streamFileToResponse(nil, resp.Download, w, r, t, c, log)
 }

internal/universal/head.go

@@ -14,8 +14,7 @@ import (
 )
 
 func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
-	rlog := logutil.NewLogger()
+	log := logutil.NewLogger().Named("unihead")
-	log := rlog.Named("unihead")
 
 	requestPath := path.Clean(r.URL.Path)
 	requestPath = strings.Replace(requestPath, "/http", "", 1)
@@ -42,25 +41,25 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
 	}
 
 	baseDirectory := segments[len(segments)-3]
-	torrentName := segments[len(segments)-2]
+	accessKey := segments[len(segments)-2]
 	filename := segments[len(segments)-1]
 
-	torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
+	torrent, _ := t.TorrentMap.Get(accessKey)
-	if torrents == nil {
+	if torrent == nil {
-		log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
+		log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
-		http.Error(w, "Cannot find file", http.StatusNotFound)
+		http.Error(w, "File not found", http.StatusNotFound)
 		return
 	}
 
-	_, file := getFile(torrents, filename)
+	file, _ := torrent.SelectedFiles.Get(filename)
 	if file == nil {
-		log.Errorf("Cannot find file from path %s", requestPath)
+		log.Warnf("Cannot find file from path %s", requestPath)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}
 	if file.Link == "" {
 		// This is a dead file, serve an alternate file
-		log.Errorf("File %s is no longer available", filename)
+		log.Warnf("File %s is no longer available", filename)
 		http.Error(w, "Cannot find file", http.StatusNotFound)
 		return
 	}

internal/universal/util.go

@@ -1,20 +0,0 @@
-package universal
-
-import (
-	"path/filepath"
-
-	"github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// getFile finds a link by a fragment, it might be wrong
-func getFile(torrents []torrent.Torrent, filename string) (*torrent.Torrent, *torrent.File) {
-	for t := range torrents {
-		for f, file := range torrents[t].SelectedFiles {
-			fname := filepath.Base(file.Path)
-			if filename == fname {
-				return &torrents[t], &torrents[t].SelectedFiles[f]
-			}
-		}
-	}
-	return nil, nil
-}

internal/zfs/object.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 	"syscall"
 	"time"
 
@@ -64,42 +63,34 @@ func (o Object) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 			})
 		}
 	case DIRECTORY:
-		seen := make(map[string]bool)
+		for el := o.fs.t.TorrentMap.Front(); el != nil; el = el.Next() {
-		for _, item := range o.fs.t.GetByDirectory(o.name) {
+			item := el.Value
-			if item.Progress != 100 {
+			if item.InProgress {
 				continue
 			}
-			if _, exists := seen[item.Name]; exists {
-				continue
-			}
-			seen[item.Name] = true
 			dirs = append(dirs, fuse.Dirent{
-				Name: item.Name,
+				Name: item.AccessKey,
 				Type: fuse.DT_Dir,
 			})
 		}
 	case TORRENT:
-		finalName := make(map[string]bool)
+		torrent, _ := o.fs.t.TorrentMap.Get(o.name)
-		for _, item := range o.fs.t.FindAllTorrentsWithName(o.parentName, o.name) {
+		if torrent == nil {
-			for _, file := range item.SelectedFiles {
+			return nil, syscall.ENOENT
+		}
+		for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+			file := el.Value
 			if file.Link == "" {
 				// log.Println("File has no link, skipping", file.Path)
 				continue
 			}
 			filename := filepath.Base(file.Path)
-				if finalName[filename] {
-					// fragment := davextra.GetLinkFragment(file.Link)
-					// filename = davextra.InsertLinkFragment(filename, fragment)
-					continue
-				}
-				finalName[filename] = true
 			dirs = append(dirs, fuse.Dirent{
 				Name: filename,
 				Type: fuse.DT_File,
 			})
 		}
 	}
-	}
 	return dirs, nil
 }
 
@@ -119,34 +110,37 @@ func (o Object) Lookup(ctx context.Context, name string) (fs.Node, error) {
 			}
 		}
 	case DIRECTORY:
-		for _, item := range o.fs.t.GetByDirectory(o.name) {
+		torrent, _ := o.fs.t.TorrentMap.Get(name)
-			if item.Name == name && item.Progress == 100 {
+		if torrent == nil {
+			return nil, syscall.ENOENT
+		}
 		return Object{
 			fs:         o.fs,
 			objType:    TORRENT,
 			parentName: o.name,
 			name:       name,
-					mtime:      convertRFC3339toTime(item.Added),
+			mtime:      convertRFC3339toTime(torrent.LatestAdded),
 		}, nil
-			}
+
-		}
 	case TORRENT:
-		for _, item := range o.fs.t.FindAllTorrentsWithName(o.parentName, o.name) {
+		torrent, _ := o.fs.t.TorrentMap.Get(name)
-			for _, file := range item.SelectedFiles {
+		if torrent == nil {
-				if strings.HasSuffix(file.Path, name) && file.Link != "" {
+			return nil, syscall.ENOENT
+		}
+		file, _ := torrent.SelectedFiles.Get(name)
+		if file == nil {
+			return nil, syscall.ENOENT
+		}
 		return Object{
 			fs:         o.fs,
 			objType:    FILE,
 			parentName: o.name,
 			name:       name,
-						file:       &file,
+			file:       file,
 			size:       uint64(file.Bytes),
-						mtime:      convertRFC3339toTime(item.Added),
+			mtime:      convertRFC3339toTime(torrent.LatestAdded),
 		}, nil
 	}
-			}
-		}
-	}
 	return nil, syscall.ENOENT
 }
 

pkg/chunk/download.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
+	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/logutil"
-	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"go.uber.org/zap"
 	"golang.org/x/sys/unix"
 )
@@ -23,13 +23,14 @@ type Downloader struct {
 	lock       sync.Mutex
 	storage    *Storage
 	c          config.ConfigInterface
+	t          *torrent.TorrentManager
 	log        *zap.SugaredLogger
 }
 
 type DownloadCallback func(error, []byte)
 
 // NewDownloader creates a new download manager
-func NewDownloader(threads int, storage *Storage, bufferSize int64, c config.ConfigInterface) (*Downloader, error) {
+func NewDownloader(threads int, storage *Storage, bufferSize int64, t *torrent.TorrentManager, c config.ConfigInterface) (*Downloader, error) {
 	rlog := logutil.NewLogger()
 	log := rlog.Named("downloader")
 
@@ -39,6 +40,7 @@ func NewDownloader(threads int, storage *Storage, bufferSize int64, c config.Con
 		callbacks:  make(map[RequestID][]DownloadCallback, 100),
 		storage:    storage,
 		c:          c,
+		t:          t,
 		log:        log,
 	}
 
@@ -103,10 +105,7 @@ func (d *Downloader) downloadFromAPI(request *Request, buffer []byte, delay int6
 		time.Sleep(time.Duration(delay) * time.Second)
 	}
 
-	unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
+	resp := d.t.UnrestrictUntilOk(request.file.Link)
-		return realdebrid.UnrestrictLink(d.c.GetToken(), request.file.Link)
-	}
-	resp := realdebrid.RetryUntilOk(unrestrictFn)
 	if resp == nil {
 		return fmt.Errorf("cannot unrestrict file %s %s", request.file.Path, request.file.Link)
 	}

pkg/chunk/manager.go

@@ -58,6 +58,7 @@ func NewManager(
 	checkThreads,
 	loadThreads,
 	maxChunks int,
+	t *torrent.TorrentManager,
 	c config.ConfigInterface) (*Manager, error) {
 
 	pageSize := int64(os.Getpagesize())
@@ -81,7 +82,7 @@ func NewManager(
 		return nil, err
 	}
 
-	downloader, err := NewDownloader(loadThreads, storage, chunkSize, c)
+	downloader, err := NewDownloader(loadThreads, storage, chunkSize, t, c)
 	if nil != err {
 		return nil, err
 	}

pkg/dav/response.go

@@ -2,7 +2,7 @@ package dav
 
 func Directory(path string) Response {
 	return Response{
-		Href: customPathEscape(path),
+		Href: "/" + customPathEscape(path),
 		Propstat: PropStat{
 			Prop: Prop{
 				ResourceType: ResourceType{Value: "<d:collection/>"},
@@ -14,7 +14,7 @@ func Directory(path string) Response {
 
 func File(path string, fileSize int64, added string, link string) Response {
 	return Response{
-		Href: customPathEscape(path),
+		Href: "/" + customPathEscape(path),
 		Propstat: PropStat{
 			Prop: Prop{
 				ContentLength: fileSize,

pkg/http/client.go

@@ -0,0 +1,65 @@
+package http
+
+import (
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/debridmediamanager.com/zurg/internal/config"
+	"github.com/debridmediamanager.com/zurg/pkg/logutil"
+	"go.uber.org/zap"
+)
+
+type HTTPClient struct {
+	Client          *http.Client
+	MaxRetries      int
+	Backoff         func(attempt int) time.Duration
+	CheckRespStatus func(resp *http.Response, err error, log *zap.SugaredLogger) bool
+	BearerToken     string
+	log             *zap.SugaredLogger
+	config          config.ConfigInterface
+}
+
+func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
+	if r.config != nil && strings.Contains(req.Host, "download.real-debrid.") {
+		if host := r.config.GetRandomPreferredHost(); host != "" {
+			req.Host = r.config.GetRandomPreferredHost()
+		}
+	}
+	if r.BearerToken != "" {
+		req.Header.Set("Authorization", "Bearer "+r.BearerToken)
+	}
+	var resp *http.Response
+	var err error
+	for attempt := 0; attempt < r.MaxRetries; attempt++ {
+		resp, err = r.Client.Do(req)
+		if !r.CheckRespStatus(resp, err, r.log) {
+			return resp, err
+		}
+		time.Sleep(r.Backoff(attempt))
+	}
+	return resp, err
+}
+
+func NewHTTPClient(token string, maxRetries int, c config.ConfigInterface) *HTTPClient {
+	return &HTTPClient{
+		BearerToken: token,
+		Client:      &http.Client{},
+		MaxRetries:  maxRetries,
+		Backoff: func(attempt int) time.Duration {
+			return time.Duration(attempt) * time.Second
+		},
+		CheckRespStatus: func(resp *http.Response, err error, log *zap.SugaredLogger) bool {
+			if err != nil {
+				return true
+			}
+			if resp.StatusCode == 429 {
+				return true
+			}
+			// no need to retry
+			return false
+		},
+		log:    logutil.NewLogger().Named("client"),
+		config: c,
+	}
+}

pkg/realdebrid/api.go

@@ -3,94 +3,70 @@ package realdebrid
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
+
+	"github.com/debridmediamanager.com/zurg/internal/config"
+	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
+	"go.uber.org/zap"
 )
 
-func UnrestrictCheck(accessToken, link string) (*UnrestrictResponse, error) {
+type RealDebrid struct {
+	log    *zap.SugaredLogger
+	client *zurghttp.HTTPClient
+}
+
+func NewRealDebrid(accessToken string, config config.ConfigInterface, log *zap.SugaredLogger) *RealDebrid {
+	maxRetries := 10
+	client := zurghttp.NewHTTPClient(accessToken, maxRetries, nil)
+	return &RealDebrid{
+		log:    log,
+		client: client,
+	}
+}
+
+func (rd *RealDebrid) UnrestrictCheck(link string) (*UnrestrictResponse, error) {
 	data := url.Values{}
 	data.Set("link", link)
 
 	req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/check", bytes.NewBufferString(data.Encode()))
 	if err != nil {
+		rd.log.Errorf("Error when creating a unrestrict check request: %v", err)
 		return nil, err
 	}
-
-	req.Header.Set("Authorization", "Bearer "+accessToken)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	client := &http.Client{}
+	resp, err := rd.client.Do(req)
-	resp, err := client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the unrestrict check request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
+		rd.log.Errorf("Error when reading the body of unrestrict check response: %v", err)
 		return nil, err
 	}
 
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
 	var response UnrestrictResponse
 	err = json.Unmarshal(body, &response)
 	if err != nil {
+		rd.log.Errorf("Error when decoding unrestrict check JSON: %v", err)
 		return nil, err
 	}
 
-	return &response, nil
+	rd.log.Info("Link %s is streamable? %v", response.Streamable)
-}
-
-func UnrestrictLink(accessToken, link string) (*UnrestrictResponse, error) {
-	data := url.Values{}
-	data.Set("link", link)
-
-	req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", bytes.NewBufferString(data.Encode()))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
-	var response UnrestrictResponse
-	err = json.Unmarshal(body, &response)
-	if err != nil {
-		return nil, err
-	}
-
-	if !canFetchFirstByte(response.Download) {
-		return nil, fmt.Errorf("can't fetch first byte")
-	}
 	return &response, nil
 }
 
 // GetTorrents returns all torrents, paginated
 // if customLimit is 0, the default limit of 2500 is used
-func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
+func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) {
 	baseURL := "https://api.real-debrid.com/rest/1.0/torrents"
 	var allTorrents []Torrent
 	page := 1
@@ -110,26 +86,24 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
 
 		req, err := http.NewRequest("GET", reqURL, nil)
 		if err != nil {
+			rd.log.Errorf("Error when creating a get torrents request: %v", err)
 			return nil, 0, err
 		}
 
-		req.Header.Set("Authorization", "Bearer "+accessToken)
+		resp, err := rd.client.Do(req)
-
-		client := &http.Client{}
-		resp, err := client.Do(req)
 		if err != nil {
+			rd.log.Errorf("Error when executing the get torrents request: %v", err)
 			return nil, 0, err
 		}
 		defer resp.Body.Close()
 
-		if resp.StatusCode != http.StatusOK {
+		// if status code is not 2xx, return erro
-			return nil, 0, fmt.Errorf("HTTP error: %s", resp.Status)
-		}
 
 		var torrents []Torrent
 		decoder := json.NewDecoder(resp.Body)
 		err = decoder.Decode(&torrents)
 		if err != nil {
+			rd.log.Errorf("Error when decoding get torrents JSON: %v", err)
 			return nil, 0, err
 		}
 
@@ -147,39 +121,35 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
 
 		page++
 	}
-
 	return allTorrents, totalCount, nil
 }
 
-func GetTorrentInfo(accessToken, id string) (*Torrent, error) {
+func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
 	url := "https://api.real-debrid.com/rest/1.0/torrents/info/" + id
 
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
+		rd.log.Errorf("Error when creating a get info request: %v", err)
 		return nil, err
 	}
 
-	req.Header.Set("Authorization", "Bearer "+accessToken)
+	resp, err := rd.client.Do(req)
-
-	client := &http.Client{}
-	resp, err := client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the get info request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
+		rd.log.Errorf("Error when reading the body of get info response: %v", err)
 		return nil, err
 	}
 
-	if resp.StatusCode != http.StatusOK {
+	var response TorrentInfo
-		return nil, fmt.Errorf("HTTP error: %s", resp.Status)
-	}
-
-	var response Torrent
 	err = json.Unmarshal(body, &response)
 	if err != nil {
+		rd.log.Errorf("Error when : %v", err)
 		return nil, err
 	}
 
@@ -187,86 +157,54 @@ func GetTorrentInfo(accessToken, id string) (*Torrent, error) {
 }
 
 // SelectTorrentFiles selects files of a torrent to start it.
-func SelectTorrentFiles(accessToken string, id string, files string) error {
+func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error {
-	// Prepare request data
 	data := url.Values{}
 	data.Set("files", files)
 
-	// Construct request URL
 	reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/selectFiles/%s", id)
 	req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode()))
 	if err != nil {
+		rd.log.Errorf("Error when creating a select files request: %v", err)
 		return err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
-	// Send the request
+	resp, err := rd.client.Do(req)
-	client := &http.Client{}
-	resp, err := client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the select files request: %v", err)
 		return err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
+	rd.log.Debugf("Started the download for torrent id=%s", len(strings.Split(files, ",")), id)
-	switch resp.StatusCode {
+	return nil
-	case http.StatusOK, http.StatusNoContent:
-		return nil // Success
-	case http.StatusAccepted:
-		return errors.New("action already done")
-	case http.StatusBadRequest:
-		return errors.New("bad request")
-	case http.StatusUnauthorized:
-		return errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return errors.New("permission denied (account locked or not premium)")
-	case http.StatusNotFound:
-		return errors.New("wrong parameter (invalid file id(s)) or unknown resource (invalid id)")
-	default:
-		return fmt.Errorf("unexpected HTTP error: %s", resp.Status)
-	}
 }
 
 // DeleteTorrent deletes a torrent from the torrents list.
-func DeleteTorrent(accessToken string, id string) error {
+func (rd *RealDebrid) DeleteTorrent(id string) error {
 	// Construct request URL
 	reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/delete/%s", id)
 	req, err := http.NewRequest("DELETE", reqURL, nil)
 	if err != nil {
+		rd.log.Errorf("Error when creating a delete torrent request: %v", err)
 		return err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-
 	// Send the request
-	client := &http.Client{}
+	resp, err := rd.client.Do(req)
-	resp, err := client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the delete torrent request: %v", err)
 		return err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
+	rd.log.Debugf("Deleted torrent with id=%s", id)
-	switch resp.StatusCode {
+	return nil
-	case http.StatusNoContent:
-		return nil // Success
-	case http.StatusUnauthorized:
-		return errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return errors.New("permission denied (account locked)")
-	case http.StatusNotFound:
-		return errors.New("unknown resource")
-	default:
-		return fmt.Errorf("unexpected HTTP error: %s", resp.Status)
-	}
 }
 
 // AddMagnetHash adds a magnet link to download.
-func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
+func (rd *RealDebrid) AddMagnetHash(magnet string) (*MagnetResponse, error) {
 	// Prepare request data
 	data := url.Values{}
 	data.Set("magnet", fmt.Sprintf("magnet:?xt=urn:btih:%s", magnet))
@@ -275,77 +213,94 @@ func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
 	reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet"
 	req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode()))
 	if err != nil {
+		rd.log.Errorf("Error when creating an add magnet request: %v", err)
 		return nil, err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
 	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
 
 	// Send the request
-	client := &http.Client{}
+	resp, err := rd.client.Do(req)
-	resp, err := client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the add magnet request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
-	switch resp.StatusCode {
-	case http.StatusCreated:
 	var response MagnetResponse
-		err := json.NewDecoder(resp.Body).Decode(&response)
+	err = json.NewDecoder(resp.Body).Decode(&response)
 	if err != nil {
+		rd.log.Errorf("Error when decoding add magnet JSON: %v", err)
 		return nil, err
 	}
+
+	rd.log.Debugf("Added magnet %s with id=%s", magnet, response.ID)
 	return &response, nil
-	case http.StatusBadRequest:
-		return nil, errors.New("bad request")
-	case http.StatusUnauthorized:
-		return nil, errors.New("bad token (expired or invalid)")
-	case http.StatusForbidden:
-		return nil, errors.New("permission denied (account locked or not premium)")
-	case http.StatusServiceUnavailable:
-		return nil, errors.New("service unavailable")
-	default:
-		return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status)
-	}
 }
 
 // GetActiveTorrentCount gets the number of currently active torrents and the current maximum limit.
-func GetActiveTorrentCount(accessToken string) (*ActiveTorrentCountResponse, error) {
+func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, error) {
 	// Construct request URL
 	reqURL := "https://api.real-debrid.com/rest/1.0/torrents/activeCount"
 	req, err := http.NewRequest("GET", reqURL, nil)
 	if err != nil {
+		rd.log.Errorf("Error when creating a active torrents request: %v", err)
 		return nil, err
 	}
 
-	// Set request headers
-	req.Header.Set("Authorization", "Bearer "+accessToken)
-
 	// Send the request
-	client := &http.Client{}
+	resp, err := rd.client.Do(req)
-	resp, err := client.Do(req)
 	if err != nil {
+		rd.log.Errorf("Error when executing the active torrents request: %v", err)
 		return nil, err
 	}
 	defer resp.Body.Close()
 
-	// Handle response status codes
-	switch resp.StatusCode {
-	case http.StatusOK:
 	var response ActiveTorrentCountResponse
-		err := json.NewDecoder(resp.Body).Decode(&response)
+	err = json.NewDecoder(resp.Body).Decode(&response)
 	if err != nil {
+		rd.log.Errorf("Error when decoding active torrents JSON: %v", err)
 		return nil, err
 	}
 	return &response, nil
-	case http.StatusUnauthorized:
+}
-		return nil, errors.New("bad token (expired or invalid)")
+
-	case http.StatusForbidden:
+func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
-		return nil, errors.New("permission denied (account locked)")
+	data := url.Values{}
-	default:
+	data.Set("link", link)
-		return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+
-	}
+	req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", bytes.NewBufferString(data.Encode()))
+	if err != nil {
+		rd.log.Errorf("Error when creating a unrestrict link request: %v", err)
+		return nil, err
+	}
+
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := rd.client.Do(req)
+	if err != nil {
+		rd.log.Errorf("Error when executing the unrestrict link request: %v", err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		rd.log.Errorf("Error when reading the body of unrestrict link response: %v", err)
+		return nil, err
+	}
+
+	var response UnrestrictResponse
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		rd.log.Errorf("Error when decoding unrestrict link JSON: %v", err)
+		return nil, err
+	}
+
+	if !canFetchFirstByte(response.Download) {
+		return nil, fmt.Errorf("can't fetch first byte")
+	}
+
+	// rd.log.Debugf("Unrestricted link %s into %s", link, response.Download)
+	return &response, nil
 }

pkg/realdebrid/network.go

@@ -0,0 +1,106 @@
+package realdebrid
+
+import (
+	"fmt"
+	"os/exec"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+type IPInfo struct {
+	Address string
+	Hops    int
+	Latency time.Duration
+}
+
+func traceroute(ip string) (int, time.Duration, error) {
+	cmd := exec.Command("traceroute", "-n", "-q", "1", "-w", "1", ip)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	output := string(out)
+	lines := strings.Split(output, "\n")
+
+	hopCount := len(lines) - 1
+
+	var latency time.Duration
+	if hopCount > 0 {
+		lastLine := lines[hopCount-1]
+		fmt.Println(lastLine)
+		parts := strings.Fields(lastLine)
+		if len(parts) >= 3 {
+			latencyValue, parseErr := strconv.ParseFloat(parts[2], 64)
+			if parseErr == nil {
+				latency = time.Duration(latencyValue * float64(time.Millisecond))
+			}
+		}
+	}
+
+	return hopCount, latency, nil
+}
+
+func RunTest() {
+	fmt.Println("Running network test...")
+
+	ips := []string{"20.download.real-debrid.cloud", "20.download.real-debrid.com", "21.download.real-debrid.cloud", "21.download.real-debrid.com", "22.download.real-debrid.cloud", "22.download.real-debrid.com", "23.download.real-debrid.cloud", "23.download.real-debrid.com", "30.download.real-debrid.cloud", "30.download.real-debrid.com", "31.download.real-debrid.cloud", "31.download.real-debrid.com", "32.download.real-debrid.cloud", "32.download.real-debrid.com", "34.download.real-debrid.cloud", "34.download.real-debrid.com", "40.download.real-debrid.cloud", "40.download.real-debrid.com", "41.download.real-debrid.cloud", "41.download.real-debrid.com", "42.download.real-debrid.cloud", "42.download.real-debrid.com", "43.download.real-debrid.cloud", "43.download.real-debrid.com", "44.download.real-debrid.cloud", "44.download.real-debrid.com", "45.download.real-debrid.cloud", "45.download.real-debrid.com", "50.download.real-debrid.cloud", "50.download.real-debrid.com", "51.download.real-debrid.cloud", "51.download.real-debrid.com", "52.download.real-debrid.cloud", "52.download.real-debrid.com", "53.download.real-debrid.cloud", "53.download.real-debrid.com", "54.download.real-debrid.cloud", "54.download.real-debrid.com", "55.download.real-debrid.cloud", "55.download.real-debrid.com", "56.download.real-debrid.cloud", "56.download.real-debrid.com", "57.download.real-debrid.cloud", "57.download.real-debrid.com", "58.download.real-debrid.cloud", "58.download.real-debrid.com", "59.download.real-debrid.cloud", "59.download.real-debrid.com", "60.download.real-debrid.cloud", "60.download.real-debrid.com", "61.download.real-debrid.cloud", "61.download.real-debrid.com", "62.download.real-debrid.cloud", "62.download.real-debrid.com", "63.download.real-debrid.cloud", "63.download.real-debrid.com", "64.download.real-debrid.cloud", "64.download.real-debrid.com", "65.download.real-debrid.cloud", "65.download.real-debrid.com", "66.download.real-debrid.cloud", "66.download.real-debrid.com", "67.download.real-debrid.cloud", "67.download.real-debrid.com", "68.download.real-debrid.cloud", "68.download.real-debrid.com", "69.download.real-debrid.cloud", "69.download.real-debrid.com", "hkg1.download.real-debrid.com", "lax1.download.real-debrid.com", "lon1.download.real-debrid.com", "mum1.download.real-debrid.com", "rbx.download.real-debrid.com", "sgp1.download.real-debrid.com", "tlv1.download.real-debrid.com", "tyo1.download.real-debrid.com"}
+
+	var wg sync.WaitGroup
+	infoChan := make(chan IPInfo, len(ips))
+	semaphore := make(chan struct{}, 10)
+
+	for _, ip := range ips {
+		wg.Add(1)
+		semaphore <- struct{}{}
+		go func(ip string) {
+			defer wg.Done()
+			hops, latency, err := traceroute(ip)
+			if err != nil {
+				fmt.Println("Error performing traceroute:", err)
+			} else {
+				infoChan <- IPInfo{Address: ip, Hops: hops, Latency: latency}
+			}
+			<-semaphore
+		}(ip)
+	}
+
+	wg.Wait()
+	close(infoChan)
+
+	var ipInfos []IPInfo
+	for info := range infoChan {
+		ipInfos = append(ipInfos, info)
+	}
+
+	sort.Slice(ipInfos, func(i, j int) bool {
+		if ipInfos[i].Hops == ipInfos[j].Hops {
+			return ipInfos[i].Latency < ipInfos[j].Latency
+		}
+		return ipInfos[i].Hops < ipInfos[j].Hops
+	})
+	var lowestLatency time.Duration
+	if len(ipInfos) > 0 {
+		lowestLatency = ipInfos[0].Latency
+		for _, info := range ipInfos {
+			if info.Latency < lowestLatency {
+				lowestLatency = info.Latency
+			}
+		}
+	}
+
+	latencyThreshold := lowestLatency + lowestLatency/3
+
+	var okIPs []IPInfo
+	for _, info := range ipInfos {
+		if info.Latency <= latencyThreshold {
+			okIPs = append(okIPs, info)
+		}
+	}
+	for _, info := range okIPs {
+		fmt.Printf("Host: %s, Hops: %d, Latency: %v\n", info.Address, info.Hops, info.Latency)
+	}
+}

pkg/realdebrid/types.go

@@ -22,29 +22,43 @@ type UnrestrictResponse struct {
 type Torrent struct {
 	ID       string   `json:"id"`
 	Name     string   `json:"filename"`
-	OriginalName string   `json:"original_filename"`
 	Hash     string   `json:"hash"`
 	Progress int      `json:"-"`
 	Added    string   `json:"added"`
 	Bytes    int64    `json:"bytes"`
 	Links    []string `json:"links"`
-	Files        []File   `json:"files,omitempty"`
 }
 
-func (t *Torrent) UnmarshalJSON(data []byte) error {
+type TorrentInfo struct {
-	type Alias Torrent
+	ID            string   `json:"id"`
+	Name          string   `json:"filename"`
+	Hash          string   `json:"hash"`
+	Progress      int      `json:"-"`
+	Status        string   `json:"status"`
+	Added         string   `json:"added"`
+	Bytes         int64    `json:"bytes"`
+	Links         []string `json:"links"`
+	OriginalName  string   `json:"original_filename"` // from info
+	OriginalBytes int64    `json:"original_bytes"`    // from info
+	Files         []File   `json:"files"`             // from info
+	ForRepair     bool     `json:"-"`
+	Version       string   `json:"-"`
+}
+
+func (i *TorrentInfo) UnmarshalJSON(data []byte) error {
+	type Alias TorrentInfo
 	aux := &struct {
 		Progress float64 `json:"progress"`
 		*Alias
 	}{
-		Alias: (*Alias)(t),
+		Alias: (*Alias)(i),
 	}
 
 	if err := json.Unmarshal(data, &aux); err != nil {
 		return err
 	}
 
-	t.Progress = int(math.Round(aux.Progress))
+	i.Progress = int(math.Round(aux.Progress))
 	return nil
 }
 

pkg/realdebrid/unrestrict.go

@@ -7,7 +7,19 @@ import (
 	"time"
 )
 
-func RetryUntilOk[T any](fn func() (T, error)) T {
+func (rd *RealDebrid) UnrestrictUntilOk(link string) *UnrestrictResponse {
+	if link == "" {
+		return nil
+	}
+	unrestrictFn := func(link string) (*UnrestrictResponse, error) {
+		return rd.UnrestrictLink(link)
+	}
+	return retryUntilOk(func() (*UnrestrictResponse, error) {
+		return unrestrictFn(link)
+	})
+}
+
+func retryUntilOk[T any](fn func() (T, error)) T {
 	const initialDelay = 1 * time.Second
 	const maxDelay = 128 * time.Second
 	for i := 0; ; i++ {