diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 946ecf6..e792dfc 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -10,7 +10,7 @@ on:
env:
REGISTRY: ghcr.io
- IMAGE_NAME: ${{ github.repository }}
+ IMAGE_NAME: ${{ github.repository }}-testing
jobs:
build-and-push-image:
diff --git a/Dockerfile b/Dockerfile
index 2c8efde..d8c8cd9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -9,11 +9,11 @@ COPY . .
RUN CGO_ENABLED=0 GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags="-s -w" -o zurg cmd/zurg/main.go
# Obfuscation stage
-# FROM alpine:3 AS obfuscator
-# WORKDIR /app
-# COPY --from=builder /app/zurg .
-# RUN apk add --no-cache upx
-# RUN upx --brute zurg
+FROM alpine:3 AS obfuscator
+WORKDIR /app
+COPY --from=builder /app/zurg .
+RUN apk add --no-cache upx
+RUN upx --brute zurg
# Final stage
FROM alpine:3
diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go
index de6c243..e904121 100644
--- a/cmd/zurg/main.go
+++ b/cmd/zurg/main.go
@@ -16,12 +16,17 @@ import (
"github.com/debridmediamanager.com/zurg/internal/zfs"
"github.com/debridmediamanager.com/zurg/pkg/chunk"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
+ "github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"github.com/hashicorp/golang-lru/v2/expirable"
)
func main() {
- rlog := logutil.NewLogger()
- log := rlog.Named("zurg")
+ if len(os.Args) > 1 && os.Args[1] == "networktest" {
+ realdebrid.RunTest()
+ return
+ }
+
+ log := logutil.NewLogger().Named("zurg")
config, configErr := config.LoadZurgConfig("./config.yml")
if configErr != nil {
@@ -30,7 +35,9 @@ func main() {
cache := expirable.NewLRU[string, string](1e4, nil, time.Hour)
- torrentMgr := torrent.NewTorrentManager(config, cache)
+ rd := realdebrid.NewRealDebrid(config.GetToken(), config, logutil.NewLogger().Named("realdebrid"))
+
+ torrentMgr := torrent.NewTorrentManager(config, rd)
mux := http.NewServeMux()
net.Router(mux, config, torrentMgr, cache)
@@ -53,7 +60,8 @@ func main() {
1, // 1 chunk - load ahead (1MB total)
max(runtime.NumCPU()/2, 1), // check threads
max(runtime.NumCPU()/2, 1), // load threads
- runtime.NumCPU()*2, // max chunks
+ runtime.NumCPU()*2,
+ torrentMgr, // max chunks
config)
if nil != err {
log.Panicf("Failed to initialize chunk manager: %v", err)
diff --git a/go.mod b/go.mod
index eefa01b..60d7ecb 100644
--- a/go.mod
+++ b/go.mod
@@ -4,9 +4,10 @@ go 1.21.3
require (
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5
+ github.com/elliotchance/orderedmap/v2 v2.2.0
github.com/hashicorp/golang-lru/v2 v2.0.7
go.uber.org/zap v1.26.0
- golang.org/x/sys v0.4.0
+ golang.org/x/sys v0.14.0
gopkg.in/yaml.v3 v3.0.1
)
diff --git a/go.sum b/go.sum
index 252ca36..b762d23 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,8 @@ bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 h1:A0NsYy4lDBZAC6QiYeJ4N+XuHIK
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5/go.mod h1:gG3RZAMXCa/OTes6rr9EwusmR1OH1tDDy+cg9c5YliY=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
+github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -16,8 +18,8 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
-golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/hosts.txt b/hosts.txt
new file mode 100644
index 0000000..cd5a887
--- /dev/null
+++ b/hosts.txt
@@ -0,0 +1,76 @@
+20.download.real-debrid.cloud
+21.download.real-debrid.cloud
+22.download.real-debrid.cloud
+23.download.real-debrid.cloud
+30.download.real-debrid.cloud
+31.download.real-debrid.cloud
+32.download.real-debrid.cloud
+34.download.real-debrid.cloud
+40.download.real-debrid.cloud
+41.download.real-debrid.cloud
+42.download.real-debrid.cloud
+43.download.real-debrid.cloud
+44.download.real-debrid.cloud
+45.download.real-debrid.cloud
+50.download.real-debrid.cloud
+51.download.real-debrid.cloud
+52.download.real-debrid.cloud
+53.download.real-debrid.cloud
+54.download.real-debrid.cloud
+55.download.real-debrid.cloud
+56.download.real-debrid.cloud
+57.download.real-debrid.cloud
+58.download.real-debrid.cloud
+59.download.real-debrid.cloud
+60.download.real-debrid.cloud
+61.download.real-debrid.cloud
+62.download.real-debrid.cloud
+63.download.real-debrid.cloud
+64.download.real-debrid.cloud
+65.download.real-debrid.cloud
+66.download.real-debrid.cloud
+67.download.real-debrid.cloud
+68.download.real-debrid.cloud
+69.download.real-debrid.cloud
+20.download.real-debrid.com
+21.download.real-debrid.com
+22.download.real-debrid.com
+23.download.real-debrid.com
+30.download.real-debrid.com
+31.download.real-debrid.com
+32.download.real-debrid.com
+34.download.real-debrid.com
+40.download.real-debrid.com
+41.download.real-debrid.com
+42.download.real-debrid.com
+43.download.real-debrid.com
+44.download.real-debrid.com
+45.download.real-debrid.com
+50.download.real-debrid.com
+51.download.real-debrid.com
+52.download.real-debrid.com
+53.download.real-debrid.com
+54.download.real-debrid.com
+55.download.real-debrid.com
+56.download.real-debrid.com
+57.download.real-debrid.com
+58.download.real-debrid.com
+59.download.real-debrid.com
+60.download.real-debrid.com
+61.download.real-debrid.com
+62.download.real-debrid.com
+63.download.real-debrid.com
+64.download.real-debrid.com
+65.download.real-debrid.com
+66.download.real-debrid.com
+67.download.real-debrid.com
+68.download.real-debrid.com
+69.download.real-debrid.com
+hkg1.download.real-debrid.com
+lax1.download.real-debrid.com
+lon1.download.real-debrid.com
+mum1.download.real-debrid.com
+rbx.download.real-debrid.com
+sgp1.download.real-debrid.com
+tlv1.download.real-debrid.com
+tyo1.download.real-debrid.com
diff --git a/internal/config/load.go b/internal/config/load.go
index 10139bd..b1bb3ba 100644
--- a/internal/config/load.go
+++ b/internal/config/load.go
@@ -23,11 +23,11 @@ type ConfigInterface interface {
GetNetworkBufferSize() int
GetMountPoint() string
EnableRetainFolderNameExtension() bool
+ GetRandomPreferredHost() string
}
func LoadZurgConfig(filename string) (ConfigInterface, error) {
- rlog := logutil.NewLogger()
- log := rlog.Named("config")
+ log := logutil.NewLogger().Named("config")
log.Debug("Loading config file ", filename)
content, err := os.ReadFile(filename)
diff --git a/internal/config/types.go b/internal/config/types.go
index ddca6a5..109554c 100644
--- a/internal/config/types.go
+++ b/internal/config/types.go
@@ -1,18 +1,21 @@
package config
+import "math/rand"
+
type ZurgConfig struct {
- Version string `yaml:"zurg"`
- Token string `yaml:"token"`
- Host string `yaml:"host"`
- Port string `yaml:"port"`
- NumOfWorkers int `yaml:"concurrent_workers"`
- RefreshEverySeconds int `yaml:"check_for_changes_every_secs"`
- CacheTimeHours int `yaml:"info_cache_time_hours"`
- CanRepair bool `yaml:"enable_repair"`
- OnLibraryUpdate string `yaml:"on_library_update"`
- NetworkBufferSize int `yaml:"network_buffer_size"`
- MountPoint string `yaml:"mount_point"`
- RetainFolderNameExtension bool `yaml:"retain_folder_name_extension"`
+ Version string `yaml:"zurg"`
+ Token string `yaml:"token"`
+ Host string `yaml:"host"`
+ Port string `yaml:"port"`
+ NumOfWorkers int `yaml:"concurrent_workers"`
+ RefreshEverySeconds int `yaml:"check_for_changes_every_secs"`
+ CacheTimeHours int `yaml:"info_cache_time_hours"`
+ CanRepair bool `yaml:"enable_repair"`
+ OnLibraryUpdate string `yaml:"on_library_update"`
+ NetworkBufferSize int `yaml:"network_buffer_size"`
+ MountPoint string `yaml:"mount_point"`
+ RetainFolderNameExtension bool `yaml:"retain_folder_name_extension"`
+ PreferredHosts []string `yaml:"preferred_hosts"`
}
func (z *ZurgConfig) GetToken() string {
@@ -73,3 +76,11 @@ func (z *ZurgConfig) GetMountPoint() string {
func (z *ZurgConfig) EnableRetainFolderNameExtension() bool {
return z.RetainFolderNameExtension
}
+
+func (z *ZurgConfig) GetRandomPreferredHost() string {
+ if len(z.PreferredHosts) == 0 {
+ return ""
+ }
+ randomIndex := rand.Intn(len(z.PreferredHosts))
+ return z.PreferredHosts[randomIndex]
+}
diff --git a/internal/config/v1.go b/internal/config/v1.go
index a199854..dd2bf9b 100644
--- a/internal/config/v1.go
+++ b/internal/config/v1.go
@@ -5,7 +5,6 @@ import (
"sort"
"strings"
- "github.com/debridmediamanager.com/zurg/pkg/logutil"
"gopkg.in/yaml.v3"
)
@@ -33,9 +32,6 @@ func (z *ZurgConfigV1) GetDirectories() []string {
}
func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
- rlog := logutil.NewLogger()
- log := rlog.Named("config")
-
var groupMap = make(map[string][]string)
var groupOrderMap = make(map[string]int) // To store GroupOrder for each directory
@@ -43,7 +39,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
for directory, val := range z.Directories {
groupMap[val.Group] = append(groupMap[val.Group], directory)
groupOrderMap[directory] = val.GroupOrder
- log.Debugf("Added directory to group: %s, group: %s, order: %d", directory, val.Group, val.GroupOrder)
}
// Sort the slice based on GroupOrder and then directory name for deterministic order
@@ -55,7 +50,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
return groupOrderMap[dirs[i]] < groupOrderMap[dirs[j]]
})
groupMap[group] = dirs
- log.Debugf("Sorted directories within a group: %s %v", group, dirs)
}
// Return a deep copy of the map
diff --git a/internal/dav/listing.go b/internal/dav/listing.go
index d2653e6..b9bb1ce 100644
--- a/internal/dav/listing.go
+++ b/internal/dav/listing.go
@@ -5,29 +5,21 @@ import (
"fmt"
"net/http"
"path"
+ "path/filepath"
"strings"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/pkg/dav"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
- "github.com/hashicorp/golang-lru/v2/expirable"
)
-func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
- rlog := logutil.NewLogger()
- log := rlog.Named("dav")
+func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
+ log := logutil.NewLogger().Named("dav")
requestPath := path.Clean(r.URL.Path)
requestPath = strings.Trim(requestPath, "/")
- if data, exists := cache.Get(requestPath); exists {
- w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
- w.WriteHeader(http.StatusMultiStatus)
- fmt.Fprint(w, data)
- return
- }
-
var output []byte
var err error
@@ -41,12 +33,16 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
case len(filteredSegments) == 2:
output, err = handleSingleTorrent(requestPath, w, r, t)
default:
- log.Errorf("Request %s %s not found", r.Method, requestPath)
+ log.Warnf("Request %s %s not found", r.Method, requestPath)
http.Error(w, "Not Found", http.StatusNotFound)
return
}
if err != nil {
+ if strings.Contains(err.Error(), "cannot find") {
+ http.Error(w, "Not Found", http.StatusNotFound)
+ return
+ }
log.Errorf("Error processing request: %v", err)
http.Error(w, "Server error", http.StatusInternalServerError)
return
@@ -54,8 +50,6 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
if output != nil {
respBody := fmt.Sprintf("\n%s\n", output)
- cache.Add(requestPath, respBody)
-
w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
w.WriteHeader(http.StatusMultiStatus)
fmt.Fprint(w, respBody)
@@ -64,9 +58,9 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface) ([]byte, error) {
var responses []dav.Response
- responses = append(responses, dav.Directory("/"))
+ responses = append(responses, dav.Directory(""))
for _, directory := range c.GetDirectories() {
- responses = append(responses, dav.Directory("/"+directory))
+ responses = append(responses, dav.Directory(directory))
}
rootResponse := dav.MultiStatus{
XMLNS: "DAV:",
@@ -80,10 +74,28 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
for _, directory := range c.GetDirectories() {
if basePath == directory {
- torrents := t.GetByDirectory(basePath)
- resp, err := createMultiTorrentResponse("/"+basePath, torrents)
- if err != nil {
- return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+ var responses []dav.Response
+
+ responses = append(responses, dav.Directory(basePath))
+
+ for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+ accessKey := el.Key
+ torrent := el.Value
+ if torrent.InProgress {
+ continue
+ }
+ for _, dir := range torrent.Directories {
+ if dir == basePath {
+ path := filepath.Join(basePath, accessKey)
+ responses = append(responses, dav.Directory(path))
+ break
+ }
+ }
+ }
+
+ resp := &dav.MultiStatus{
+ XMLNS: "DAV:",
+ Response: responses,
}
return xml.Marshal(resp)
}
@@ -93,17 +105,37 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
}
func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) {
- directory := path.Dir(requestPath)
- torrentName := path.Base(requestPath)
-
- sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
- if len(sameNameTorrents) == 0 {
- return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
+ accessKey := path.Base(requestPath)
+ torrent, exists := t.TorrentMap.Get(accessKey)
+ if !exists {
+ return nil, fmt.Errorf("cannot find torrent %s", accessKey)
}
- resp, err := createSingleTorrentResponse("/"+directory, sameNameTorrents)
- if err != nil {
- return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
+ var responses []dav.Response
+
+ // initial response is the directory itself
+ responses = append(responses, dav.Directory(requestPath))
+
+ for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+ file := el.Value
+ if file.Link == "" {
+ // will be caught by torrent manager's repairAll
+ // just skip it for now
+ continue
+ }
+ filename := filepath.Base(file.Path)
+ filePath := filepath.Join(requestPath, filename)
+ responses = append(responses, dav.File(
+ filePath,
+ file.Bytes,
+ convertRFC3339toRFC1123(torrent.LatestAdded),
+ file.Link,
+ ))
+ }
+
+ resp := &dav.MultiStatus{
+ XMLNS: "DAV:",
+ Response: responses,
}
return xml.Marshal(resp)
}
diff --git a/internal/dav/response.go b/internal/dav/response.go
deleted file mode 100644
index 6098b6f..0000000
--- a/internal/dav/response.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package dav
-
-import (
- "path/filepath"
-
- "github.com/debridmediamanager.com/zurg/internal/torrent"
- "github.com/debridmediamanager.com/zurg/pkg/dav"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*dav.MultiStatus, error) {
- var responses []dav.Response
- responses = append(responses, dav.Directory(basePath))
-
- seen := make(map[string]bool)
-
- for _, item := range torrents {
- if item.Progress != 100 {
- continue
- }
- if _, exists := seen[item.Name]; exists {
- continue
- }
- seen[item.Name] = true
-
- path := filepath.Join(basePath, item.Name)
- responses = append(responses, dav.Directory(path))
- }
-
- return &dav.MultiStatus{
- XMLNS: "DAV:",
- Response: responses,
- }, nil
-}
-
-// createTorrentResponse creates a WebDAV response for a single torrent
-// but it also handles the case where there are many torrents with the same name
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (*dav.MultiStatus, error) {
- var responses []dav.Response
-
- // initial response is the directory itself
- currentPath := filepath.Join(basePath, torrents[0].Name)
- responses = append(responses, dav.Directory(currentPath))
-
- finalName := make(map[string]bool)
-
- var torrentResponses []dav.Response
-
- for _, torrent := range torrents {
- for _, file := range torrent.SelectedFiles {
- if file.Link == "" {
- // TODO: fix the file?
- // log.Println("File has no link, skipping (repairing links take time)", file.Path)
- continue
- }
-
- filename := filepath.Base(file.Path)
- if finalName[filename] {
- continue
- }
- finalName[filename] = true
-
- filePath := filepath.Join(currentPath, filename)
- torrentResponses = append(torrentResponses, dav.File(
- filePath,
- file.Bytes,
- convertRFC3339toRFC1123(torrent.Added),
- file.Link,
- ))
- }
- }
-
- responses = append(responses, torrentResponses...)
-
- return &dav.MultiStatus{
- XMLNS: "DAV:",
- Response: responses,
- }, nil
-}
diff --git a/internal/http/listing.go b/internal/http/listing.go
index f641062..ddf0db7 100644
--- a/internal/http/listing.go
+++ b/internal/http/listing.go
@@ -5,27 +5,19 @@ import (
"net/http"
"net/url"
"path"
+ "path/filepath"
"strings"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
- "github.com/hashicorp/golang-lru/v2/expirable"
)
-func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
- rlog := logutil.NewLogger()
- log := rlog.Named("http")
+func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) {
+ log := logutil.NewLogger().Named("http")
requestPath := path.Clean(r.URL.Path)
- if data, exists := cache.Get(requestPath); exists {
- w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
- w.WriteHeader(http.StatusOK)
- fmt.Fprint(w, data)
- return
- }
-
var output *string
var err error
@@ -38,20 +30,21 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
case len(filteredSegments) == 3:
output, err = handleSingleTorrent(requestPath, w, r, t)
default:
- log.Errorf("Request %s %s not found", r.Method, requestPath)
+ log.Warnf("Request %s %s not found", r.Method, requestPath)
http.Error(w, "Not Found", http.StatusNotFound)
return
}
-
if err != nil {
+ if strings.Contains(err.Error(), "cannot find") {
+ http.Error(w, "Not Found", http.StatusNotFound)
+ return
+ }
log.Errorf("Error processing request: %v", err)
http.Error(w, "Server error", http.StatusInternalServerError)
return
}
if output != nil {
- cache.Add(requestPath, *output)
-
w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, *output)
@@ -74,12 +67,21 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
for _, directory := range c.GetDirectories() {
if basePath == directory {
- torrents := t.GetByDirectory(basePath)
- resp, err := createMultiTorrentResponse(requestPath, torrents)
- if err != nil {
- return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err)
+ htmlDoc := "
"
+ for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+ accessKey := el.Key
+ torrent := el.Value
+ if torrent.InProgress {
+ continue
+ }
+ for _, dir := range torrent.Directories {
+ if dir == basePath {
+ htmlDoc += fmt.Sprintf("- %s
", filepath.Join(requestPath, url.PathEscape(accessKey)), accessKey)
+ break
+ }
+ }
}
- return &resp, nil
+ return &htmlDoc, nil
}
}
@@ -87,18 +89,23 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
}
func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) {
- fullDir := path.Dir(requestPath)
- directory := path.Base(fullDir)
- torrentName := path.Base(requestPath)
-
- sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName)
- if len(sameNameTorrents) == 0 {
- return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath)
+ accessKey := path.Base(requestPath)
+ torrent, _ := t.TorrentMap.Get(accessKey)
+ if torrent == nil {
+ return nil, fmt.Errorf("cannot find torrent %s", accessKey)
}
- resp, err := createSingleTorrentResponse(requestPath, sameNameTorrents)
- if err != nil {
- return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err)
+ htmlDoc := ""
+ for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+ file := el.Value
+ if file.Link == "" {
+ // will be caught by torrent manager's repairAll
+ // just skip it for now
+ continue
+ }
+ filename := filepath.Base(file.Path)
+ filePath := filepath.Join(requestPath, url.PathEscape(filename))
+ htmlDoc += fmt.Sprintf("- %s
", filePath, filename)
}
- return &resp, nil
+ return &htmlDoc, nil
}
diff --git a/internal/http/response.go b/internal/http/response.go
deleted file mode 100644
index f488257..0000000
--- a/internal/http/response.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package http
-
-import (
- "fmt"
- "net/url"
- "path/filepath"
-
- "github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// createMultiTorrentResponse creates a WebDAV response for a list of torrents
-func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
- htmlDoc := ""
-
- seen := make(map[string]bool)
-
- for _, item := range torrents {
- if item.Progress != 100 {
- continue
- }
- if _, exists := seen[item.Name]; exists {
- continue
- }
- seen[item.Name] = true
-
- path := filepath.Join(basePath, url.PathEscape(item.Name))
- htmlDoc += fmt.Sprintf("- %s
", path, item.Name)
- }
-
- return htmlDoc, nil
-}
-
-func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) {
- htmlDoc := ""
-
- finalName := make(map[string]bool)
-
- currentPath := filepath.Join(basePath)
-
- for _, torrent := range torrents {
- for _, file := range torrent.SelectedFiles {
- if file.Link == "" {
- // TODO: fix the file?
- // log.Println("File has no link, skipping", file.Path)
- continue
- }
-
- filename := filepath.Base(file.Path)
- if finalName[filename] {
- continue
- }
- finalName[filename] = true
-
- filePath := filepath.Join(currentPath, url.PathEscape(filename))
- htmlDoc += fmt.Sprintf("- %s
", filePath, filename)
- }
- }
-
- return htmlDoc, nil
-}
diff --git a/internal/net/router.go b/internal/net/router.go
index 1c5f4fc..c5d8caa 100644
--- a/internal/net/router.go
+++ b/internal/net/router.go
@@ -16,8 +16,7 @@ import (
// Router creates a WebDAV router
func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentManager, cache *expirable.LRU[string, string]) {
- rlog := logutil.NewLogger()
- log := rlog.Named("net")
+ log := logutil.NewLogger().Named("net")
mux.HandleFunc("/http/", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
@@ -26,7 +25,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 {
universal.HandleGetRequest(w, r, t, c, cache)
} else {
- intHttp.HandleDirectoryListing(w, r, t, c, cache)
+ intHttp.HandleDirectoryListing(w, r, t, c)
}
case http.MethodHead:
@@ -41,7 +40,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "PROPFIND":
- dav.HandlePropfindRequest(w, r, t, c, cache)
+ dav.HandlePropfindRequest(w, r, t, c)
case http.MethodGet:
universal.HandleGetRequest(w, r, t, c, cache)
diff --git a/internal/torrent/hooks.go b/internal/torrent/hooks.go
index 9015f24..d149f07 100644
--- a/internal/torrent/hooks.go
+++ b/internal/torrent/hooks.go
@@ -30,8 +30,7 @@ func (se *ScriptExecutor) Execute() (string, error) {
}
func OnLibraryUpdateHook(config config.ConfigInterface) {
- rlog := logutil.NewLogger()
- log := rlog.Named("hooks")
+ log := logutil.NewLogger().Named("hooks")
executor := &ScriptExecutor{
Script: config.GetOnLibraryUpdate(),
diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go
index 3b3b2aa..4faff65 100644
--- a/internal/torrent/manager.go
+++ b/internal/torrent/manager.go
@@ -5,6 +5,7 @@ import (
"fmt"
"math"
"os"
+ "path/filepath"
"strings"
"sync"
"time"
@@ -12,114 +13,119 @@ import (
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
- "github.com/hashicorp/golang-lru/v2/expirable"
+ "github.com/elliotchance/orderedmap/v2"
"go.uber.org/zap"
)
type TorrentManager struct {
- requiredVersion string
- torrents []Torrent
- inProgress []string
- checksum string
- config config.ConfigInterface
- cache *expirable.LRU[string, string]
- workerPool chan bool
- directoryMap map[string][]string
- processedTorrents map[string][]string
- mu *sync.Mutex
- log *zap.SugaredLogger
+ TorrentMap *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
+ repairMap *orderedmap.OrderedMap[string, time.Time] // accessKey -> time last repaired
+ requiredVersion string
+ rd *realdebrid.RealDebrid
+ checksum string
+ config config.ConfigInterface
+ workerPool chan bool
+ mu *sync.Mutex
+ log *zap.SugaredLogger
}
// NewTorrentManager creates a new torrent manager
// it will fetch all torrents and their info in the background
-// and store them in-memory; it is called only once at startup
-func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
+// and store them in-memory and cached in files
+func NewTorrentManager(config config.ConfigInterface, rd *realdebrid.RealDebrid) *TorrentManager {
t := &TorrentManager{
- requiredVersion: fmt.Sprintf("4.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
- config: config,
- cache: cache,
- workerPool: make(chan bool, config.GetNumOfWorkers()),
- directoryMap: make(map[string][]string),
- processedTorrents: make(map[string][]string),
- mu: &sync.Mutex{},
- log: logutil.NewLogger().Named("manager"),
+ TorrentMap: orderedmap.NewOrderedMap[string, *Torrent](),
+ repairMap: orderedmap.NewOrderedMap[string, time.Time](),
+ requiredVersion: "10.11.2023",
+ rd: rd,
+ config: config,
+ workerPool: make(chan bool, config.GetNumOfWorkers()),
+ mu: &sync.Mutex{},
+ log: logutil.NewLogger().Named("manager"),
}
- // Initialize torrents for the first time
t.mu.Lock()
- t.torrents = t.getFreshListFromAPI()
- t.checksum = t.getChecksum()
- t.mu.Unlock()
- // log.Println("First checksum", t.checksum)
+ newTorrents, _, err := t.rd.GetTorrents(0)
+ if err != nil {
+ t.log.Fatalf("Cannot get torrents: %v\n", err)
+ }
+ torrentsChan := make(chan *Torrent, len(newTorrents))
var wg sync.WaitGroup
-
- for i := range t.torrents {
+ for i := range newTorrents {
wg.Add(1)
go func(idx int) {
defer wg.Done()
t.workerPool <- true
- t.addMoreInfo(&t.torrents[idx])
+ torrentsChan <- t.getMoreInfo(newTorrents[idx])
<-t.workerPool
}(i)
}
+ t.log.Infof("Got %d torrents", len(newTorrents))
+ wg.Wait()
+ close(torrentsChan)
+ for newTorrent := range torrentsChan {
+ if newTorrent == nil {
+ continue
+ }
+ torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
+ if torrent != nil {
+ t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
+ } else {
+ t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
+ }
+ }
+ t.log.Infof("Compiled to %d unique movies and shows", t.TorrentMap.Len())
+ t.checksum = t.getChecksum()
+ t.mu.Unlock()
if t.config.EnableRepair() {
- go t.repairAll(&wg)
+ go t.repairAll()
}
-
- wg.Wait()
- t.mapToDirectories()
go t.startRefreshJob()
return t
}
-// GetByDirectory returns all torrents that have a file in the specified directory
-func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
- var torrents []Torrent
- for i := range t.torrents {
- for _, dir := range t.directoryMap[t.torrents[i].Name] {
- if dir == directory {
- torrents = append(torrents, t.torrents[i])
- }
+func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
+ merged := t1
+
+ // Merge SelectedFiles
+ // side note: iteration works!
+ for el := t2.SelectedFiles.Front(); el != nil; el = el.Next() {
+ if _, ok := merged.SelectedFiles.Get(el.Key); !ok {
+ merged.SelectedFiles.Set(el.Key, el.Value)
}
}
- return torrents
-}
-// HideTheFile marks a file as deleted
-func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) {
- file.Unavailable = true
- t.repair(torrent.ID, torrent.SelectedFiles, false)
-}
+ // Merge Instances
+ merged.Instances = append(t1.Instances, t2.Instances...)
-// FindAllTorrentsWithName finds all torrents in a given directory with a given name
-func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string) []Torrent {
- var matchingTorrents []Torrent
- torrents := t.GetByDirectory(directory)
- for i := range torrents {
- if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) {
- matchingTorrents = append(matchingTorrents, torrents[i])
+ // LatestAdded
+ if t1.LatestAdded < t2.LatestAdded {
+ merged.LatestAdded = t2.LatestAdded
+ }
+
+ // InProgress - if one of the instances is in progress, then the whole torrent is in progress
+ for _, instance := range merged.Instances {
+ if instance.Progress != 100 {
+ merged.InProgress = true
+ }
+ if instance.ForRepair {
+ merged.ForRepair = true
}
}
- return matchingTorrents
+
+ return merged
}
-// findAllDownloadedFilesFromHash finds all files that were with a given hash
-func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File {
- var files []File
- for _, torrent := range t.torrents {
- if torrent.Hash == hash {
- for _, file := range torrent.SelectedFiles {
- if file.Link != "" {
- files = append(files, file)
- }
- }
- }
- }
- return files
+// proxy
+func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictResponse {
+ t.workerPool <- true
+ ret := t.rd.UnrestrictUntilOk(link)
+ <-t.workerPool
+ return ret
}
type torrentsResponse struct {
@@ -127,14 +133,15 @@ type torrentsResponse struct {
totalCount int
}
+// generates a checksum based on the number of torrents, the first torrent id and the number of active torrents
func (t *TorrentManager) getChecksum() string {
- torrentsChan := make(chan torrentsResponse)
- countChan := make(chan int)
+ torrentsChan := make(chan torrentsResponse, 1)
+ countChan := make(chan int, 1)
errChan := make(chan error, 2) // accommodate errors from both goroutines
// GetTorrents request
go func() {
- torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1)
+ torrents, totalCount, err := t.rd.GetTorrents(1)
if err != nil {
errChan <- err
return
@@ -144,7 +151,7 @@ func (t *TorrentManager) getChecksum() string {
// GetActiveTorrentCount request
go func() {
- count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+ count, err := t.rd.GetActiveTorrentCount()
if err != nil {
errChan <- err
return
@@ -162,7 +169,7 @@ func (t *TorrentManager) getChecksum() string {
totalCount = torrentsResp.totalCount
case count = <-countChan:
case err := <-errChan:
- t.log.Errorf("Checksum API Error: %v\n", err)
+ t.log.Warnf("Checksum API Error: %v\n", err)
return ""
}
}
@@ -173,6 +180,7 @@ func (t *TorrentManager) getChecksum() string {
}
checksum := fmt.Sprintf("%d%s%d", totalCount, torrents[0].ID, count)
+ t.log.Debugf("Checksum: %s", checksum)
return checksum
}
@@ -186,165 +194,172 @@ func (t *TorrentManager) startRefreshJob() {
if checksum == t.checksum {
continue
}
- t.cache.Purge()
- newTorrents := t.getFreshListFromAPI()
+ t.mu.Lock()
+
+ newTorrents, _, err := t.rd.GetTorrents(0)
+ if err != nil {
+ t.log.Warnf("Cannot get torrents: %v\n", err)
+ continue
+ }
+ t.log.Infof("Detected changes! Refreshing %d torrents", len(newTorrents))
+
+ torrentsChan := make(chan *Torrent, len(newTorrents))
var wg sync.WaitGroup
-
for i := range newTorrents {
wg.Add(1)
go func(idx int) {
defer wg.Done()
t.workerPool <- true
- t.addMoreInfo(&newTorrents[idx])
+ torrentsChan <- t.getMoreInfo(newTorrents[idx])
<-t.workerPool
}(i)
}
- wg.Wait()
- // apply side effects
- t.mu.Lock()
- t.torrents = newTorrents
+ // side note: iteration works!
+ var toDelete []string
+ for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+ found := false
+ for _, newTorrent := range newTorrents {
+ if newTorrent.ID == el.Value.AccessKey {
+ found = true
+ break
+ }
+ }
+ if !found {
+ toDelete = append(toDelete, el.Key)
+ }
+ }
+ for _, accessKey := range toDelete {
+ t.TorrentMap.Delete(accessKey)
+ }
+
+ wg.Wait()
+ close(torrentsChan)
+ for newTorrent := range torrentsChan {
+ if newTorrent == nil {
+ continue
+ }
+ torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
+ if torrent != nil {
+ t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
+ } else {
+ t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
+ }
+ }
t.checksum = t.getChecksum()
t.mu.Unlock()
- // log.Println("Checksum changed", t.checksum)
if t.config.EnableRepair() {
- go t.repairAll(&wg)
+ go t.repairAll()
}
- go t.mapToDirectories()
go OnLibraryUpdateHook(t.config)
}
}
-// getFreshListFromAPI returns all torrents
-func (t *TorrentManager) getFreshListFromAPI() []Torrent {
- torrents, _, err := realdebrid.GetTorrents(t.config.GetToken(), 0)
- if err != nil {
- t.log.Errorf("Cannot get torrents: %v\n", err)
- return nil
- }
-
- // convert to own internal type without SelectedFiles yet
- // populate inProgress
- var torrentsV2 []Torrent
- t.inProgress = t.inProgress[:0] // reset
- for _, torrent := range torrents {
- torrent.Name = strings.TrimSuffix(torrent.Name, "/")
- torrentV2 := Torrent{
- Torrent: torrent,
- SelectedFiles: nil,
- ForRepair: false,
- lock: &sync.Mutex{},
- }
- torrentsV2 = append(torrentsV2, torrentV2)
-
- if torrent.Progress != 100 {
- t.inProgress = append(t.inProgress, torrent.Hash)
- }
- }
-
- t.log.Infof("Fetched %d torrents", len(torrentsV2))
- return torrentsV2
-}
-
-// addMoreInfo updates the selected files for a torrent
-func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
+// getMoreInfo gets original name, size and files for a torrent
+func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
+ var info *realdebrid.TorrentInfo
+ var err error
// file cache
- torrentFromFile := t.readFromFile(torrent.ID)
- if torrentFromFile != nil {
+ torrentFromFile := t.readFromFile(rdTorrent.ID)
+ if torrentFromFile != nil && len(torrentFromFile.ID) > 0 && len(torrentFromFile.Links) == len(rdTorrent.Links) {
// see if api data and file data still match
// then it means data is still usable
- if len(torrentFromFile.Links) == len(torrent.Links) {
- torrent.Name = t.getName(torrentFromFile)
- torrent.ForRepair = torrentFromFile.ForRepair
- torrent.SelectedFiles = torrentFromFile.SelectedFiles[:]
- return
+ info = torrentFromFile
+ }
+ if info == nil {
+ info, err = t.rd.GetTorrentInfo(rdTorrent.ID)
+ if err != nil {
+ t.log.Warnf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
+ return nil
}
}
- // no file data yet as it is still downloading
- if torrent.Progress != 100 {
- return
- }
-
- // t.log.Println("Getting info for", torrent.ID)
- info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrent.ID)
- if err != nil {
- t.log.Errorf("Cannot get info: %v\n", err)
- return
- }
-
// SelectedFiles is a subset of Files with only the selected ones
// it also has a Link field, which can be empty
// if it is empty, it means the file is no longer available
// Files+Links together are the same as SelectedFiles
- var selectedFiles []File
- var streamableFiles []File
+ selectedFiles := orderedmap.NewOrderedMap[string, *File]()
+ streamableCount := 0
// if some Links are empty, we need to repair it
forRepair := false
for _, file := range info.Files {
if isStreamable(file.Path) {
- streamableFiles = append(streamableFiles, File{
- File: file,
- Link: "",
- })
+ streamableCount++
}
if file.Selected == 0 {
continue
}
- selectedFiles = append(selectedFiles, File{
+ selectedFiles.Set(filepath.Base(file.Path), &File{
File: file,
- Link: "",
+ Link: "", // no link yet
})
}
- if len(selectedFiles) > len(info.Links) && info.Progress == 100 {
- t.log.Debugf("Some links has expired for %s %s: %d selected but only %d link(s)", info.ID, info.Name, len(selectedFiles), len(info.Links))
+ if selectedFiles.Len() > len(info.Links) && info.Progress == 100 {
// chaotic file means RD will not output the desired file selection
// e.g. even if we select just a single mkv, it will output a rar
var isChaotic bool
- selectedFiles, isChaotic = t.organizeChaos(info, selectedFiles)
+ selectedFiles, isChaotic = t.organizeChaos(info.Links, selectedFiles)
if isChaotic {
- t.log.Infof("Torrent %s %s is unfixable, it's always returning an unstreamable link, ignoring", info.ID, info.Name)
- t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+ // t.log.Warnf("Torrent id=%s %s is unfixable, it is always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
+ // t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+ return nil
} else {
- if len(streamableFiles) > 1 {
- t.log.Infof("Torrent %s %s marked for repair", info.ID, info.Name)
+ if streamableCount > 1 {
+ // case for repair 1: it's missing some links (or all links)
+ // if we download it as is, we might get the same file over and over again
+ // so we need to redownload it with other files selected
+ // that is why we check if there are other streamable files
+ t.log.Infof("Torrent id=%s %s marked for repair", info.ID, info.Name)
forRepair = true
} else {
- t.log.Infof("Torrent %s %s is unfixable, the lone streamable link has expired, ignoring", info.ID, info.Name)
- t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+ // t.log.Warnf("Torrent id=%s %s is unfixable, the lone streamable link has expired (it is no longer shown in your directories)", info.ID, info.Name)
+ // t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
+ return nil
}
}
- } else {
+ } else if selectedFiles.Len() == len(info.Links) {
// all links are still intact! good!
- for i, link := range info.Links {
- selectedFiles[i].Link = link
+ // side note: iteration works!
+ i := 0
+ for el := selectedFiles.Front(); el != nil; el = el.Next() {
+ if i < len(info.Links) {
+ file := el.Value
+ file.Link = info.Links[i] // verified working!
+ selectedFiles.Set(el.Key, file)
+ i++
+ }
}
}
- // update file cache
- torrent.OriginalName = info.OriginalName
- torrent.Name = t.getName(torrent)
- if len(selectedFiles) > 0 {
- // update the torrent with more data!
- torrent.SelectedFiles = selectedFiles
- torrent.ForRepair = forRepair
- t.writeToFile(torrent)
+
+ info.ForRepair = forRepair
+ torrent := Torrent{
+ AccessKey: t.getName(info.Name, info.OriginalName),
+ SelectedFiles: selectedFiles,
+ Directories: t.getDirectories(info),
+ LatestAdded: info.Added,
+ InProgress: info.Progress != 100,
+ Instances: []realdebrid.TorrentInfo{*info},
}
+ if selectedFiles.Len() > 0 && torrentFromFile == nil {
+ t.writeToFile(info) // only when there are selected files, else it's useless
+ }
+ return &torrent
}
-func (t *TorrentManager) getName(torrent *Torrent) string {
+func (t *TorrentManager) getName(name, originalName string) string {
// drop the extension from the name
- if t.config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
- return torrent.Name
+ if t.config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
+ return name
} else {
- ret := strings.TrimSuffix(torrent.OriginalName, ".mp4")
+ ret := strings.TrimSuffix(originalName, ".mp4")
ret = strings.TrimSuffix(ret, ".mkv")
return ret
}
}
-// mapToDirectories maps torrents to directories
-func (t *TorrentManager) mapToDirectories() {
+func (t *TorrentManager) getDirectories(torrent *realdebrid.TorrentInfo) []string {
+ var ret []string
// Map torrents to directories
switch t.config.GetVersion() {
case "v1":
@@ -352,74 +367,29 @@ func (t *TorrentManager) mapToDirectories() {
groupMap := configV1.GetGroupMap()
// for every group, iterate over every torrent
// and then sprinkle/distribute the torrents to the directories of the group
- for group, directories := range groupMap {
- counter := make(map[string]int)
- for i := range t.torrents {
- // don't process torrents that are already mapped if it is not the first run
- alreadyMappedToGroup := false
- for _, mappedGroup := range t.processedTorrents[t.torrents[i].Name] {
- if mappedGroup == group {
- alreadyMappedToGroup = true
+ for _, directories := range groupMap {
+ for _, directory := range directories {
+ var filenames []string
+ for _, file := range torrent.Files {
+ if file.Selected == 0 {
+ continue
}
+ filenames = append(filenames, file.Path)
}
- if alreadyMappedToGroup {
- continue
+ accessKey := t.getName(torrent.Name, torrent.OriginalName)
+ if configV1.MeetsConditions(directory, torrent.ID, accessKey, filenames) {
+ ret = append(ret, directory)
+ break // we found a directory for this torrent for this group, so we can stop looking for more
}
-
- for _, directory := range directories {
- var filenames []string
- for _, file := range t.torrents[i].SelectedFiles {
- filenames = append(filenames, file.Path)
- }
- if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].Name, filenames) {
- found := false
- // check if it is already mapped to this directory
- for _, dir := range t.directoryMap[t.torrents[i].Name] {
- if dir == directory {
- found = true
- break // it is already mapped to this directory
- }
- }
- if !found {
- counter[directory]++
- t.mu.Lock()
- t.directoryMap[t.torrents[i].Name] = append(t.directoryMap[t.torrents[i].Name], directory)
- t.mu.Unlock()
- break // we found a directory for this torrent, so we can stop looking for more
- }
- }
- }
- t.mu.Lock()
- t.processedTorrents[t.torrents[i].Name] = append(t.processedTorrents[t.torrents[i].Name], group)
- t.mu.Unlock()
- }
- sum := 0
- for _, count := range counter {
- sum += count
- }
- if sum > 0 {
- t.log.Infof("Group processing completed: %s %v total: %d", group, counter, sum)
- } else {
- t.log.Infof("No new additions to directory group %s", group)
}
}
default:
t.log.Error("Unknown config version")
}
+ return ret
}
-// getByID returns a torrent by its ID
-func (t *TorrentManager) getByID(torrentID string) *Torrent {
- for i := range t.torrents {
- if t.torrents[i].ID == torrentID {
- return &t.torrents[i]
- }
- }
- return nil
-}
-
-// writeToFile writes a torrent to a file
-func (t *TorrentManager) writeToFile(torrent *Torrent) {
+func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) {
filePath := fmt.Sprintf("data/%s.bin", torrent.ID)
file, err := os.Create(filePath)
if err != nil {
@@ -433,8 +403,7 @@ func (t *TorrentManager) writeToFile(torrent *Torrent) {
dataEncoder.Encode(torrent)
}
-// readFromFile reads a torrent from a file
-func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
+func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo {
filePath := fmt.Sprintf("data/%s.bin", torrentID)
fileInfo, err := os.Stat(filePath)
if err != nil {
@@ -451,7 +420,7 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
}
defer file.Close()
- var torrent Torrent
+ var torrent realdebrid.TorrentInfo
dataDecoder := gob.NewDecoder(file)
err = dataDecoder.Decode(&torrent)
if err != nil {
@@ -463,246 +432,232 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
return &torrent
}
-func (t *TorrentManager) repairAll(wg *sync.WaitGroup) {
- wg.Wait()
- for _, torrent := range t.torrents {
- if torrent.ForRepair {
- t.log.Infof("Issues were detected on %s %s; fixing...", torrent.ID, torrent.Name)
- t.repair(torrent.ID, torrent.SelectedFiles, true)
- }
- if len(torrent.Links) == 0 && torrent.Progress == 100 {
- // If the torrent has no links
- // and already processing repair
- // delete it!
- t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.Name)
- realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
- }
- }
-}
-
-func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReinsertionFirst bool) {
- torrent := t.getByID(torrentID)
- if torrent == nil {
- return
- }
-
- // check if it is already "being" repaired
- found := false
- for _, hash := range t.inProgress {
- if hash == torrent.Hash {
- found = true
- break
- }
- }
- if found {
- t.log.Infof("Repair in progress, skipping %s", torrentID)
- return
- }
-
- // check if it is already repaired
- foundFiles := t.findAllDownloadedFilesFromHash(torrent.Hash)
- var missingFiles []File
- for _, sFile := range selectedFiles {
- if sFile.Link == "" || sFile.Unavailable {
- found := false
- for _, fFile := range foundFiles {
- // same file but different link, then yes it has been repaired
- if sFile.Path == fFile.Path && sFile.Link != fFile.Link {
- found = true
- break
- }
- }
- if !found {
- missingFiles = append(missingFiles, sFile)
- }
- }
- }
- if len(missingFiles) == 0 {
- t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.Name)
- return
- }
-
- // then we repair it!
- t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.Name)
- // check if we can still add more downloads
- proceed := t.canCapacityHandle()
- if !proceed {
- t.log.Error("Cannot add more torrents, exiting")
- return
- }
-
- // first solution: add the same selection, maybe it can be fixed by reinsertion?
- success := false
- if tryReinsertionFirst {
- success = t.reinsertTorrent(torrent, "", true)
- }
- if !success {
- // if all the selected files are missing but there are other streamable files
- var otherStreamableFileIDs []int
- for _, file := range torrent.Files {
- found := false
- for _, selectedFile := range selectedFiles {
- if selectedFile.ID == file.ID {
- found = true
- break
- }
- }
- if !found && isStreamable(file.Path) {
- otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID)
- }
- }
- if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 {
- // we will download 1 extra streamable file to force a redownload of the missing files
- // or if there's only 1 missing file, we will download 1 more to prevent a rename
- missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",")
- t.log.Infof("Redownloading %d missing files", len(missingFiles))
- t.reinsertTorrent(torrent, missingFilesPlus1, false)
- } else if len(selectedFiles) > 1 {
- // if not, last resort: add only the missing files but do it in 2 batches
- half := len(missingFiles) / 2
- missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
- missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
- if missingFiles1 != "" {
- t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1))
- t.reinsertTorrent(torrent, missingFiles1, false)
- }
- if missingFiles2 != "" {
- t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2))
- t.reinsertTorrent(torrent, missingFiles2, false)
- } else {
- t.log.Info("No other missing files left to reinsert")
- }
- } else {
- t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.Name)
- t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash)
- return
- }
- t.log.Info("Waiting for downloads to finish")
- }
-}
-
-func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool {
- // if missingFiles is not provided, look for missing files
- if missingFiles == "" {
- t.log.Info("Redownloading whole torrent", torrent.Name)
- var selection string
- for _, file := range torrent.SelectedFiles {
- selection += fmt.Sprintf("%d,", file.ID)
- }
- if selection == "" {
- return false
- }
- if len(selection) > 0 {
- missingFiles = selection[:len(selection)-1]
- }
- }
-
- // redownload torrent
- resp, err := realdebrid.AddMagnetHash(t.config.GetToken(), torrent.Hash)
- if err != nil {
- t.log.Errorf("Cannot redownload torrent: %v", err)
- return false
- }
- newTorrentID := resp.ID
- err = realdebrid.SelectTorrentFiles(t.config.GetToken(), newTorrentID, missingFiles)
- if err != nil {
- t.log.Errorf("Cannot start redownloading: %v", err)
- }
-
- if deleteIfFailed {
- if err != nil {
- realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
- return false
- }
- time.Sleep(1 * time.Second)
- // see if the torrent is ready
- info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), newTorrentID)
- if err != nil {
- t.log.Errorf("Cannot get info on redownloaded torrent: %v", err)
- if deleteIfFailed {
- realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
- }
- return false
- }
- time.Sleep(1 * time.Second)
-
- if info.Progress != 100 {
- t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress)
- realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
- return false
- }
-
- if len(info.Links) != len(torrent.SelectedFiles) {
- t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles))
- realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
- return false
- }
- t.log.Info("Redownload successful, deleting old torrent")
- realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
- }
- return true
-}
-
-func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) ([]File, bool) {
+func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap.OrderedMap[string, *File]) (*orderedmap.OrderedMap[string, *File], bool) {
type Result struct {
Response *realdebrid.UnrestrictResponse
}
- resultsChan := make(chan Result, len(info.Links))
+ resultsChan := make(chan Result, len(links))
var wg sync.WaitGroup
- // Limit concurrency
- sem := make(chan struct{}, t.config.GetNumOfWorkers())
-
- for _, link := range info.Links {
+ for _, link := range links {
wg.Add(1)
- sem <- struct{}{}
go func(lnk string) {
defer wg.Done()
- defer func() { <-sem }()
-
- unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
- return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk)
- }
- resp := realdebrid.RetryUntilOk(unrestrictFn)
- if resp != nil {
- resultsChan <- Result{Response: resp}
- }
+ t.workerPool <- true
+ resp := t.rd.UnrestrictUntilOk(lnk)
+ <-t.workerPool
+ resultsChan <- Result{Response: resp}
}(link)
}
go func() {
wg.Wait()
- close(sem)
close(resultsChan)
}()
isChaotic := false
for result := range resultsChan {
+ if result.Response == nil {
+ continue
+ }
found := false
- for i := range selectedFiles {
- if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) {
- selectedFiles[i].Link = result.Response.Link
+ // side note: iteration works!
+ for el := selectedFiles.Front(); el != nil; el = el.Next() {
+ if file, _ := selectedFiles.Get(el.Key); strings.Contains(file.Path, result.Response.Filename) {
+ file.Link = result.Response.Link
found = true
}
}
if !found {
- // "chaos" file, we don't know where it belongs
- isChaotic = !isStreamable(result.Response.Filename)
- selectedFiles = append(selectedFiles, File{
- File: realdebrid.File{
- Path: result.Response.Filename,
- Bytes: result.Response.Filesize,
- Selected: 1,
- },
- Link: result.Response.Link,
- })
+ if result.Response.Streamable == 1 {
+ selectedFiles.Set(filepath.Base(result.Response.Filename), &File{
+ File: realdebrid.File{
+ ID: math.MaxInt32,
+ Path: result.Response.Filename,
+ Bytes: result.Response.Filesize,
+ Selected: 1,
+ },
+ Link: result.Response.Link,
+ })
+ } else {
+ isChaotic = true
+ }
}
}
return selectedFiles, isChaotic
}
+func (t *TorrentManager) repairAll() {
+ // side note: iteration works!
+ for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
+ torrent := el.Value
+ // do not repair if in progress
+ if torrent.InProgress {
+ continue
+ }
+
+ // do not repair if all files have links
+ forRepair := false
+ for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() {
+ file := el2.Value
+ if file.Link == "" {
+ forRepair = true
+ break
+ }
+ }
+ if !forRepair {
+ // if it was marked for repair, unmark it
+ torrent.ForRepair = false
+ t.TorrentMap.Set(torrent.AccessKey, torrent)
+ continue
+ }
+
+ // when getting info, we mark it for repair if it's missing some links
+ if torrent.ForRepair {
+ t.log.Infof("Found torrent for repair %s", torrent.AccessKey)
+ t.Repair(torrent.AccessKey)
+ break // only repair the first one for repair and then move on
+ }
+ }
+}
+
+func (t *TorrentManager) Repair(accessKey string) {
+ if lastRepair, ok := t.repairMap.Get(accessKey); ok {
+ if time.Since(lastRepair) < time.Duration(24*time.Hour) { // magic number: 24 hrs
+ return
+ }
+ }
+ t.repairMap.Set(accessKey, time.Now())
+
+ if !t.config.EnableRepair() {
+ t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair")
+ return
+ }
+
+ torrent, _ := t.TorrentMap.Get(accessKey)
+ if torrent == nil {
+ t.log.Warnf("Cannot find torrent %s anymore to repair it", accessKey)
+ return
+ }
+ if torrent.InProgress {
+ t.log.Infof("Torrent %s is in progress, cannot repair", torrent.AccessKey)
+ return
+ }
+
+ // check if we can still add more downloads
+ proceed := t.canCapacityHandle()
+ if !proceed {
+ t.log.Error("Cannot add more torrents, ignoring repair request")
+ return
+ }
+
+ // make the file messy
+ var links []string
+ for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+ file := el.Value
+ if file.Link != "" {
+ links = append(links, file.Link)
+ }
+ file.Link = ""
+ }
+ selectedFiles, _ := t.organizeChaos(links, torrent.SelectedFiles)
+ torrent.SelectedFiles = selectedFiles
+ t.TorrentMap.Set(torrent.AccessKey, torrent)
+
+ // first solution: add the same selection, maybe it can be fixed by reinsertion?
+ if t.reinsertTorrent(torrent, "") {
+ t.log.Infof("Successfully downloaded torrent %s to repair it", torrent.AccessKey)
+ return
+ }
+ // if all the selected files are missing but there are other streamable files
+ var missingFiles []File
+ for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+ file := el.Value
+ if file.Link == "" {
+ missingFiles = append(missingFiles, *file)
+ }
+ }
+ if len(missingFiles) > 0 {
+ t.log.Infof("Redownloading %d missing files for torrent %s", len(missingFiles), torrent.AccessKey)
+ // if not, last resort: add only the missing files but do it in 2 batches
+ half := len(missingFiles) / 2
+ missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
+ missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
+ if missingFiles1 != "" {
+ t.reinsertTorrent(torrent, missingFiles1)
+ }
+ if missingFiles2 != "" {
+ t.reinsertTorrent(torrent, missingFiles2)
+ }
+ }
+}
+
+func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string) bool {
+ // if missingFiles is not provided, look for missing files
+ if missingFiles == "" {
+ var tmpSelection string
+ for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+ file := el.Value
+ tmpSelection += fmt.Sprintf("%d,", file.ID)
+ }
+ if tmpSelection == "" {
+ return false
+ }
+ if len(tmpSelection) > 0 {
+ missingFiles = tmpSelection[:len(tmpSelection)-1]
+ }
+ }
+
+ // redownload torrent
+ resp, err := t.rd.AddMagnetHash(torrent.Instances[0].Hash)
+ if err != nil {
+ t.log.Warnf("Cannot redownload torrent: %v", err)
+ return false
+ }
+ time.Sleep(1 * time.Second)
+
+ // select files
+ newTorrentID := resp.ID
+ err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
+ if err != nil {
+ t.log.Warnf("Cannot start redownloading: %v", err)
+ t.rd.DeleteTorrent(newTorrentID)
+ return false
+ }
+ time.Sleep(10 * time.Second)
+
+ // see if the torrent is ready
+ info, err := t.rd.GetTorrentInfo(newTorrentID)
+ if err != nil {
+ t.log.Warnf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
+ t.rd.DeleteTorrent(newTorrentID)
+ return false
+ }
+
+ if info.Status == "magnet_error" || info.Status == "error" || info.Status == "virus" || info.Status == "dead" {
+ t.log.Warnf("The redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
+ t.rd.DeleteTorrent(newTorrentID)
+ return false
+ }
+
+ if info.Progress != 100 {
+ t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion (this should fix the issue already)", info.ID)
+ return true
+ }
+
+ missingCount := len(strings.Split(missingFiles, ","))
+ if len(info.Links) != missingCount {
+ t.log.Infof("It did not fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount)
+ t.rd.DeleteTorrent(newTorrentID)
+ return false
+ }
+
+ t.log.Infof("Repair successful id=%s", newTorrentID)
+ return true
+}
+
func (t *TorrentManager) canCapacityHandle() bool {
// max waiting time is 45 minutes
const maxRetries = 50
@@ -710,9 +665,9 @@ func (t *TorrentManager) canCapacityHandle() bool {
const maxDelay = 60 * time.Second
retryCount := 0
for {
- count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
+ count, err := t.rd.GetActiveTorrentCount()
if err != nil {
- t.log.Errorf("Cannot get active downloads count: %v", err)
+ t.log.Warnf("Cannot get active downloads count: %v", err)
if retryCount >= maxRetries {
t.log.Error("Max retries reached. Exiting.")
return false
diff --git a/internal/torrent/types.go b/internal/torrent/types.go
index 37b0156..9e1888f 100644
--- a/internal/torrent/types.go
+++ b/internal/torrent/types.go
@@ -1,21 +1,22 @@
package torrent
import (
- "sync"
-
"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
+ "github.com/elliotchance/orderedmap/v2"
)
type Torrent struct {
- Version string
- realdebrid.Torrent
- SelectedFiles []File
+ AccessKey string
+ SelectedFiles *orderedmap.OrderedMap[string, *File]
+ Directories []string
+ LatestAdded string
+ InProgress bool
ForRepair bool
- lock *sync.Mutex
+
+ Instances []realdebrid.TorrentInfo
}
type File struct {
realdebrid.File
- Link string
- Unavailable bool
+ Link string
}
diff --git a/internal/universal/get.go b/internal/universal/get.go
index 8a8c3e9..55d3e3d 100644
--- a/internal/universal/get.go
+++ b/internal/universal/get.go
@@ -11,16 +11,15 @@ import (
"github.com/debridmediamanager.com/zurg/internal/dav"
intHttp "github.com/debridmediamanager.com/zurg/internal/http"
"github.com/debridmediamanager.com/zurg/internal/torrent"
+ zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
- "github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"github.com/hashicorp/golang-lru/v2/expirable"
"go.uber.org/zap"
)
// HandleGetRequest handles a GET request universally for both WebDAV and HTTP
func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
- rlog := logutil.NewLogger()
- log := rlog.Named("uniget")
+ log := logutil.NewLogger().Named("uniget")
requestPath := path.Clean(r.URL.Path)
isDav := true
@@ -35,93 +34,99 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent
// If there are less than 3 segments, return an error or adjust as needed
if len(segments) <= 3 {
if isDav {
- dav.HandlePropfindRequest(w, r, t, c, cache)
+ dav.HandlePropfindRequest(w, r, t, c)
} else {
- intHttp.HandleDirectoryListing(w, r, t, c, cache)
+ intHttp.HandleDirectoryListing(w, r, t, c)
}
return
}
- if data, exists := cache.Get(requestPath); exists {
- streamFileToResponse(data, w, r, c, log)
- return
- }
baseDirectory := segments[len(segments)-3]
- torrentName := segments[len(segments)-2]
+ accessKey := segments[len(segments)-2]
filename := segments[len(segments)-1]
- torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
- if torrents == nil {
- log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
+ torrent, _ := t.TorrentMap.Get(accessKey)
+ if torrent == nil {
+ log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
http.Error(w, "File not found", http.StatusNotFound)
return
}
- torrent, file := getFile(torrents, filename)
+ file, _ := torrent.SelectedFiles.Get(filename)
if file == nil {
- log.Errorf("Cannot find file from path %s", requestPath)
+ log.Warnf("Cannot find file from path %s", requestPath)
http.Error(w, "File not found", http.StatusNotFound)
return
}
+
+ if data, exists := cache.Get(requestPath); exists {
+ streamFileToResponse(torrent, data, w, r, t, c, log)
+ return
+ }
+
if file.Link == "" {
// This is a dead file, serve an alternate file
- log.Errorf("File %s is no longer available", filename)
- streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, c, log)
+ log.Warnf("File %s is not yet available, zurg is repairing the torrent", filename)
+ streamErrorVideo("https://www.youtube.com/watch?v=bGTqwt6vdcY", w, r, t, c, log)
return
}
link := file.Link
- unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
- return realdebrid.UnrestrictLink(c.GetToken(), link)
- }
- resp := realdebrid.RetryUntilOk(unrestrictFn)
+ resp := t.UnrestrictUntilOk(link)
if resp == nil {
- if !file.Unavailable {
- log.Errorf("Cannot unrestrict file %s %s", filename, link)
- t.HideTheFile(torrent, file)
- }
- streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, c, log)
+ go t.Repair(torrent.AccessKey)
+ log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path)
+ streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
return
} else if resp.Filename != filename {
actualExt := filepath.Ext(resp.Filename)
expectedExt := filepath.Ext(filename)
if actualExt != expectedExt && resp.Streamable != 1 {
- log.Errorf("File extension mismatch: %s and %s", filename, resp.Filename)
- streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, c, log)
+ log.Warnf("File was changed and is not streamable: %s and %s", filename, resp.Filename)
+ streamErrorVideo("https://www.youtube.com/watch?v=t9VgOriBHwE", w, r, t, c, log)
return
} else {
- log.Errorf("Filename mismatch: %s and %s", filename, resp.Filename)
+ log.Warnf("Filename mismatch: %s and %s", filename, resp.Filename)
}
}
cache.Add(requestPath, resp.Download)
- streamFileToResponse(resp.Download, w, r, c, log)
+ streamFileToResponse(torrent, resp.Download, w, r, t, c, log)
}
-func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
+func streamFileToResponse(torrent *torrent.Torrent, url string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
+ // Create a new request for the file download.
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
- log.Errorf("Error creating new request %v", err)
- streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, c, log)
+ log.Errorf("Error creating new request: %v", err)
+ streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, t, c, log)
return
}
- for k, values := range r.Header {
- for _, v := range values {
- req.Header.Add(k, v)
- }
+ // copy range header if it exists
+ if r.Header.Get("Range") != "" {
+ req.Header.Add("Range", r.Header.Get("Range"))
}
- resp, err := http.DefaultClient.Do(req)
+ // Create a custom HTTP client
+ client := zurghttp.NewHTTPClient(c.GetToken(), 10, c)
+
+ resp, err := client.Do(req)
if err != nil {
- log.Errorf("Error downloading file %v", err)
- streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, c, log)
+ log.Warnf("Cannot download file %v ; torrent is marked for repair", err)
+ if torrent != nil {
+ go t.Repair(torrent.AccessKey)
+ }
+ streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, t, c, log)
return
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
- log.Errorf("Received a nonOK status code %d", resp.StatusCode)
- streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, c, log)
+ log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status)
+ if torrent != nil {
+ go t.Repair(torrent.AccessKey)
+ }
+ streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, t, c, log)
return
}
@@ -135,14 +140,11 @@ func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, c
io.CopyBuffer(w, resp.Body, buf)
}
-func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, c config.ConfigInterface, log *zap.SugaredLogger) {
- unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
- return realdebrid.UnrestrictLink(c.GetToken(), link)
- }
- resp := realdebrid.RetryUntilOk(unrestrictFn)
+func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
+ resp := t.UnrestrictUntilOk(link)
if resp == nil {
http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError)
return
}
- streamFileToResponse(resp.Download, w, r, c, log)
+ streamFileToResponse(nil, resp.Download, w, r, t, c, log)
}
diff --git a/internal/universal/head.go b/internal/universal/head.go
index 28d3803..346839f 100644
--- a/internal/universal/head.go
+++ b/internal/universal/head.go
@@ -14,8 +14,7 @@ import (
)
func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
- rlog := logutil.NewLogger()
- log := rlog.Named("unihead")
+ log := logutil.NewLogger().Named("unihead")
requestPath := path.Clean(r.URL.Path)
requestPath = strings.Replace(requestPath, "/http", "", 1)
@@ -42,25 +41,25 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
}
baseDirectory := segments[len(segments)-3]
- torrentName := segments[len(segments)-2]
+ accessKey := segments[len(segments)-2]
filename := segments[len(segments)-1]
- torrents := t.FindAllTorrentsWithName(baseDirectory, torrentName)
- if torrents == nil {
- log.Errorf("Cannot find torrent %s in the directory %s", requestPath, baseDirectory)
- http.Error(w, "Cannot find file", http.StatusNotFound)
+ torrent, _ := t.TorrentMap.Get(accessKey)
+ if torrent == nil {
+ log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
+ http.Error(w, "File not found", http.StatusNotFound)
return
}
- _, file := getFile(torrents, filename)
+ file, _ := torrent.SelectedFiles.Get(filename)
if file == nil {
- log.Errorf("Cannot find file from path %s", requestPath)
+ log.Warnf("Cannot find file from path %s", requestPath)
http.Error(w, "Cannot find file", http.StatusNotFound)
return
}
if file.Link == "" {
// This is a dead file, serve an alternate file
- log.Errorf("File %s is no longer available", filename)
+ log.Warnf("File %s is no longer available", filename)
http.Error(w, "Cannot find file", http.StatusNotFound)
return
}
diff --git a/internal/universal/util.go b/internal/universal/util.go
deleted file mode 100644
index e9d5351..0000000
--- a/internal/universal/util.go
+++ /dev/null
@@ -1,20 +0,0 @@
-package universal
-
-import (
- "path/filepath"
-
- "github.com/debridmediamanager.com/zurg/internal/torrent"
-)
-
-// getFile finds a link by a fragment, it might be wrong
-func getFile(torrents []torrent.Torrent, filename string) (*torrent.Torrent, *torrent.File) {
- for t := range torrents {
- for f, file := range torrents[t].SelectedFiles {
- fname := filepath.Base(file.Path)
- if filename == fname {
- return &torrents[t], &torrents[t].SelectedFiles[f]
- }
- }
- }
- return nil, nil
-}
diff --git a/internal/zfs/object.go b/internal/zfs/object.go
index 8b4fe0e..7557941 100644
--- a/internal/zfs/object.go
+++ b/internal/zfs/object.go
@@ -5,7 +5,6 @@ import (
"fmt"
"os"
"path/filepath"
- "strings"
"syscall"
"time"
@@ -64,40 +63,32 @@ func (o Object) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
})
}
case DIRECTORY:
- seen := make(map[string]bool)
- for _, item := range o.fs.t.GetByDirectory(o.name) {
- if item.Progress != 100 {
+ for el := o.fs.t.TorrentMap.Front(); el != nil; el = el.Next() {
+ item := el.Value
+ if item.InProgress {
continue
}
- if _, exists := seen[item.Name]; exists {
- continue
- }
- seen[item.Name] = true
dirs = append(dirs, fuse.Dirent{
- Name: item.Name,
+ Name: item.AccessKey,
Type: fuse.DT_Dir,
})
}
case TORRENT:
- finalName := make(map[string]bool)
- for _, item := range o.fs.t.FindAllTorrentsWithName(o.parentName, o.name) {
- for _, file := range item.SelectedFiles {
- if file.Link == "" {
- // log.Println("File has no link, skipping", file.Path)
- continue
- }
- filename := filepath.Base(file.Path)
- if finalName[filename] {
- // fragment := davextra.GetLinkFragment(file.Link)
- // filename = davextra.InsertLinkFragment(filename, fragment)
- continue
- }
- finalName[filename] = true
- dirs = append(dirs, fuse.Dirent{
- Name: filename,
- Type: fuse.DT_File,
- })
+ torrent, _ := o.fs.t.TorrentMap.Get(o.name)
+ if torrent == nil {
+ return nil, syscall.ENOENT
+ }
+ for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
+ file := el.Value
+ if file.Link == "" {
+ // log.Println("File has no link, skipping", file.Path)
+ continue
}
+ filename := filepath.Base(file.Path)
+ dirs = append(dirs, fuse.Dirent{
+ Name: filename,
+ Type: fuse.DT_File,
+ })
}
}
return dirs, nil
@@ -119,33 +110,36 @@ func (o Object) Lookup(ctx context.Context, name string) (fs.Node, error) {
}
}
case DIRECTORY:
- for _, item := range o.fs.t.GetByDirectory(o.name) {
- if item.Name == name && item.Progress == 100 {
- return Object{
- fs: o.fs,
- objType: TORRENT,
- parentName: o.name,
- name: name,
- mtime: convertRFC3339toTime(item.Added),
- }, nil
- }
+ torrent, _ := o.fs.t.TorrentMap.Get(name)
+ if torrent == nil {
+ return nil, syscall.ENOENT
}
+ return Object{
+ fs: o.fs,
+ objType: TORRENT,
+ parentName: o.name,
+ name: name,
+ mtime: convertRFC3339toTime(torrent.LatestAdded),
+ }, nil
+
case TORRENT:
- for _, item := range o.fs.t.FindAllTorrentsWithName(o.parentName, o.name) {
- for _, file := range item.SelectedFiles {
- if strings.HasSuffix(file.Path, name) && file.Link != "" {
- return Object{
- fs: o.fs,
- objType: FILE,
- parentName: o.name,
- name: name,
- file: &file,
- size: uint64(file.Bytes),
- mtime: convertRFC3339toTime(item.Added),
- }, nil
- }
- }
+ torrent, _ := o.fs.t.TorrentMap.Get(name)
+ if torrent == nil {
+ return nil, syscall.ENOENT
}
+ file, _ := torrent.SelectedFiles.Get(name)
+ if file == nil {
+ return nil, syscall.ENOENT
+ }
+ return Object{
+ fs: o.fs,
+ objType: FILE,
+ parentName: o.name,
+ name: name,
+ file: file,
+ size: uint64(file.Bytes),
+ mtime: convertRFC3339toTime(torrent.LatestAdded),
+ }, nil
}
return nil, syscall.ENOENT
}
diff --git a/pkg/chunk/download.go b/pkg/chunk/download.go
index a9cd6ec..c26be05 100644
--- a/pkg/chunk/download.go
+++ b/pkg/chunk/download.go
@@ -9,8 +9,8 @@ import (
"time"
"github.com/debridmediamanager.com/zurg/internal/config"
+ "github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
- "github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"go.uber.org/zap"
"golang.org/x/sys/unix"
)
@@ -23,13 +23,14 @@ type Downloader struct {
lock sync.Mutex
storage *Storage
c config.ConfigInterface
+ t *torrent.TorrentManager
log *zap.SugaredLogger
}
type DownloadCallback func(error, []byte)
// NewDownloader creates a new download manager
-func NewDownloader(threads int, storage *Storage, bufferSize int64, c config.ConfigInterface) (*Downloader, error) {
+func NewDownloader(threads int, storage *Storage, bufferSize int64, t *torrent.TorrentManager, c config.ConfigInterface) (*Downloader, error) {
rlog := logutil.NewLogger()
log := rlog.Named("downloader")
@@ -39,6 +40,7 @@ func NewDownloader(threads int, storage *Storage, bufferSize int64, c config.Con
callbacks: make(map[RequestID][]DownloadCallback, 100),
storage: storage,
c: c,
+ t: t,
log: log,
}
@@ -103,10 +105,7 @@ func (d *Downloader) downloadFromAPI(request *Request, buffer []byte, delay int6
time.Sleep(time.Duration(delay) * time.Second)
}
- unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
- return realdebrid.UnrestrictLink(d.c.GetToken(), request.file.Link)
- }
- resp := realdebrid.RetryUntilOk(unrestrictFn)
+ resp := d.t.UnrestrictUntilOk(request.file.Link)
if resp == nil {
return fmt.Errorf("cannot unrestrict file %s %s", request.file.Path, request.file.Link)
}
diff --git a/pkg/chunk/manager.go b/pkg/chunk/manager.go
index 6713c49..681a812 100644
--- a/pkg/chunk/manager.go
+++ b/pkg/chunk/manager.go
@@ -58,6 +58,7 @@ func NewManager(
checkThreads,
loadThreads,
maxChunks int,
+ t *torrent.TorrentManager,
c config.ConfigInterface) (*Manager, error) {
pageSize := int64(os.Getpagesize())
@@ -81,7 +82,7 @@ func NewManager(
return nil, err
}
- downloader, err := NewDownloader(loadThreads, storage, chunkSize, c)
+ downloader, err := NewDownloader(loadThreads, storage, chunkSize, t, c)
if nil != err {
return nil, err
}
diff --git a/pkg/dav/response.go b/pkg/dav/response.go
index 14a1ca3..21ee25a 100644
--- a/pkg/dav/response.go
+++ b/pkg/dav/response.go
@@ -2,7 +2,7 @@ package dav
func Directory(path string) Response {
return Response{
- Href: customPathEscape(path),
+ Href: "/" + customPathEscape(path),
Propstat: PropStat{
Prop: Prop{
ResourceType: ResourceType{Value: ""},
@@ -14,7 +14,7 @@ func Directory(path string) Response {
func File(path string, fileSize int64, added string, link string) Response {
return Response{
- Href: customPathEscape(path),
+ Href: "/" + customPathEscape(path),
Propstat: PropStat{
Prop: Prop{
ContentLength: fileSize,
diff --git a/pkg/http/client.go b/pkg/http/client.go
new file mode 100644
index 0000000..342fd91
--- /dev/null
+++ b/pkg/http/client.go
@@ -0,0 +1,65 @@
+package http
+
+import (
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/debridmediamanager.com/zurg/internal/config"
+ "github.com/debridmediamanager.com/zurg/pkg/logutil"
+ "go.uber.org/zap"
+)
+
+type HTTPClient struct {
+ Client *http.Client
+ MaxRetries int
+ Backoff func(attempt int) time.Duration
+ CheckRespStatus func(resp *http.Response, err error, log *zap.SugaredLogger) bool
+ BearerToken string
+ log *zap.SugaredLogger
+ config config.ConfigInterface
+}
+
+func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
+ if r.config != nil && strings.Contains(req.Host, "download.real-debrid.") {
+ if host := r.config.GetRandomPreferredHost(); host != "" {
+ req.Host = r.config.GetRandomPreferredHost()
+ }
+ }
+ if r.BearerToken != "" {
+ req.Header.Set("Authorization", "Bearer "+r.BearerToken)
+ }
+ var resp *http.Response
+ var err error
+ for attempt := 0; attempt < r.MaxRetries; attempt++ {
+ resp, err = r.Client.Do(req)
+ if !r.CheckRespStatus(resp, err, r.log) {
+ return resp, err
+ }
+ time.Sleep(r.Backoff(attempt))
+ }
+ return resp, err
+}
+
+func NewHTTPClient(token string, maxRetries int, c config.ConfigInterface) *HTTPClient {
+ return &HTTPClient{
+ BearerToken: token,
+ Client: &http.Client{},
+ MaxRetries: maxRetries,
+ Backoff: func(attempt int) time.Duration {
+ return time.Duration(attempt) * time.Second
+ },
+ CheckRespStatus: func(resp *http.Response, err error, log *zap.SugaredLogger) bool {
+ if err != nil {
+ return true
+ }
+ if resp.StatusCode == 429 {
+ return true
+ }
+ // no need to retry
+ return false
+ },
+ log: logutil.NewLogger().Named("client"),
+ config: c,
+ }
+}
diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go
index 3d765b2..5367d7f 100644
--- a/pkg/realdebrid/api.go
+++ b/pkg/realdebrid/api.go
@@ -3,94 +3,70 @@ package realdebrid
import (
"bytes"
"encoding/json"
- "errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
+ "strings"
+
+ "github.com/debridmediamanager.com/zurg/internal/config"
+ zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
+ "go.uber.org/zap"
)
-func UnrestrictCheck(accessToken, link string) (*UnrestrictResponse, error) {
+type RealDebrid struct {
+ log *zap.SugaredLogger
+ client *zurghttp.HTTPClient
+}
+
+func NewRealDebrid(accessToken string, config config.ConfigInterface, log *zap.SugaredLogger) *RealDebrid {
+ maxRetries := 10
+ client := zurghttp.NewHTTPClient(accessToken, maxRetries, nil)
+ return &RealDebrid{
+ log: log,
+ client: client,
+ }
+}
+
+func (rd *RealDebrid) UnrestrictCheck(link string) (*UnrestrictResponse, error) {
data := url.Values{}
data.Set("link", link)
req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/check", bytes.NewBufferString(data.Encode()))
if err != nil {
+ rd.log.Errorf("Error when creating a unrestrict check request: %v", err)
return nil, err
}
-
- req.Header.Set("Authorization", "Bearer "+accessToken)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the unrestrict check request: %v", err)
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
+ rd.log.Errorf("Error when reading the body of unrestrict check response: %v", err)
return nil, err
}
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("HTTP error: %s", resp.Status)
- }
-
var response UnrestrictResponse
err = json.Unmarshal(body, &response)
if err != nil {
+ rd.log.Errorf("Error when decoding unrestrict check JSON: %v", err)
return nil, err
}
- return &response, nil
-}
-
-func UnrestrictLink(accessToken, link string) (*UnrestrictResponse, error) {
- data := url.Values{}
- data.Set("link", link)
-
- req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", bytes.NewBufferString(data.Encode()))
- if err != nil {
- return nil, err
- }
-
- req.Header.Set("Authorization", "Bearer "+accessToken)
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
-
- client := &http.Client{}
- resp, err := client.Do(req)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
-
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return nil, err
- }
-
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("HTTP error: %s", resp.Status)
- }
-
- var response UnrestrictResponse
- err = json.Unmarshal(body, &response)
- if err != nil {
- return nil, err
- }
-
- if !canFetchFirstByte(response.Download) {
- return nil, fmt.Errorf("can't fetch first byte")
- }
+ rd.log.Info("Link %s is streamable? %v", response.Streamable)
return &response, nil
}
// GetTorrents returns all torrents, paginated
// if customLimit is 0, the default limit of 2500 is used
-func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
+func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) {
baseURL := "https://api.real-debrid.com/rest/1.0/torrents"
var allTorrents []Torrent
page := 1
@@ -110,26 +86,24 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
+ rd.log.Errorf("Error when creating a get torrents request: %v", err)
return nil, 0, err
}
- req.Header.Set("Authorization", "Bearer "+accessToken)
-
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the get torrents request: %v", err)
return nil, 0, err
}
defer resp.Body.Close()
- if resp.StatusCode != http.StatusOK {
- return nil, 0, fmt.Errorf("HTTP error: %s", resp.Status)
- }
+ // if status code is not 2xx, return erro
var torrents []Torrent
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(&torrents)
if err != nil {
+ rd.log.Errorf("Error when decoding get torrents JSON: %v", err)
return nil, 0, err
}
@@ -147,39 +121,35 @@ func GetTorrents(accessToken string, customLimit int) ([]Torrent, int, error) {
page++
}
-
return allTorrents, totalCount, nil
}
-func GetTorrentInfo(accessToken, id string) (*Torrent, error) {
+func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) {
url := "https://api.real-debrid.com/rest/1.0/torrents/info/" + id
req, err := http.NewRequest("GET", url, nil)
if err != nil {
+ rd.log.Errorf("Error when creating a get info request: %v", err)
return nil, err
}
- req.Header.Set("Authorization", "Bearer "+accessToken)
-
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the get info request: %v", err)
return nil, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
+ rd.log.Errorf("Error when reading the body of get info response: %v", err)
return nil, err
}
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("HTTP error: %s", resp.Status)
- }
-
- var response Torrent
+ var response TorrentInfo
err = json.Unmarshal(body, &response)
if err != nil {
+ rd.log.Errorf("Error when : %v", err)
return nil, err
}
@@ -187,86 +157,54 @@ func GetTorrentInfo(accessToken, id string) (*Torrent, error) {
}
// SelectTorrentFiles selects files of a torrent to start it.
-func SelectTorrentFiles(accessToken string, id string, files string) error {
- // Prepare request data
+func (rd *RealDebrid) SelectTorrentFiles(id string, files string) error {
data := url.Values{}
data.Set("files", files)
- // Construct request URL
reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/selectFiles/%s", id)
req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode()))
if err != nil {
+ rd.log.Errorf("Error when creating a select files request: %v", err)
return err
}
- // Set request headers
- req.Header.Set("Authorization", "Bearer "+accessToken)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- // Send the request
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the select files request: %v", err)
return err
}
defer resp.Body.Close()
- // Handle response status codes
- switch resp.StatusCode {
- case http.StatusOK, http.StatusNoContent:
- return nil // Success
- case http.StatusAccepted:
- return errors.New("action already done")
- case http.StatusBadRequest:
- return errors.New("bad request")
- case http.StatusUnauthorized:
- return errors.New("bad token (expired or invalid)")
- case http.StatusForbidden:
- return errors.New("permission denied (account locked or not premium)")
- case http.StatusNotFound:
- return errors.New("wrong parameter (invalid file id(s)) or unknown resource (invalid id)")
- default:
- return fmt.Errorf("unexpected HTTP error: %s", resp.Status)
- }
+ rd.log.Debugf("Started the download for torrent id=%s", len(strings.Split(files, ",")), id)
+ return nil
}
// DeleteTorrent deletes a torrent from the torrents list.
-func DeleteTorrent(accessToken string, id string) error {
+func (rd *RealDebrid) DeleteTorrent(id string) error {
// Construct request URL
reqURL := fmt.Sprintf("https://api.real-debrid.com/rest/1.0/torrents/delete/%s", id)
req, err := http.NewRequest("DELETE", reqURL, nil)
if err != nil {
+ rd.log.Errorf("Error when creating a delete torrent request: %v", err)
return err
}
- // Set request headers
- req.Header.Set("Authorization", "Bearer "+accessToken)
-
// Send the request
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the delete torrent request: %v", err)
return err
}
defer resp.Body.Close()
- // Handle response status codes
- switch resp.StatusCode {
- case http.StatusNoContent:
- return nil // Success
- case http.StatusUnauthorized:
- return errors.New("bad token (expired or invalid)")
- case http.StatusForbidden:
- return errors.New("permission denied (account locked)")
- case http.StatusNotFound:
- return errors.New("unknown resource")
- default:
- return fmt.Errorf("unexpected HTTP error: %s", resp.Status)
- }
+ rd.log.Debugf("Deleted torrent with id=%s", id)
+ return nil
}
// AddMagnetHash adds a magnet link to download.
-func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
+func (rd *RealDebrid) AddMagnetHash(magnet string) (*MagnetResponse, error) {
// Prepare request data
data := url.Values{}
data.Set("magnet", fmt.Sprintf("magnet:?xt=urn:btih:%s", magnet))
@@ -275,77 +213,94 @@ func AddMagnetHash(accessToken, magnet string) (*MagnetResponse, error) {
reqURL := "https://api.real-debrid.com/rest/1.0/torrents/addMagnet"
req, err := http.NewRequest("POST", reqURL, bytes.NewBufferString(data.Encode()))
if err != nil {
+ rd.log.Errorf("Error when creating an add magnet request: %v", err)
return nil, err
}
- // Set request headers
- req.Header.Set("Authorization", "Bearer "+accessToken)
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
// Send the request
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the add magnet request: %v", err)
return nil, err
}
defer resp.Body.Close()
- // Handle response status codes
- switch resp.StatusCode {
- case http.StatusCreated:
- var response MagnetResponse
- err := json.NewDecoder(resp.Body).Decode(&response)
- if err != nil {
- return nil, err
- }
- return &response, nil
- case http.StatusBadRequest:
- return nil, errors.New("bad request")
- case http.StatusUnauthorized:
- return nil, errors.New("bad token (expired or invalid)")
- case http.StatusForbidden:
- return nil, errors.New("permission denied (account locked or not premium)")
- case http.StatusServiceUnavailable:
- return nil, errors.New("service unavailable")
- default:
- return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+ var response MagnetResponse
+ err = json.NewDecoder(resp.Body).Decode(&response)
+ if err != nil {
+ rd.log.Errorf("Error when decoding add magnet JSON: %v", err)
+ return nil, err
}
+
+ rd.log.Debugf("Added magnet %s with id=%s", magnet, response.ID)
+ return &response, nil
}
// GetActiveTorrentCount gets the number of currently active torrents and the current maximum limit.
-func GetActiveTorrentCount(accessToken string) (*ActiveTorrentCountResponse, error) {
+func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, error) {
// Construct request URL
reqURL := "https://api.real-debrid.com/rest/1.0/torrents/activeCount"
req, err := http.NewRequest("GET", reqURL, nil)
if err != nil {
+ rd.log.Errorf("Error when creating a active torrents request: %v", err)
return nil, err
}
- // Set request headers
- req.Header.Set("Authorization", "Bearer "+accessToken)
-
// Send the request
- client := &http.Client{}
- resp, err := client.Do(req)
+ resp, err := rd.client.Do(req)
if err != nil {
+ rd.log.Errorf("Error when executing the active torrents request: %v", err)
return nil, err
}
defer resp.Body.Close()
- // Handle response status codes
- switch resp.StatusCode {
- case http.StatusOK:
- var response ActiveTorrentCountResponse
- err := json.NewDecoder(resp.Body).Decode(&response)
- if err != nil {
- return nil, err
- }
- return &response, nil
- case http.StatusUnauthorized:
- return nil, errors.New("bad token (expired or invalid)")
- case http.StatusForbidden:
- return nil, errors.New("permission denied (account locked)")
- default:
- return nil, fmt.Errorf("unexpected HTTP error: %s", resp.Status)
+ var response ActiveTorrentCountResponse
+ err = json.NewDecoder(resp.Body).Decode(&response)
+ if err != nil {
+ rd.log.Errorf("Error when decoding active torrents JSON: %v", err)
+ return nil, err
}
+ return &response, nil
+}
+
+func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
+ data := url.Values{}
+ data.Set("link", link)
+
+ req, err := http.NewRequest("POST", "https://api.real-debrid.com/rest/1.0/unrestrict/link", bytes.NewBufferString(data.Encode()))
+ if err != nil {
+ rd.log.Errorf("Error when creating a unrestrict link request: %v", err)
+ return nil, err
+ }
+
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+ resp, err := rd.client.Do(req)
+ if err != nil {
+ rd.log.Errorf("Error when executing the unrestrict link request: %v", err)
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ rd.log.Errorf("Error when reading the body of unrestrict link response: %v", err)
+ return nil, err
+ }
+
+ var response UnrestrictResponse
+ err = json.Unmarshal(body, &response)
+ if err != nil {
+ rd.log.Errorf("Error when decoding unrestrict link JSON: %v", err)
+ return nil, err
+ }
+
+ if !canFetchFirstByte(response.Download) {
+ return nil, fmt.Errorf("can't fetch first byte")
+ }
+
+ // rd.log.Debugf("Unrestricted link %s into %s", link, response.Download)
+ return &response, nil
}
diff --git a/pkg/realdebrid/network.go b/pkg/realdebrid/network.go
new file mode 100644
index 0000000..c33e62d
--- /dev/null
+++ b/pkg/realdebrid/network.go
@@ -0,0 +1,106 @@
+package realdebrid
+
+import (
+ "fmt"
+ "os/exec"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+)
+
+type IPInfo struct {
+ Address string
+ Hops int
+ Latency time.Duration
+}
+
+func traceroute(ip string) (int, time.Duration, error) {
+ cmd := exec.Command("traceroute", "-n", "-q", "1", "-w", "1", ip)
+ out, err := cmd.CombinedOutput()
+ if err != nil {
+ return 0, 0, err
+ }
+
+ output := string(out)
+ lines := strings.Split(output, "\n")
+
+ hopCount := len(lines) - 1
+
+ var latency time.Duration
+ if hopCount > 0 {
+ lastLine := lines[hopCount-1]
+ fmt.Println(lastLine)
+ parts := strings.Fields(lastLine)
+ if len(parts) >= 3 {
+ latencyValue, parseErr := strconv.ParseFloat(parts[2], 64)
+ if parseErr == nil {
+ latency = time.Duration(latencyValue * float64(time.Millisecond))
+ }
+ }
+ }
+
+ return hopCount, latency, nil
+}
+
+func RunTest() {
+ fmt.Println("Running network test...")
+
+ ips := []string{"20.download.real-debrid.cloud", "20.download.real-debrid.com", "21.download.real-debrid.cloud", "21.download.real-debrid.com", "22.download.real-debrid.cloud", "22.download.real-debrid.com", "23.download.real-debrid.cloud", "23.download.real-debrid.com", "30.download.real-debrid.cloud", "30.download.real-debrid.com", "31.download.real-debrid.cloud", "31.download.real-debrid.com", "32.download.real-debrid.cloud", "32.download.real-debrid.com", "34.download.real-debrid.cloud", "34.download.real-debrid.com", "40.download.real-debrid.cloud", "40.download.real-debrid.com", "41.download.real-debrid.cloud", "41.download.real-debrid.com", "42.download.real-debrid.cloud", "42.download.real-debrid.com", "43.download.real-debrid.cloud", "43.download.real-debrid.com", "44.download.real-debrid.cloud", "44.download.real-debrid.com", "45.download.real-debrid.cloud", "45.download.real-debrid.com", "50.download.real-debrid.cloud", "50.download.real-debrid.com", "51.download.real-debrid.cloud", "51.download.real-debrid.com", "52.download.real-debrid.cloud", "52.download.real-debrid.com", "53.download.real-debrid.cloud", "53.download.real-debrid.com", "54.download.real-debrid.cloud", "54.download.real-debrid.com", "55.download.real-debrid.cloud", "55.download.real-debrid.com", "56.download.real-debrid.cloud", "56.download.real-debrid.com", "57.download.real-debrid.cloud", "57.download.real-debrid.com", "58.download.real-debrid.cloud", "58.download.real-debrid.com", "59.download.real-debrid.cloud", "59.download.real-debrid.com", "60.download.real-debrid.cloud", "60.download.real-debrid.com", "61.download.real-debrid.cloud", "61.download.real-debrid.com", "62.download.real-debrid.cloud", "62.download.real-debrid.com", "63.download.real-debrid.cloud", "63.download.real-debrid.com", "64.download.real-debrid.cloud", "64.download.real-debrid.com", "65.download.real-debrid.cloud", "65.download.real-debrid.com", "66.download.real-debrid.cloud", "66.download.real-debrid.com", "67.download.real-debrid.cloud", "67.download.real-debrid.com", "68.download.real-debrid.cloud", "68.download.real-debrid.com", "69.download.real-debrid.cloud", "69.download.real-debrid.com", "hkg1.download.real-debrid.com", "lax1.download.real-debrid.com", "lon1.download.real-debrid.com", "mum1.download.real-debrid.com", "rbx.download.real-debrid.com", "sgp1.download.real-debrid.com", "tlv1.download.real-debrid.com", "tyo1.download.real-debrid.com"}
+
+ var wg sync.WaitGroup
+ infoChan := make(chan IPInfo, len(ips))
+ semaphore := make(chan struct{}, 10)
+
+ for _, ip := range ips {
+ wg.Add(1)
+ semaphore <- struct{}{}
+ go func(ip string) {
+ defer wg.Done()
+ hops, latency, err := traceroute(ip)
+ if err != nil {
+ fmt.Println("Error performing traceroute:", err)
+ } else {
+ infoChan <- IPInfo{Address: ip, Hops: hops, Latency: latency}
+ }
+ <-semaphore
+ }(ip)
+ }
+
+ wg.Wait()
+ close(infoChan)
+
+ var ipInfos []IPInfo
+ for info := range infoChan {
+ ipInfos = append(ipInfos, info)
+ }
+
+ sort.Slice(ipInfos, func(i, j int) bool {
+ if ipInfos[i].Hops == ipInfos[j].Hops {
+ return ipInfos[i].Latency < ipInfos[j].Latency
+ }
+ return ipInfos[i].Hops < ipInfos[j].Hops
+ })
+ var lowestLatency time.Duration
+ if len(ipInfos) > 0 {
+ lowestLatency = ipInfos[0].Latency
+ for _, info := range ipInfos {
+ if info.Latency < lowestLatency {
+ lowestLatency = info.Latency
+ }
+ }
+ }
+
+ latencyThreshold := lowestLatency + lowestLatency/3
+
+ var okIPs []IPInfo
+ for _, info := range ipInfos {
+ if info.Latency <= latencyThreshold {
+ okIPs = append(okIPs, info)
+ }
+ }
+ for _, info := range okIPs {
+ fmt.Printf("Host: %s, Hops: %d, Latency: %v\n", info.Address, info.Hops, info.Latency)
+ }
+}
diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go
index ca08b21..a27fb92 100644
--- a/pkg/realdebrid/types.go
+++ b/pkg/realdebrid/types.go
@@ -20,31 +20,45 @@ type UnrestrictResponse struct {
}
type Torrent struct {
- ID string `json:"id"`
- Name string `json:"filename"`
- OriginalName string `json:"original_filename"`
- Hash string `json:"hash"`
- Progress int `json:"-"`
- Added string `json:"added"`
- Bytes int64 `json:"bytes"`
- Links []string `json:"links"`
- Files []File `json:"files,omitempty"`
+ ID string `json:"id"`
+ Name string `json:"filename"`
+ Hash string `json:"hash"`
+ Progress int `json:"-"`
+ Added string `json:"added"`
+ Bytes int64 `json:"bytes"`
+ Links []string `json:"links"`
}
-func (t *Torrent) UnmarshalJSON(data []byte) error {
- type Alias Torrent
+type TorrentInfo struct {
+ ID string `json:"id"`
+ Name string `json:"filename"`
+ Hash string `json:"hash"`
+ Progress int `json:"-"`
+ Status string `json:"status"`
+ Added string `json:"added"`
+ Bytes int64 `json:"bytes"`
+ Links []string `json:"links"`
+ OriginalName string `json:"original_filename"` // from info
+ OriginalBytes int64 `json:"original_bytes"` // from info
+ Files []File `json:"files"` // from info
+ ForRepair bool `json:"-"`
+ Version string `json:"-"`
+}
+
+func (i *TorrentInfo) UnmarshalJSON(data []byte) error {
+ type Alias TorrentInfo
aux := &struct {
Progress float64 `json:"progress"`
*Alias
}{
- Alias: (*Alias)(t),
+ Alias: (*Alias)(i),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
- t.Progress = int(math.Round(aux.Progress))
+ i.Progress = int(math.Round(aux.Progress))
return nil
}
diff --git a/pkg/realdebrid/util.go b/pkg/realdebrid/unrestrict.go
similarity index 80%
rename from pkg/realdebrid/util.go
rename to pkg/realdebrid/unrestrict.go
index e1b2f65..72fee99 100644
--- a/pkg/realdebrid/util.go
+++ b/pkg/realdebrid/unrestrict.go
@@ -7,7 +7,19 @@ import (
"time"
)
-func RetryUntilOk[T any](fn func() (T, error)) T {
+func (rd *RealDebrid) UnrestrictUntilOk(link string) *UnrestrictResponse {
+ if link == "" {
+ return nil
+ }
+ unrestrictFn := func(link string) (*UnrestrictResponse, error) {
+ return rd.UnrestrictLink(link)
+ }
+ return retryUntilOk(func() (*UnrestrictResponse, error) {
+ return unrestrictFn(link)
+ })
+}
+
+func retryUntilOk[T any](fn func() (T, error)) T {
const initialDelay = 1 * time.Second
const maxDelay = 128 * time.Second
for i := 0; ; i++ {