From 15a0ba95d8530600989d59db958969f31598ca7d Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Thu, 9 Nov 2023 02:34:04 +0100 Subject: [PATCH] Refactor torrent manager --- cmd/zurg/main.go | 12 +- go.mod | 11 + go.sum | 27 ++ internal/config/v1.go | 5 - internal/dav/listing.go | 12 +- internal/dav/response.go | 4 +- internal/http/listing.go | 53 ++- internal/http/response.go | 60 --- internal/net/router.go | 4 +- internal/torrent/manager.go | 721 ++++++++++++++++-------------------- internal/torrent/types.go | 10 +- internal/universal/get.go | 8 +- pkg/realdebrid/api.go | 7 +- pkg/realdebrid/types.go | 18 +- 14 files changed, 436 insertions(+), 516 deletions(-) delete mode 100644 internal/http/response.go diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go index d5b1e16..33f9e7d 100644 --- a/cmd/zurg/main.go +++ b/cmd/zurg/main.go @@ -14,6 +14,7 @@ import ( "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/logutil" "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/nutsdb/nutsdb" ) func main() { @@ -25,9 +26,18 @@ func main() { log.Panicf("Config failed to load: %v", configErr) } + db, err := nutsdb.Open( + nutsdb.DefaultOptions, + nutsdb.WithDir("/tmp/nutsdb"), + ) + if err != nil { + log.Fatal(err) + } + defer db.Close() + cache := expirable.NewLRU[string, string](1e4, nil, time.Hour) - torrentMgr := torrent.NewTorrentManager(config, cache) + torrentMgr := torrent.NewTorrentManager(config, cache, db) mux := http.NewServeMux() net.Router(mux, config, torrentMgr, cache) diff --git a/go.mod b/go.mod index ad5f920..1b30310 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,18 @@ require ( ) require ( + github.com/antlabs/stl v0.0.1 // indirect + github.com/antlabs/timer v0.0.11 // indirect + github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/nutsdb/nutsdb v0.14.1 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/tidwall/btree v1.6.0 // indirect + github.com/xujiajun/mmap-go v1.0.1 // indirect + github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect go.uber.org/multierr v1.10.0 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/go.sum b/go.sum index 5cfeeab..6061d0f 100644 --- a/go.sum +++ b/go.sum @@ -1,22 +1,49 @@ +github.com/antlabs/stl v0.0.1 h1:TRD3csCrjREeLhLoQ/supaoCvFhNLBTNIwuRGrDIs6Q= +github.com/antlabs/stl v0.0.1/go.mod h1:wvVwP1loadLG3cRjxUxK8RL4Co5xujGaZlhbztmUEqQ= +github.com/antlabs/timer v0.0.11 h1:z75oGFLeTqJHMOcWzUPBKsBbQAz4Ske3AfqJ7bsdcwU= +github.com/antlabs/timer v0.0.11/go.mod h1:JNV8J3yGvMKhCavGXgj9HXrVZkfdQyKCcqXBT8RdyuU= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/nutsdb/nutsdb v0.14.1 h1:z+Kth/kz2oYqKmOMBZho1YK2183xjrcl6KExRtCFl18= +github.com/nutsdb/nutsdb v0.14.1/go.mod h1:6inOji9rFBporXeHDjJny4g50RpQbkjSK5jI1hht0j8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= +github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= +github.com/xujiajun/mmap-go v1.0.1 h1:7Se7ss1fLPPRW+ePgqGpCkfGIZzJV6JPq9Wq9iv/WHc= +github.com/xujiajun/mmap-go v1.0.1/go.mod h1:CNN6Sw4SL69Sui00p0zEzcZKbt+5HtEnYUsc6BKKRMg= +github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 h1:w0si+uee0iAaCJO9q86T6yrhdadgcsoNuh47LrUykzg= +github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235/go.mod h1:MR4+0R6A9NS5IABnIM3384FfOq8QFVnm7WDrBOhIaMU= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/v1.go b/internal/config/v1.go index 85fbfb2..dd2bf9b 100644 --- a/internal/config/v1.go +++ b/internal/config/v1.go @@ -5,7 +5,6 @@ import ( "sort" "strings" - "github.com/debridmediamanager.com/zurg/pkg/logutil" "gopkg.in/yaml.v3" ) @@ -33,8 +32,6 @@ func (z *ZurgConfigV1) GetDirectories() []string { } func (z *ZurgConfigV1) GetGroupMap() map[string][]string { - log := logutil.NewLogger().Named("config") - var groupMap = make(map[string][]string) var groupOrderMap = make(map[string]int) // To store GroupOrder for each directory @@ -42,7 +39,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string { for directory, val := range z.Directories { groupMap[val.Group] = append(groupMap[val.Group], directory) groupOrderMap[directory] = val.GroupOrder - log.Debugf("Added directory to group: %s, group: %s, order: %d", directory, val.Group, val.GroupOrder) } // Sort the slice based on GroupOrder and then directory name for deterministic order @@ -54,7 +50,6 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string { return groupOrderMap[dirs[i]] < groupOrderMap[dirs[j]] }) groupMap[group] = dirs - log.Debugf("Sorted directories within a group: %s %v", group, dirs) } // Return a deep copy of the map diff --git a/internal/dav/listing.go b/internal/dav/listing.go index 805017f..0a9dae5 100644 --- a/internal/dav/listing.go +++ b/internal/dav/listing.go @@ -11,22 +11,14 @@ import ( "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/dav" "github.com/debridmediamanager.com/zurg/pkg/logutil" - "github.com/hashicorp/golang-lru/v2/expirable" ) -func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) { +func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) { log := logutil.NewLogger().Named("dav") requestPath := path.Clean(r.URL.Path) requestPath = strings.Trim(requestPath, "/") - if data, exists := cache.Get(requestPath); exists { - w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"") - w.WriteHeader(http.StatusMultiStatus) - fmt.Fprint(w, data) - return - } - var output []byte var err error @@ -53,8 +45,6 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To if output != nil { respBody := fmt.Sprintf("\n%s\n", output) - cache.Add(requestPath, respBody) - w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"") w.WriteHeader(http.StatusMultiStatus) fmt.Fprint(w, respBody) diff --git a/internal/dav/response.go b/internal/dav/response.go index c89d623..15d00b8 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -15,7 +15,7 @@ func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (*d seen := make(map[string]bool) for _, item := range torrents { - if item.Progress != 100 { + if item.InProgress { continue } if _, exists := seen[item.AccessKey]; exists { @@ -64,7 +64,7 @@ func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (* torrentResponses = append(torrentResponses, dav.File( filePath, file.Bytes, - convertRFC3339toRFC1123(torrent.Added), + convertRFC3339toRFC1123(torrent.LatestAdded), file.Link, )) } diff --git a/internal/http/listing.go b/internal/http/listing.go index 24746a3..6e39b9b 100644 --- a/internal/http/listing.go +++ b/internal/http/listing.go @@ -5,26 +5,19 @@ import ( "net/http" "net/url" "path" + "path/filepath" "strings" "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/logutil" - "github.com/hashicorp/golang-lru/v2/expirable" ) -func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) { +func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) { log := logutil.NewLogger().Named("http") requestPath := path.Clean(r.URL.Path) - if data, exists := cache.Get(requestPath); exists { - w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"") - w.WriteHeader(http.StatusOK) - fmt.Fprint(w, data) - return - } - var output *string var err error @@ -49,8 +42,6 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T } if output != nil { - cache.Add(requestPath, *output) - w.Header().Set("Content-Type", "text/html; charset=\"utf-8\"") w.WriteHeader(http.StatusOK) fmt.Fprint(w, *output) @@ -73,12 +64,19 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req for _, directory := range c.GetDirectories() { if basePath == directory { - torrents := t.GetByDirectory(basePath) - resp, err := createMultiTorrentResponse(requestPath, torrents) - if err != nil { - return nil, fmt.Errorf("cannot read directory (%s): %w", basePath, err) + htmlDoc := "
    " + for name, torrent := range t.TorrentMap { + if len(torrent.SelectedFiles) == 0 { + continue + } + for _, dir := range torrent.Directories { + if dir == basePath { + htmlDoc += fmt.Sprintf("
  1. %s
    2. ", filepath.Join(requestPath, url.PathEscape(name)), name) + break + } + } } - return &resp, nil + return &htmlDoc, nil } } @@ -86,18 +84,17 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req } func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) { - fullDir := path.Dir(requestPath) - directory := path.Base(fullDir) torrentName := path.Base(requestPath) - - sameNameTorrents := t.FindAllTorrentsWithName(directory, torrentName) - if len(sameNameTorrents) == 0 { - return nil, fmt.Errorf("cannot find directory when generating single torrent: %s", requestPath) + htmlDoc := "
      " + for _, file := range t.TorrentMap[torrentName].SelectedFiles { + if file.Link == "" { + // TODO: fix the file? + fmt.Printf("File %s has no link, skipping\n", file.Path) + continue + } + filename := filepath.Base(file.Path) + filePath := filepath.Join(requestPath, url.PathEscape(filename)) + htmlDoc += fmt.Sprintf("
    1. %s
      2. ", filePath, filename) } - - resp, err := createSingleTorrentResponse(requestPath, sameNameTorrents) - if err != nil { - return nil, fmt.Errorf("cannot read directory (%s): %w", requestPath, err) - } - return &resp, nil + return &htmlDoc, nil } diff --git a/internal/http/response.go b/internal/http/response.go deleted file mode 100644 index 13e0ec9..0000000 --- a/internal/http/response.go +++ /dev/null @@ -1,60 +0,0 @@ -package http - -import ( - "fmt" - "net/url" - "path/filepath" - - "github.com/debridmediamanager.com/zurg/internal/torrent" -) - -// createMultiTorrentResponse creates a WebDAV response for a list of torrents -func createMultiTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) { - htmlDoc := "
        " - - seen := make(map[string]bool) - - for _, item := range torrents { - if item.Progress != 100 { - continue - } - if _, exists := seen[item.AccessKey]; exists { - continue - } - seen[item.AccessKey] = true - - path := filepath.Join(basePath, url.PathEscape(item.AccessKey)) - htmlDoc += fmt.Sprintf("
      • %s
        • ", path, item.AccessKey) - } - - return htmlDoc, nil -} - -func createSingleTorrentResponse(basePath string, torrents []torrent.Torrent) (string, error) { - htmlDoc := "
          " - - finalName := make(map[string]bool) - - currentPath := filepath.Join(basePath) - - for _, torrent := range torrents { - for _, file := range torrent.SelectedFiles { - if file.Link == "" { - // TODO: fix the file? - // log.Println("File has no link, skipping", file.Path) - continue - } - - filename := filepath.Base(file.Path) - if finalName[filename] { - continue - } - finalName[filename] = true - - filePath := filepath.Join(currentPath, url.PathEscape(filename)) - htmlDoc += fmt.Sprintf("
        • %s
          • ", filePath, filename) - } - } - - return htmlDoc, nil -} diff --git a/internal/net/router.go b/internal/net/router.go index 08cd26b..c5d8caa 100644 --- a/internal/net/router.go +++ b/internal/net/router.go @@ -25,7 +25,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 { universal.HandleGetRequest(w, r, t, c, cache) } else { - intHttp.HandleDirectoryListing(w, r, t, c, cache) + intHttp.HandleDirectoryListing(w, r, t, c) } case http.MethodHead: @@ -40,7 +40,7 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { switch r.Method { case "PROPFIND": - dav.HandlePropfindRequest(w, r, t, c, cache) + dav.HandlePropfindRequest(w, r, t, c) case http.MethodGet: universal.HandleGetRequest(w, r, t, c, cache) diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index e6b0780..21c6667 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -3,7 +3,6 @@ package torrent import ( "encoding/gob" "fmt" - "math" "os" "strings" "sync" @@ -13,18 +12,18 @@ import ( "github.com/debridmediamanager.com/zurg/pkg/logutil" "github.com/debridmediamanager.com/zurg/pkg/realdebrid" "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/nutsdb/nutsdb" "go.uber.org/zap" ) type TorrentManager struct { + TorrentMap map[string]*Torrent requiredVersion string rd *realdebrid.RealDebrid - torrents []Torrent - torrentMap map[string]*Torrent - inProgress []string checksum string config config.ConfigInterface cache *expirable.LRU[string, string] + db *nutsdb.DB workerPool chan bool directoryMap map[string][]string processedTorrents map[string][]string @@ -34,14 +33,15 @@ type TorrentManager struct { // NewTorrentManager creates a new torrent manager // it will fetch all torrents and their info in the background -// and store them in-memory; it is called only once at startup -func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager { +// and store them in-memory and cached in files +func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string], db *nutsdb.DB) *TorrentManager { t := &TorrentManager{ + TorrentMap: make(map[string]*Torrent), requiredVersion: fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()), rd: realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")), - torrentMap: make(map[string]*Torrent), config: config, cache: cache, + db: db, workerPool: make(chan bool, config.GetNumOfWorkers()), directoryMap: make(map[string][]string), processedTorrents: make(map[string][]string), @@ -51,8 +51,6 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin // start with a clean slate t.mu.Lock() - t.cache.Purge() - t.torrents = nil newTorrents, _, err := t.rd.GetTorrents(0) if err != nil { @@ -76,46 +74,74 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin if newTorrent == nil { continue } - t.torrents = append(t.torrents, *newTorrent) - if _, exists := t.torrentMap[newTorrent.AccessKey]; exists { - t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files - t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links - t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles - t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair + if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists { + t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent) } else { - t.torrentMap[newTorrent.AccessKey] = newTorrent + t.TorrentMap[newTorrent.AccessKey] = newTorrent } } t.checksum = t.getChecksum() t.mu.Unlock() - if t.config.EnableRepair() { - go t.repairAll() - } + // if t.config.EnableRepair() { + // go t.repairAll() + // } go t.startRefreshJob() return t } +func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent { + merged := t1 + + // Merge SelectedFiles + fileMap := make(map[int]File) + for _, f := range append(t1.SelectedFiles, t2.SelectedFiles...) { + if _, exists := fileMap[f.ID]; !exists { + fileMap[f.ID] = f + } + } + for _, f := range fileMap { + merged.SelectedFiles = append(merged.SelectedFiles, f) + } + + // Merge Instances + merged.Instances = append(t1.Instances, t2.Instances...) + + // LatestAdded + if t1.LatestAdded < t2.LatestAdded { + merged.LatestAdded = t2.LatestAdded + } + + // InProgress + for _, instance := range merged.Instances { + if instance.Progress != 100 { + merged.InProgress = true + break + } + } + + return merged +} + // GetByDirectory returns all torrents that have a file in the specified directory func (t *TorrentManager) GetByDirectory(directory string) []Torrent { var torrents []Torrent - for i := range t.torrents { - for _, dir := range t.directoryMap[t.torrents[i].AccessKey] { + for k, v := range t.TorrentMap { + found := false + for _, dir := range v.Directories { if dir == directory { - torrents = append(torrents, t.torrents[i]) + found = true + break } } + if found { + torrents = append(torrents, *t.TorrentMap[k]) + } } return torrents } -// HideTheFile marks a file as deleted -func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) { - file.Unavailable = true - t.repair(torrent.ID, torrent.SelectedFiles, false) -} - // FindAllTorrentsWithName finds all torrents in a given directory with a given name func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string) []Torrent { var matchingTorrents []Torrent @@ -133,26 +159,12 @@ func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictRe return t.rd.UnrestrictUntilOk(link) } -// findAllDownloadedFilesFromHash finds all files that were with a given hash -func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File { - var files []File - for _, torrent := range t.torrents { - if torrent.Hash == hash { - for _, file := range torrent.SelectedFiles { - if file.Link != "" { - files = append(files, file) - } - } - } - } - return files -} - type torrentsResponse struct { torrents []realdebrid.Torrent totalCount int } +// generates a checksum based on the number of torrents, the first torrent id and the number of active torrents func (t *TorrentManager) getChecksum() string { torrentsChan := make(chan torrentsResponse) countChan := make(chan int) @@ -214,8 +226,6 @@ func (t *TorrentManager) startRefreshJob() { } t.mu.Lock() - t.cache.Purge() - t.torrents = nil newTorrents, _, err := t.rd.GetTorrents(0) if err != nil { @@ -229,7 +239,6 @@ func (t *TorrentManager) startRefreshJob() { wg.Add(1) go func(idx int) { defer wg.Done() - t.log.Debug(newTorrents[idx].ID) t.workerPool <- true torrentsChan <- t.getMoreInfo(newTorrents[idx]) <-t.workerPool @@ -241,73 +250,58 @@ func (t *TorrentManager) startRefreshJob() { if newTorrent == nil { continue } - t.torrents = append(t.torrents, *newTorrent) - if _, exists := t.torrentMap[newTorrent.AccessKey]; exists { - t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files - t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links - t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles - t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair + if _, exists := t.TorrentMap[newTorrent.AccessKey]; exists { + t.TorrentMap[newTorrent.AccessKey] = t.mergeToMain(t.TorrentMap[newTorrent.AccessKey], newTorrent) } else { - t.torrentMap[newTorrent.AccessKey] = newTorrent + t.TorrentMap[newTorrent.AccessKey] = newTorrent } } t.checksum = t.getChecksum() t.mu.Unlock() - if t.config.EnableRepair() { - go t.repairAll() - } + // if t.config.EnableRepair() { + // go t.repairAll() + // } go OnLibraryUpdateHook(t.config) } } -// getMoreInfo updates the selected files for a torrent +// getMoreInfo gets original name, size and files for a torrent func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { - t.log.Info("Getting more info for", rdTorrent.ID) + var info *realdebrid.TorrentInfo + var err error // file cache torrentFromFile := t.readFromFile(rdTorrent.ID) - if torrentFromFile != nil { + if torrentFromFile != nil && len(torrentFromFile.ID) > 0 && len(torrentFromFile.Links) == len(rdTorrent.Links) { // see if api data and file data still match // then it means data is still usable - if len(torrentFromFile.Links) == len(rdTorrent.Links) { - return torrentFromFile + info = torrentFromFile + } + if info == nil { + info, err = t.rd.GetTorrentInfo(rdTorrent.ID) + if err != nil { + t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err) + return nil } } - t.log.Debug("Getting info for", rdTorrent.ID) - info, err := t.rd.GetTorrentInfo(rdTorrent.ID) - if err != nil { - t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err) - return nil - } - - torrent := Torrent{ - Version: t.requiredVersion, - Torrent: *info, - SelectedFiles: nil, - ForRepair: false, - } - // SelectedFiles is a subset of Files with only the selected ones // it also has a Link field, which can be empty // if it is empty, it means the file is no longer available // Files+Links together are the same as SelectedFiles var selectedFiles []File - var streamableFiles []File + streamableCount := 0 // if some Links are empty, we need to repair it forRepair := false for _, file := range info.Files { if isStreamable(file.Path) { - streamableFiles = append(streamableFiles, File{ - File: file, - Link: "", - }) + streamableCount++ } if file.Selected == 0 { continue } selectedFiles = append(selectedFiles, File{ File: file, - Link: "", + Link: "", // no link yet }) } if len(selectedFiles) > len(info.Links) && info.Progress == 100 { @@ -315,12 +309,12 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { // chaotic file means RD will not output the desired file selection // e.g. even if we select just a single mkv, it will output a rar var isChaotic bool - selectedFiles, isChaotic = t.organizeChaos(info, selectedFiles) + selectedFiles, isChaotic = t.organizeChaos(&rdTorrent, selectedFiles) if isChaotic { t.log.Infof("Torrent %s %s is unfixable, it's always returning an unstreamable link, ignoring", info.ID, info.Name) t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash) } else { - if len(streamableFiles) > 1 { + if streamableCount > 1 { t.log.Infof("Torrent %s %s marked for repair", info.ID, info.Name) forRepair = true } else { @@ -328,22 +322,25 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash) } } - } else { + } else if len(selectedFiles) > 0 { // all links are still intact! good! for i, link := range info.Links { selectedFiles[i].Link = link } } - torrent.ForRepair = forRepair - torrent.SelectedFiles = selectedFiles - torrent.AccessKey = t.getName(info.Name, info.OriginalName) - - // update file cache - if len(selectedFiles) > 0 { - t.writeToFile(&torrent) + info.ForRepair = forRepair + torrent := Torrent{ + AccessKey: t.getName(info.Name, info.OriginalName), + SelectedFiles: selectedFiles, + Directories: t.getDirectories(info), + LatestAdded: info.Added, + InProgress: info.Progress != 100, + Instances: []realdebrid.TorrentInfo{*info}, + } + if len(selectedFiles) > 0 && torrentFromFile == nil { + t.writeToFile(info) // only when there are selected files, else it's useless } - t.log.Debugf("Got info for %s %s", torrent.ID, torrent.AccessKey) return &torrent } @@ -358,72 +355,7 @@ func (t *TorrentManager) getName(name, originalName string) string { } } -// mapToDirectories maps torrents to directories -func (t *TorrentManager) mapToDirectories() { - // Map torrents to directories - switch t.config.GetVersion() { - case "v1": - configV1 := t.config.(*config.ZurgConfigV1) - groupMap := configV1.GetGroupMap() - // for every group, iterate over every torrent - // and then sprinkle/distribute the torrents to the directories of the group - for group, directories := range groupMap { - counter := make(map[string]int) - for i := range t.torrents { - // don't process torrents that are already mapped if it is not the first run - alreadyMappedToGroup := false - for _, mappedGroup := range t.processedTorrents[t.torrents[i].AccessKey] { - if mappedGroup == group { - alreadyMappedToGroup = true - } - } - if alreadyMappedToGroup { - continue - } - - for _, directory := range directories { - var filenames []string - for _, file := range t.torrents[i].SelectedFiles { - filenames = append(filenames, file.Path) - } - if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].AccessKey, filenames) { - found := false - // check if it is already mapped to this directory - for _, dir := range t.directoryMap[t.torrents[i].AccessKey] { - if dir == directory { - found = true - break // it is already mapped to this directory - } - } - if !found { - counter[directory]++ - t.mu.Lock() - t.directoryMap[t.torrents[i].AccessKey] = append(t.directoryMap[t.torrents[i].AccessKey], directory) - t.mu.Unlock() - break // we found a directory for this torrent, so we can stop looking for more - } - } - } - t.mu.Lock() - t.processedTorrents[t.torrents[i].AccessKey] = append(t.processedTorrents[t.torrents[i].AccessKey], group) - t.mu.Unlock() - } - sum := 0 - for _, count := range counter { - sum += count - } - if sum > 0 { - t.log.Infof("Group processing completed: %s %v total: %d", group, counter, sum) - } else { - t.log.Infof("No new additions to directory group %s", group) - } - } - default: - t.log.Error("Unknown config version") - } -} - -func (t *TorrentManager) getDirectories(torrent *Torrent) []string { +func (t *TorrentManager) getDirectories(torrent *realdebrid.TorrentInfo) []string { var ret []string // Map torrents to directories switch t.config.GetVersion() { @@ -435,10 +367,14 @@ func (t *TorrentManager) getDirectories(torrent *Torrent) []string { for _, directories := range groupMap { for _, directory := range directories { var filenames []string - for _, file := range torrent.SelectedFiles { + for _, file := range torrent.Files { + if file.Selected == 0 { + continue + } filenames = append(filenames, file.Path) } - if configV1.MeetsConditions(directory, torrent.ID, torrent.AccessKey, filenames) { + accessKey := t.getName(torrent.Name, torrent.OriginalName) + if configV1.MeetsConditions(directory, torrent.ID, accessKey, filenames) { ret = append(ret, directory) break // we found a directory for this torrent for this group, so we can stop looking for more } @@ -450,18 +386,7 @@ func (t *TorrentManager) getDirectories(torrent *Torrent) []string { return ret } -// getByID returns a torrent by its ID -func (t *TorrentManager) getByID(torrentID string) *Torrent { - for i := range t.torrents { - if t.torrents[i].ID == torrentID { - return &t.torrents[i] - } - } - return nil -} - -// writeToFile writes a torrent to a file -func (t *TorrentManager) writeToFile(torrent *Torrent) { +func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) { filePath := fmt.Sprintf("data/%s.bin", torrent.ID) file, err := os.Create(filePath) if err != nil { @@ -475,8 +400,7 @@ func (t *TorrentManager) writeToFile(torrent *Torrent) { dataEncoder.Encode(torrent) } -// readFromFile reads a torrent from a file -func (t *TorrentManager) readFromFile(torrentID string) *Torrent { +func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo { filePath := fmt.Sprintf("data/%s.bin", torrentID) fileInfo, err := os.Stat(filePath) if err != nil { @@ -493,7 +417,7 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent { } defer file.Close() - var torrent Torrent + var torrent realdebrid.TorrentInfo dataDecoder := gob.NewDecoder(file) err = dataDecoder.Decode(&torrent) if err != nil { @@ -505,185 +429,6 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent { return &torrent } -func (t *TorrentManager) repairAll() { - for _, torrent := range t.torrents { - if torrent.ForRepair { - t.log.Infof("There were less links than was expected on %s %s; fixing...", torrent.ID, torrent.AccessKey) - t.repair(torrent.ID, torrent.SelectedFiles, true) - } - if len(torrent.Links) == 0 && torrent.Progress == 100 { - // If the torrent has no links - // and already processing repair - // delete it! - t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.AccessKey) - t.rd.DeleteTorrent(torrent.ID) - } - } -} - -func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReinsertionFirst bool) { - torrent := t.getByID(torrentID) - if torrent == nil { - return - } - - // check if it is already "being" repaired - found := false - for _, hash := range t.inProgress { - if hash == torrent.Hash { - found = true - break - } - } - if found { - t.log.Infof("Repair in progress, skipping %s", torrentID) - return - } - - // check if it is already repaired - foundFiles := t.findAllDownloadedFilesFromHash(torrent.Hash) - var missingFiles []File - for _, sFile := range selectedFiles { - if sFile.Link == "" || sFile.Unavailable { - found := false - for _, fFile := range foundFiles { - // same file but different link, then yes it has been repaired - if sFile.Path == fFile.Path && sFile.Link != fFile.Link { - found = true - break - } - } - if !found { - missingFiles = append(missingFiles, sFile) - } - } - } - if len(missingFiles) == 0 { - t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.AccessKey) - return - } - - // then we repair it! - t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.AccessKey) - // check if we can still add more downloads - proceed := t.canCapacityHandle() - if !proceed { - t.log.Error("Cannot add more torrents, exiting") - return - } - - // first solution: add the same selection, maybe it can be fixed by reinsertion? - success := false - if tryReinsertionFirst { - success = t.reinsertTorrent(torrent, "", true) - } - if !success { - // if all the selected files are missing but there are other streamable files - var otherStreamableFileIDs []int - for _, file := range torrent.Files { - found := false - for _, selectedFile := range selectedFiles { - if selectedFile.ID == file.ID { - found = true - break - } - } - if !found && isStreamable(file.Path) { - otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID) - } - } - if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 { - // we will download 1 extra streamable file to force a redownload of the missing files - // or if there's only 1 missing file, we will download 1 more to prevent a rename - missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",") - t.log.Infof("Redownloading %d missing files", len(missingFiles)) - t.reinsertTorrent(torrent, missingFilesPlus1, false) - } else if len(selectedFiles) > 1 { - // if not, last resort: add only the missing files but do it in 2 batches - half := len(missingFiles) / 2 - missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",") - missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",") - if missingFiles1 != "" { - t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1)) - t.reinsertTorrent(torrent, missingFiles1, false) - } - if missingFiles2 != "" { - t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2)) - t.reinsertTorrent(torrent, missingFiles2, false) - } else { - t.log.Info("No other missing files left to reinsert") - } - } else { - t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.AccessKey) - t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash) - return - } - t.log.Info("Waiting for downloads to finish") - } -} - -func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool { - // if missingFiles is not provided, look for missing files - if missingFiles == "" { - t.log.Info("Redownloading whole torrent", torrent.AccessKey) - var selection string - for _, file := range torrent.SelectedFiles { - selection += fmt.Sprintf("%d,", file.ID) - } - if selection == "" { - return false - } - if len(selection) > 0 { - missingFiles = selection[:len(selection)-1] - } - } - - // redownload torrent - resp, err := t.rd.AddMagnetHash(torrent.Hash) - if err != nil { - t.log.Errorf("Cannot redownload torrent: %v", err) - return false - } - newTorrentID := resp.ID - err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles) - if err != nil { - t.log.Errorf("Cannot start redownloading: %v", err) - } - - if deleteIfFailed { - if err != nil { - t.rd.DeleteTorrent(newTorrentID) - return false - } - time.Sleep(1 * time.Second) - // see if the torrent is ready - info, err := t.rd.GetTorrentInfo(newTorrentID) - if err != nil { - t.log.Errorf("Cannot get info on redownloaded torrent: %v", err) - if deleteIfFailed { - t.rd.DeleteTorrent(newTorrentID) - } - return false - } - time.Sleep(1 * time.Second) - - if info.Progress != 100 { - t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress) - t.rd.DeleteTorrent(newTorrentID) - return false - } - - if len(info.Links) != len(torrent.SelectedFiles) { - t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles)) - t.rd.DeleteTorrent(newTorrentID) - return false - } - t.log.Info("Redownload successful, deleting old torrent") - t.rd.DeleteTorrent(torrent.ID) - } - return true -} - func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles []File) ([]File, bool) { type Result struct { Response *realdebrid.UnrestrictResponse @@ -744,43 +489,235 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [ return selectedFiles, isChaotic } -func (t *TorrentManager) canCapacityHandle() bool { - // max waiting time is 45 minutes - const maxRetries = 50 - const baseDelay = 1 * time.Second - const maxDelay = 60 * time.Second - retryCount := 0 - for { - count, err := t.rd.GetActiveTorrentCount() - if err != nil { - t.log.Errorf("Cannot get active downloads count: %v", err) - if retryCount >= maxRetries { - t.log.Error("Max retries reached. Exiting.") - return false - } - delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay - if delay > maxDelay { - delay = maxDelay - } - time.Sleep(delay) - retryCount++ - continue - } +// HideTheFile marks a file as deleted +// func (t *TorrentManager) HideTheFile(torrent *Torrent, file *File) { +// file.Unavailable = true +// t.repair(torrent, false) +// } - if count.DownloadingCount < count.MaxNumberOfTorrents { - t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount) - return true - } +// func (t *TorrentManager) repairAll() { +// for _, torrent := range t.torrentMap { +// // do not repair if: +// // in progress +// hasInProgress := false +// for _, info := range torrent.Instances { +// if info.Progress != 100 { +// hasInProgress = true +// break +// } +// } +// if hasInProgress { +// continue +// } +// // already repaired based on other instances +// var missingFiles []File +// for _, file := range torrent.SelectedFiles { +// if file.Link == "" || file.Unavailable { +// missingFiles = append(missingFiles, file) +// } +// } +// for _, sFile := range selectedFiles { +// if sFile.Link == "" || sFile.Unavailable { +// found := false +// for _, fFile := range foundFiles { +// // same file but different link, then yes it has been repaired +// if sFile.Path == fFile.Path && sFile.Link != fFile.Link { +// found = true +// break +// } +// } +// if !found { +// missingFiles = append(missingFiles, sFile) +// } +// } +// } +// if len(missingFiles) == 0 { +// t.log.Infof("Torrent id=%s is already repaired", info.ID) +// return +// } - if retryCount >= maxRetries { - t.log.Error("Max retries reached, exiting") - return false - } - delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay - if delay > maxDelay { - delay = maxDelay - } - time.Sleep(delay) - retryCount++ - } -} +// for _, info := range torrent.Instances { +// if info.Progress != 100 { +// continue +// } +// if info.ForRepair { +// t.log.Infof("There were less links than was expected on %s %s; fixing...", info.ID, info.Name) +// t.repair(&info, true) +// break // only repair the first one for repair and then move on +// } +// if len(info.Links) == 0 && info.Progress == 100 { +// // If the torrent has no links +// // and already processing repair +// // delete it! +// t.log.Infof("Deleting broken torrent id=%s as it doesn't contain any files", info.ID) +// t.rd.DeleteTorrent(info.ID) +// } +// } +// } +// } + +// func (t *TorrentManager) repair(info *realdebrid.TorrentInfo, tryReinsertionFirst bool) { +// // then we repair it! +// t.log.Infof("Repairing torrent id=%s", info.ID) +// // check if we can still add more downloads +// proceed := t.canCapacityHandle() +// if !proceed { +// t.log.Error("Cannot add more torrents, exiting") +// return +// } + +// // first solution: add the same selection, maybe it can be fixed by reinsertion? +// success := false +// if tryReinsertionFirst { +// success = t.reinsertTorrent(info, "", true) +// } +// if !success { +// // if all the selected files are missing but there are other streamable files +// var otherStreamableFileIDs []int +// for _, file := range info.Files { +// found := false +// for _, selectedFile := range selectedFiles { +// if selectedFile.ID == file.ID { +// found = true +// break +// } +// } +// if !found && isStreamable(file.Path) { +// otherStreamableFileIDs = append(otherStreamableFileIDs, file.ID) +// } +// } +// if (len(missingFiles) == len(selectedFiles) || len(missingFiles) == 1) && len(otherStreamableFileIDs) > 0 { +// // we will download 1 extra streamable file to force a redownload of the missing files +// // or if there's only 1 missing file, we will download 1 more to prevent a rename +// missingFilesPlus1 := strings.Join(getFileIDs(missingFiles), ",") +// t.log.Infof("Redownloading %d missing files", len(missingFiles)) +// t.reinsertTorrent(info, missingFilesPlus1, false) +// } else if len(selectedFiles) > 1 { +// // if not, last resort: add only the missing files but do it in 2 batches +// half := len(missingFiles) / 2 +// missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",") +// missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",") +// if missingFiles1 != "" { +// t.log.Infof("Redownloading %d missing files; batch 1 of 2", len(missingFiles1)) +// t.reinsertTorrent(info, missingFiles1, false) +// } +// if missingFiles2 != "" { +// t.log.Infof("Redownloading %d missing files; batch 2 of 2", len(missingFiles2)) +// t.reinsertTorrent(info, missingFiles2, false) +// } else { +// t.log.Info("No other missing files left to reinsert") +// } +// } else { +// t.log.Infof("Torrent id=%s is unfixable as the only link cached in RD is already broken", info.ID) +// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash) +// return +// } +// t.log.Info("Waiting for downloads to finish") +// } +// } + +// func (t *TorrentManager) reinsertTorrent(torrent *realdebrid.TorrentInfo, missingFiles string, deleteIfFailed bool) bool { +// // if missingFiles is not provided, look for missing files +// if missingFiles == "" { +// var tmpSelection string +// for _, file := range torrent.Files { +// if file.Selected == 0 { +// continue +// } +// tmpSelection += fmt.Sprintf("%d,", file.ID) +// } +// if tmpSelection == "" { +// return false +// } +// if len(tmpSelection) > 0 { +// missingFiles = tmpSelection[:len(tmpSelection)-1] +// } +// } + +// // redownload torrent +// resp, err := t.rd.AddMagnetHash(torrent.Hash) +// if err != nil { +// t.log.Errorf("Cannot redownload torrent: %v", err) +// return false +// } +// newTorrentID := resp.ID +// err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles) +// if err != nil { +// t.log.Errorf("Cannot start redownloading: %v", err) +// } + +// if deleteIfFailed { +// if err != nil { +// t.rd.DeleteTorrent(newTorrentID) +// return false +// } +// time.Sleep(1 * time.Second) +// // see if the torrent is ready +// info, err := t.rd.GetTorrentInfo(newTorrentID) +// if err != nil { +// t.log.Errorf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err) +// if deleteIfFailed { +// t.rd.DeleteTorrent(newTorrentID) +// } +// return false +// } +// time.Sleep(1 * time.Second) + +// if info.Progress != 100 { +// t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion, currently %d%%", info.ID, info.Progress) +// t.rd.DeleteTorrent(newTorrentID) +// return false +// } + +// missingCount := len(strings.Split(missingFiles, ",")) +// if len(info.Links) != missingCount { +// t.log.Infof("It didn't fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount) +// t.rd.DeleteTorrent(newTorrentID) +// return false +// } +// t.log.Infof("Redownload successful id=%s, deleting old torrent id=%s", newTorrentID, torrent.ID) +// t.rd.DeleteTorrent(torrent.ID) +// } +// return true +// } + +// func (t *TorrentManager) canCapacityHandle() bool { +// // max waiting time is 45 minutes +// const maxRetries = 50 +// const baseDelay = 1 * time.Second +// const maxDelay = 60 * time.Second +// retryCount := 0 +// for { +// count, err := t.rd.GetActiveTorrentCount() +// if err != nil { +// t.log.Errorf("Cannot get active downloads count: %v", err) +// if retryCount >= maxRetries { +// t.log.Error("Max retries reached. Exiting.") +// return false +// } +// delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay +// if delay > maxDelay { +// delay = maxDelay +// } +// time.Sleep(delay) +// retryCount++ +// continue +// } + +// if count.DownloadingCount < count.MaxNumberOfTorrents { +// t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount) +// return true +// } + +// if retryCount >= maxRetries { +// t.log.Error("Max retries reached, exiting") +// return false +// } +// delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay +// if delay > maxDelay { +// delay = maxDelay +// } +// time.Sleep(delay) +// retryCount++ +// } +// } diff --git a/internal/torrent/types.go b/internal/torrent/types.go index b5dfb42..a1e4641 100644 --- a/internal/torrent/types.go +++ b/internal/torrent/types.go @@ -5,11 +5,13 @@ import ( ) type Torrent struct { - AccessKey string - Version string - realdebrid.Torrent + AccessKey string SelectedFiles []File - ForRepair bool + Directories []string + LatestAdded string + InProgress bool + + Instances []realdebrid.TorrentInfo } type File struct { diff --git a/internal/universal/get.go b/internal/universal/get.go index d7524ba..4b5f95a 100644 --- a/internal/universal/get.go +++ b/internal/universal/get.go @@ -35,9 +35,9 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent // If there are less than 3 segments, return an error or adjust as needed if len(segments) <= 3 { if isDav { - dav.HandlePropfindRequest(w, r, t, c, cache) + dav.HandlePropfindRequest(w, r, t, c) } else { - intHttp.HandleDirectoryListing(w, r, t, c, cache) + intHttp.HandleDirectoryListing(w, r, t, c) } return } @@ -57,7 +57,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent return } - torrent, file := getFile(torrents, filename) + _, file := getFile(torrents, filename) if file == nil { log.Errorf("Cannot find file from path %s", requestPath) http.Error(w, "File not found", http.StatusNotFound) @@ -75,7 +75,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torrent if resp == nil { if !file.Unavailable { log.Errorf("Cannot unrestrict file %s %s", filename, link) - t.HideTheFile(torrent, file) + // t.HideTheFile(torrent, file) } streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log) return diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index f808f64..6eecce7 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -135,7 +135,7 @@ func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) { return allTorrents, totalCount, nil } -func (rd *RealDebrid) GetTorrentInfo(id string) (*Torrent, error) { +func (rd *RealDebrid) GetTorrentInfo(id string) (*TorrentInfo, error) { url := "https://api.real-debrid.com/rest/1.0/torrents/info/" + id req, err := http.NewRequest("GET", url, nil) @@ -162,7 +162,7 @@ func (rd *RealDebrid) GetTorrentInfo(id string) (*Torrent, error) { return nil, fmt.Errorf("HTTP error: %s", resp.Status) } - var response Torrent + var response TorrentInfo err = json.Unmarshal(body, &response) if err != nil { rd.log.Errorf("Error when : %v", err) @@ -314,8 +314,7 @@ func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) { req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - client := &http.Client{} - resp, err := client.Do(req) + resp, err := rd.client.Do(req) if err != nil { rd.log.Errorf("Error when executing the unrestrict link request: %v", err) return nil, err diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go index 7f441b2..d0ce0f5 100644 --- a/pkg/realdebrid/types.go +++ b/pkg/realdebrid/types.go @@ -20,16 +20,28 @@ type UnrestrictResponse struct { } type Torrent struct { + ID string `json:"id"` + Name string `json:"filename"` + Hash string `json:"hash"` + Progress int `json:"-"` + Added string `json:"added"` + Bytes int64 `json:"bytes"` + Links []string `json:"links"` +} + +type TorrentInfo struct { ID string `json:"id"` Name string `json:"filename"` - OriginalName string `json:"original_filename"` Hash string `json:"hash"` Progress int `json:"-"` Added string `json:"added"` Bytes int64 `json:"bytes"` - OriginalBytes int64 `json:"original_bytes"` Links []string `json:"links"` - Files []File `json:"files,omitempty"` + OriginalName string `json:"original_filename"` // from info + OriginalBytes int64 `json:"original_bytes"` // from info + Files []File `json:"files"` // from info + ForRepair bool `json:"-"` + Version string `json:"-"` } func (t *Torrent) UnmarshalJSON(data []byte) error {