From bd72dc45400644ab57e6af30595dd2213912c7d4 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Wed, 18 Oct 2023 02:06:01 +0200 Subject: [PATCH] Everything is in memory now without libs --- internal/dav/getfile.go | 2 +- internal/dav/response.go | 67 ++++------------- internal/dav/util.go | 5 +- internal/torrent/cache.go | 1 - internal/torrent/manager.go | 141 +++++++++++++++++++++++++----------- internal/torrent/types.go | 13 ++++ 6 files changed, 128 insertions(+), 101 deletions(-) delete mode 100644 internal/torrent/cache.go create mode 100644 internal/torrent/types.go diff --git a/internal/dav/getfile.go b/internal/dav/getfile.go index 18bc093..a69095d 100644 --- a/internal/dav/getfile.go +++ b/internal/dav/getfile.go @@ -79,7 +79,7 @@ func extractIDFromFilename(filename string) (string, string) { } // findLinkByFragment finds a link by a fragment, it might be wrong -func findLinkByFragment(torrents []realdebrid.Torrent, fragment string) string { +func findLinkByFragment(torrents []torrent.Torrent, fragment string) string { for _, torrent := range torrents { for _, link := range torrent.Links { if strings.HasPrefix(link, fmt.Sprintf("https://real-debrid.com/d/%s", fragment)) { diff --git a/internal/dav/response.go b/internal/dav/response.go index d7072be..bef863a 100644 --- a/internal/dav/response.go +++ b/internal/dav/response.go @@ -1,16 +1,14 @@ package dav import ( - "fmt" "path/filepath" "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/dav" - "github.com/debridmediamanager.com/zurg/pkg/realdebrid" ) // createMultiTorrentResponse creates a WebDAV response for a list of torrents -func createMultiTorrentResponse(torrents []realdebrid.Torrent) (*dav.MultiStatus, error) { +func createMultiTorrentResponse(torrents []torrent.Torrent) (*dav.MultiStatus, error) { var responses []dav.Response responses = append(responses, dav.Directory("/torrents")) @@ -36,7 +34,7 @@ func createMultiTorrentResponse(torrents []realdebrid.Torrent) (*dav.MultiStatus } // createTorrentResponse creates a WebDAV response for torrents with the same name -func createCombinedTorrentResponse(torrents []realdebrid.Torrent, t *torrent.TorrentManager) (*dav.MultiStatus, error) { +func createCombinedTorrentResponse(torrents []torrent.Torrent, t *torrent.TorrentManager) (*dav.MultiStatus, error) { var responses []dav.Response // initial response is the directory itself currentPath := filepath.Join("/torrents", torrents[0].Filename) @@ -51,57 +49,20 @@ func createCombinedTorrentResponse(torrents []realdebrid.Torrent, t *torrent.Tor if info == nil { continue } - - var selectedFiles []realdebrid.File - for _, file := range info.Files { - if file.Selected == 0 { + for _, file := range info.SelectedFiles { + filename := filepath.Base(file.Path) + if _, exists := seen[filename]; exists { continue } - selectedFiles = append(selectedFiles, file) - } - - if len(selectedFiles) != len(info.Links) { - fmt.Println("Links and files do not match", info.Filename) - // TODO: Add auto-healing for this - // for _, link := range info.Links { - // unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { - // return realdebrid.UnrestrictCheck(os.Getenv("RD_TOKEN"), link) - // } - // resp := realdebrid.RetryUntilOk(unrestrictFn) - // if resp == nil { - // continue - // } else { - // if _, exists := seen[resp.Filename]; exists { - // continue - // } - // seen[resp.Filename] = true - // filePath := filepath.Join(currentPath, resp.Filename) - // torrentResponses = append(torrentResponses, - // dav.File( - // filePath, - // resp.Filesize, - // info.Added, - // resp.Link, - // ), - // ) - // } - // } - } else { - for _, file := range selectedFiles { - filename := filepath.Base(file.Path) - if _, exists := seen[filename]; exists { - continue - } - seen[filename] = true - filePath := filepath.Join(currentPath, filename) - torrentResponses = append(torrentResponses, dav.File( - filePath, - file.Bytes, - convertDate(info.Added), - info.Links[idx], - )) - idx++ - } + seen[filename] = true + filePath := filepath.Join(currentPath, filename) + torrentResponses = append(torrentResponses, dav.File( + filePath, + file.Bytes, + convertDate(info.Added), + info.Links[idx], + )) + idx++ } } responses = append(responses, torrentResponses...) diff --git a/internal/dav/util.go b/internal/dav/util.go index d0dfe0b..bd5f7ed 100644 --- a/internal/dav/util.go +++ b/internal/dav/util.go @@ -5,7 +5,6 @@ import ( "time" "github.com/debridmediamanager.com/zurg/internal/torrent" - "github.com/debridmediamanager.com/zurg/pkg/realdebrid" ) // convertDate converts a date from RFC3339 to RFC1123 @@ -19,8 +18,8 @@ func convertDate(input string) string { } // findAllTorrentsWithName finds all torrents with a given name -func findAllTorrentsWithName(t *torrent.TorrentManager, filename string) []realdebrid.Torrent { - var matchingTorrents []realdebrid.Torrent +func findAllTorrentsWithName(t *torrent.TorrentManager, filename string) []torrent.Torrent { + var matchingTorrents []torrent.Torrent torrents := t.GetAll() for _, torrent := range torrents { diff --git a/internal/torrent/cache.go b/internal/torrent/cache.go deleted file mode 100644 index 10cbafc..0000000 --- a/internal/torrent/cache.go +++ /dev/null @@ -1 +0,0 @@ -package torrent diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index ab5386b..8caffb7 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -2,39 +2,30 @@ package torrent import ( "log" + "strings" + "sync" "github.com/debridmediamanager.com/zurg/pkg/realdebrid" - "github.com/dgraph-io/ristretto" ) type TorrentManager struct { token string - cache *ristretto.Cache + torrents []Torrent workerPool chan bool } // NewTorrentManager creates a new torrent manager // it will fetch all torrents and their info in the background -// and cache them +// and store them in-memory func NewTorrentManager(token string) *TorrentManager { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e7, // number of keys to track frequency of (10M). - MaxCost: 1 << 30, // maximum cost of cache (1GB). - BufferItems: 64, // number of keys per Get buffer. - }) - if err != nil { - panic(err) - } - handler := &TorrentManager{ token: token, - cache: cache, workerPool: make(chan bool, 10), } - torrents := handler.getAll() + handler.torrents = handler.getAll() - for _, torrent := range torrents { + for _, torrent := range handler.torrents { go func(id string) { handler.workerPool <- true handler.getInfo(id) @@ -46,48 +37,112 @@ func NewTorrentManager(token string) *TorrentManager { return handler } -func (t *TorrentManager) getAll() []realdebrid.Torrent { - cacheKey := "t:all" +func (t *TorrentManager) getAll() []Torrent { torrents, err := realdebrid.GetTorrents(t.token) if err != nil { log.Printf("Cannot get torrents: %v\n", err.Error()) return nil } - t.cache.Set(cacheKey, torrents, 0) - return torrents -} - -func (t *TorrentManager) GetAll() []realdebrid.Torrent { - cacheKey := "t:all" - if data, found := t.cache.Get(cacheKey); found { - if cachedTorrents, ok := data.([]realdebrid.Torrent); ok { - return cachedTorrents - } else { - t.cache.Del(cacheKey) - } + var torrentsV2 []Torrent + for _, torrent := range torrents { + torrentsV2 = append(torrentsV2, Torrent{ + Torrent: torrent, + SelectedFiles: nil, + }) } - return t.getAll() + return torrentsV2 } -func (t *TorrentManager) getInfo(torrentID string) *realdebrid.Torrent { - cacheKey := "t:" + torrentID +func (t *TorrentManager) GetAll() []Torrent { + return t.torrents +} + +func (t *TorrentManager) getInfo(torrentID string) *Torrent { info, err := realdebrid.GetTorrentInfo(t.token, torrentID) if err != nil { log.Printf("Cannot get info: %v\n", err.Error()) return nil } - t.cache.Set(cacheKey, info, 0) - return info -} + var selectedFiles []File + for _, file := range info.Files { + if file.Selected == 0 { + continue + } + selectedFiles = append(selectedFiles, File{ + File: file, + Link: "", + }) + } + if len(selectedFiles) != len(info.Links) { + type Result struct { + Filename string + Link string + } -func (t *TorrentManager) GetInfo(torrentID string) *realdebrid.Torrent { - cacheKey := "t:" + torrentID - if data, found := t.cache.Get(cacheKey); found { - if torrent, ok := data.(*realdebrid.Torrent); ok { - return torrent - } else { - t.cache.Del(cacheKey) + resultsChan := make(chan Result, len(info.Links)) + var wg sync.WaitGroup + + // Limit concurrency + sem := make(chan struct{}, 10) // e.g., 10 concurrent requests + + for _, link := range info.Links { + wg.Add(1) + sem <- struct{}{} // Acquire semaphore + go func(lnk string) { + defer wg.Done() + defer func() { <-sem }() // Release semaphore + + unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) { + return realdebrid.UnrestrictLink(t.token, lnk) + } + resp := realdebrid.RetryUntilOk(unrestrictFn) + if resp != nil { + resultsChan <- Result{Filename: resp.Filename, Link: resp.Link} + } + }(link) + } + + go func() { + wg.Wait() + close(resultsChan) + }() + + for result := range resultsChan { + for i := range selectedFiles { + if strings.HasSuffix(selectedFiles[i].Path, result.Filename) { + selectedFiles[i].Link = result.Link + } + } + } + } else { + for i, link := range info.Links { + selectedFiles[i].Link = link } } - return t.getInfo(torrentID) + torrent := t.getByID(torrentID) + if torrent != nil { + torrent.SelectedFiles = selectedFiles + } + log.Println("Fetched info for", info.Filename) + return torrent +} + +func (t *TorrentManager) GetInfo(torrentID string) *Torrent { + for _, torrent := range t.torrents { + if torrent.ID == torrentID { + if torrent.SelectedFiles != nil { + return t.getInfo(torrentID) + } + } + } + return nil +} + +func (t *TorrentManager) getByID(torrentID string) *Torrent { + for _, torrent := range t.torrents { + if torrent.ID == torrentID { + return &torrent + } + } + return nil } diff --git a/internal/torrent/types.go b/internal/torrent/types.go new file mode 100644 index 0000000..a56dd6a --- /dev/null +++ b/internal/torrent/types.go @@ -0,0 +1,13 @@ +package torrent + +import "github.com/debridmediamanager.com/zurg/pkg/realdebrid" + +type Torrent struct { + realdebrid.Torrent + SelectedFiles []File +} + +type File struct { + realdebrid.File + Link string +}