From 54230c9eaa4bd039fd81b196a905c27db065593c Mon Sep 17 00:00:00 2001 From: Ben Adrian Sarmiento Date: Sun, 25 Aug 2024 13:30:40 +0200 Subject: [PATCH] Set verified links cache time to 3 mins --- internal/torrent/manager.go | 6 ++-- internal/torrent/repair.go | 4 +-- internal/universal/downloader.go | 4 +-- pkg/realdebrid/api.go | 50 +++++++++++++++++--------------- 4 files changed, 34 insertions(+), 30 deletions(-) diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 1a22712..47e4222 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -378,10 +378,10 @@ func (t *TorrentManager) deleteInfoFile(torrentID string) { func (t *TorrentManager) mountNewDownloads() { token := t.Config.GetToken() - tokenMap, _ := t.rd.UnrestrictMap.Get(token) + unrestrictCache, _ := t.rd.UnrestrictCache.Get(token) // clear maps - tokenMap.Clear() + unrestrictCache.Clear() t.DownloadMap.Clear() downloads := t.rd.GetDownloads() @@ -391,7 +391,7 @@ func (t *TorrentManager) mountNewDownloads() { if strings.HasPrefix(downloads[i].Link, "https://real-debrid.com/d/") { downloads[i].Link = downloads[i].Link[0:39] - tokenMap.Set(downloads[i].Link, &downloads[i]) + unrestrictCache.Set(downloads[i].Link, &downloads[i]) continue } diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index f928b05..15e67f9 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -19,7 +19,7 @@ const ( EXPIRED_LINK_TOLERANCE_HOURS = 24 ) -// StartRepairJob is a permanent job that runs every periodically to repair broken torrents +// StartRepairJob is a permanent job that runs periodically to repair broken torrents func (t *TorrentManager) StartRepairJob() { if !t.Config.EnableRepair() { t.repairLog.Warn("Repair is disabled, skipping repair job") @@ -29,7 +29,6 @@ func (t *TorrentManager) StartRepairJob() { t.RepairQueue = mapset.NewSet[*Torrent]() t.RepairAllTrigger = make(chan struct{}) - // periodic repair worker t.workerPool.Submit(func() { t.repairLog.Debug("Starting periodic repair job") repairTicker := time.NewTicker(time.Duration(t.Config.GetRepairEveryMins()) * time.Minute) @@ -48,6 +47,7 @@ func (t *TorrentManager) StartRepairJob() { } // EnqueueForRepair allows an on-demand repair to be initiated. +// if torrent is nil, all torrents will be repaired func (t *TorrentManager) EnqueueForRepair(torrent *Torrent) { if !t.Config.EnableRepair() { if torrent != nil { diff --git a/internal/universal/downloader.go b/internal/universal/downloader.go index bcfbce5..c0f341a 100644 --- a/internal/universal/downloader.go +++ b/internal/universal/downloader.go @@ -167,8 +167,8 @@ func (dl *Downloader) DownloadLink( } func (dl *Downloader) streamFileToResponse( - torrent *intTor.Torrent, - file *intTor.File, + torrent *intTor.Torrent, // can be nil if downloading a link + file *intTor.File, // can be nil if downloading a link unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index bdd7c13..4c83951 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -17,7 +17,7 @@ import ( ) type RealDebrid struct { - UnrestrictMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Download]] + UnrestrictCache cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Download]] TokenManager *DownloadTokenManager torrentsCache []Torrent verifiedLinks cmap.ConcurrentMap[string, int64] @@ -45,7 +45,7 @@ func NewRealDebrid(apiClient, } rd := &RealDebrid{ - UnrestrictMap: cmap.New[cmap.ConcurrentMap[string, *Download]](), + UnrestrictCache: cmap.New[cmap.ConcurrentMap[string, *Download]](), TokenManager: NewDownloadTokenManager(downloadTokens, log), torrentsCache: []Torrent{}, verifiedLinks: cmap.New[int64](), @@ -59,16 +59,17 @@ func NewRealDebrid(apiClient, } for _, token := range downloadTokens { - rd.UnrestrictMap.Set(token, cmap.New[*Download]()) + rd.UnrestrictCache.Set(token, cmap.New[*Download]()) } return rd } -const DOWNLOAD_LINK_EXPIRY = 60 * 60 * 1.5 // 1.5 hours +const DOWNLOAD_LINK_EXPIRY = 60 * 3 // 3 minutes func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) { for { + now := time.Now().Unix() token, err := rd.TokenManager.GetCurrentToken() if err != nil { // when all tokens are expired @@ -76,39 +77,40 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) { } // check if the link is already unrestricted - tokenMap, _ := rd.UnrestrictMap.Get(token) - if tokenMap.Has(link) { - download, _ := tokenMap.Get(link) + unrestrictCache, _ := rd.UnrestrictCache.Get(token) + if unrestrictCache.Has(link) { + download, _ := unrestrictCache.Get(link) - // check if the link is in the verified links cache - if expiry, ok := rd.verifiedLinks.Get(download.ID); ok && expiry > time.Now().Unix() { + // check if the link is in the verified links cache and not expired + if expiry, ok := rd.verifiedLinks.Get(download.ID); ok && expiry > now { return download, nil - } else if ok { - // if the link is expired, remove it from the verified links cache - rd.verifiedLinks.Remove(download.ID) } - // check if the link is still valid (not in the cache or expired) + // we need to re-verify the link + + rd.verifiedLinks.Remove(download.ID) err := rd.downloadClient.VerifyLink(download.Download) - if utils.IsBytesLimitReached(err) { + if err == nil { + // yes? then extend the expiry time? + rd.verifiedLinks.Set(download.ID, now+DOWNLOAD_LINK_EXPIRY) + return download, nil + } else if utils.IsBytesLimitReached(err) { rd.TokenManager.SetTokenAsExpired(token, "bandwidth limit exceeded") continue - } else if err == nil { - rd.verifiedLinks.Set(download.ID, time.Now().Unix()+DOWNLOAD_LINK_EXPIRY) - return download, nil } + // if verification failed, remove the link from the token map - tokenMap.Remove(link) + unrestrictCache.Remove(link) } download, err := rd.UnrestrictLinkWithToken(link, token) if err != nil { return nil, err } - download.Token = token - tokenMap.Set(link, download) + unrestrictCache.Set(link, download) + rd.verifiedLinks.Remove(download.ID) err = rd.downloadClient.VerifyLink(download.Download) if utils.IsBytesLimitReached(err) { rd.TokenManager.SetTokenAsExpired(token, "bandwidth limit exceeded") @@ -117,7 +119,7 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) { return nil, err } - rd.verifiedLinks.Set(download.ID, time.Now().Unix()+DOWNLOAD_LINK_EXPIRY) + rd.verifiedLinks.Set(download.ID, now+DOWNLOAD_LINK_EXPIRY) return download, err } @@ -162,6 +164,8 @@ func (rd *RealDebrid) UnrestrictLinkWithToken(link, token string) (*Download, er return nil, fmt.Errorf("undecodable response: %v", err) } + response.Token = token + // rd.log.Debugf("Unrestricted link %s into %s", link, response.Download) return &response, nil } @@ -428,10 +432,10 @@ func (rd *RealDebrid) MonitorExpiredTokens() { i++ expiredTokens := rd.TokenManager.GetExpiredTokens() for _, token := range expiredTokens { - tokenMap, _ := rd.UnrestrictMap.Get(token) + unrestrictCache, _ := rd.UnrestrictCache.Get(token) stillExpired := true skipAll := false - tokenMap.IterCb(func(key string, download *Download) { + unrestrictCache.IterCb(func(key string, download *Download) { if skipAll { return }