diff --git a/internal/handlers/home.go b/internal/handlers/home.go index 9fd4272..37271d7 100644 --- a/internal/handlers/home.go +++ b/internal/handlers/home.go @@ -56,7 +56,7 @@ func (zr *Handlers) generateResponse(req *http.Request) (*RootResponse, error) { var mem runtime.MemStats runtime.ReadMemStats(&mem) - allTorrents, _ := zr.torMgr.DirectoryMap.Get(config.ALL_TORRENTS) + torrents, _ := zr.torMgr.DirectoryMap.Get(config.ALL_TORRENTS) repairQueueStr := "" if zr.torMgr.RepairQueue == nil { @@ -108,7 +108,7 @@ func (zr *Handlers) generateResponse(req *http.Request) (*RootResponse, error) { Logs: fmt.Sprintf("//%s/logs/", req.Host), UserInfo: userInfo, TrafficServedPerAPI: trafficFromAPI, - LibrarySize: allTorrents.Count(), + LibrarySize: torrents.Count(), TorrentsToRepair: repairQueueStr, MemAlloc: bToMb(mem.Alloc), TotalAlloc: bToMb(mem.TotalAlloc), diff --git a/internal/torrent/delete.go b/internal/torrent/delete.go index 2ba431b..1dbb12d 100644 --- a/internal/torrent/delete.go +++ b/internal/torrent/delete.go @@ -22,8 +22,8 @@ func (t *TorrentManager) CheckDeletedStatus(torrent *Torrent) bool { } func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) { - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) - if torrent, ok := allTorrents.Get(accessKey); ok { + torrents, _ := t.DirectoryMap.Get(INT_ALL) + if torrent, ok := torrents.Get(accessKey); ok { if deleteInRD { torrent.DownloadedIDs.Clone().Each(func(torrentID string) bool { t.DeleteByID(torrentID) @@ -40,7 +40,7 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) { torrents.Remove(accessKey) }) - allTorrents.Remove(accessKey) + torrents.Remove(accessKey) } func (t *TorrentManager) DeleteByID(torrentID string) { diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index dfb6901..05f8678 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -105,12 +105,12 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w defer wg.Done() // initial load of existing *.zurgtorrent files - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) + torrents, _ := t.DirectoryMap.Get(INT_ALL) t.getTorrentFiles("data").Each(func(filePath string) bool { torrent := t.readTorrentFromFile(filePath) if torrent != nil { accessKey := t.GetKey(torrent) - allTorrents.Set(accessKey, torrent) + torrents.Set(accessKey, torrent) t.assignDirectory(torrent, false, false) } return false @@ -498,12 +498,12 @@ func (t *TorrentManager) StartDumpJob() { } func (t *TorrentManager) analyzeAllTorrents() { - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) - totalCount := allTorrents.Count() + torrents, _ := t.DirectoryMap.Get(INT_ALL) + totalCount := torrents.Count() t.log.Infof("Applying media info details to all %d torrents", totalCount) idx := 0 skipTheRest := false - allTorrents.IterCb(func(_ string, torrent *Torrent) { + torrents.IterCb(func(_ string, torrent *Torrent) { if skipTheRest { return } diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go index 280727a..4b4bc43 100644 --- a/internal/torrent/refresh.go +++ b/internal/torrent/refresh.go @@ -29,8 +29,8 @@ func (t *TorrentManager) refreshTorrents(initialRun bool) { var wg sync.WaitGroup var mergeChan = make(chan *Torrent, len(instances)) - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) - oldKeys := mapset.NewSet[string](allTorrents.Keys()...) + torrents, _ := t.DirectoryMap.Get(INT_ALL) + oldKeys := mapset.NewSet[string](torrents.Keys()...) freshIDs := mapset.NewSet[string]() freshAccessKeys := mapset.NewSet[string]() @@ -60,9 +60,9 @@ func (t *TorrentManager) refreshTorrents(initialRun bool) { freshAccessKeys.Add(accessKey) var forMerging *Torrent - mainTorrent, exists := allTorrents.Get(accessKey) + mainTorrent, exists := torrents.Get(accessKey) if !exists { - allTorrents.Set(accessKey, torrent) + torrents.Set(accessKey, torrent) t.writeTorrentToFile(torrent) t.assignDirectory(torrent, !initialRun, true) } else if !mainTorrent.DownloadedIDs.ContainsOne(tInfo.ID) { @@ -83,13 +83,13 @@ func (t *TorrentManager) refreshTorrents(initialRun bool) { continue } accessKey := t.GetKey(torrent) - existing, ok := allTorrents.Get(accessKey) + existing, ok := torrents.Get(accessKey) if !ok { t.log.Warnf("Cannot merge %s", accessKey) continue } mainTorrent := t.mergeTorrents(existing, torrent) - allTorrents.Set(accessKey, mainTorrent) + torrents.Set(accessKey, mainTorrent) t.writeTorrentToFile(mainTorrent) t.assignDirectory(mainTorrent, !initialRun, true) } @@ -100,7 +100,7 @@ func (t *TorrentManager) refreshTorrents(initialRun bool) { return } freshAccessKeys.Difference(oldKeys).Each(func(accessKey string) bool { - torrent, _ := allTorrents.Get(accessKey) + torrent, _ := torrents.Get(accessKey) t.applyMediaInfoDetails(torrent) return false }) @@ -112,7 +112,7 @@ func (t *TorrentManager) refreshTorrents(initialRun bool) { return false }) - t.log.Infof("Compiled into %d unique torrents", allTorrents.Count()) + t.log.Infof("Compiled into %d unique torrents", torrents.Count()) t.workerPool.Submit(func() { t.OnceDoneBin.Clone().Each(func(entry string) bool { @@ -140,7 +140,7 @@ func (t *TorrentManager) refreshTorrents(initialRun bool) { // cleans up DownloadedIDs field of all torrents t.workerPool.Submit(func() { - allTorrents.IterCb(func(accessKey string, torrent *Torrent) { + torrents.IterCb(func(accessKey string, torrent *Torrent) { deletedIDs := torrent.DownloadedIDs.Difference(freshIDs) if deletedIDs.Cardinality() > 0 { deletedIDs.Each(func(id string) bool { @@ -397,8 +397,8 @@ func (t *TorrentManager) assignDirectory(tor *Torrent, triggerHook bool, outputL for _, directories := range configV1.GetGroupMap() { for _, directory := range directories { if t.Config.MeetsConditions(directory, t.GetKey(tor), tor.ComputeTotalSize(), torrentIDs, filenames, fileSizes, mediaInfos) { - listing, _ := t.DirectoryMap.Get(directory) - listing.Set(accessKey, tor) + torrents, _ := t.DirectoryMap.Get(directory) + torrents.Set(accessKey, tor) if directory != config.ALL_TORRENTS { dirs = append(dirs, directory) diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index cdf0b38..2507698 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -92,19 +92,19 @@ func (t *TorrentManager) invokeRepair(torrent *Torrent) { } func (t *TorrentManager) executeRepairJob(torrent *Torrent) { - var haystack cmap.ConcurrentMap[string, *Torrent] + var torrents cmap.ConcurrentMap[string, *Torrent] if torrent == nil { - haystack, _ = t.DirectoryMap.Get(INT_ALL) + torrents, _ = t.DirectoryMap.Get(INT_ALL) } else { - haystack = cmap.New[*Torrent]() - haystack.Set("", torrent) + torrents = cmap.New[*Torrent]() + torrents.Set("", torrent) } // collect all torrents that need to be repaired asynchronously toRepair := mapset.NewSet[*Torrent]() var wg sync.WaitGroup - haystack.IterCb(func(_ string, torrent *Torrent) { + torrents.IterCb(func(_ string, torrent *Torrent) { wg.Add(1) // temp worker for finding broken torrents t.workerPool.Submit(func() { @@ -677,8 +677,8 @@ func (t *TorrentManager) ResetRepairState() { if !t.Config.EnableRepair() { return } - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) - allTorrents.IterCb(func(_ string, torrent *Torrent) { + torrents, _ := t.DirectoryMap.Get(INT_ALL) + torrents.IterCb(func(_ string, torrent *Torrent) { err := torrent.State.Event(context.Background(), "reset_repair") if err == nil { t.repairLog.Debugf("Repair state of torrent %s has been reset", t.GetKey(torrent)) diff --git a/internal/torrent/uncached.go b/internal/torrent/uncached.go index 597e506..f92cacc 100644 --- a/internal/torrent/uncached.go +++ b/internal/torrent/uncached.go @@ -8,7 +8,7 @@ import ( func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) { t.log.Debug("Checking if torrents are still cached") - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) + torrents, _ := t.DirectoryMap.Get(INT_ALL) var hashGroups []mapset.Set[string] const maxGroupSize = 399 @@ -16,7 +16,7 @@ func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) { currentGroup := mapset.NewSet[string]() hashGroups = append(hashGroups, currentGroup) - allTorrents.IterCb(func(_ string, torrent *Torrent) { + torrents.IterCb(func(_ string, torrent *Torrent) { if torrent.UnrepairableReason != "" { return } @@ -45,7 +45,7 @@ func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) { } var uncachedTorrents []*Torrent - allTorrents.IterCb(func(_ string, torrent *Torrent) { + torrents.IterCb(func(_ string, torrent *Torrent) { if torrent.UnrepairableReason != "" { return } diff --git a/pkg/http/client.go b/pkg/http/client.go index cbf8ca6..56642e5 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -201,6 +201,7 @@ func (r *HTTPClient) ensureReachableHost(req *http.Request) { if !strings.Contains(req.Host, ".download.real-debrid.") { return } + // skip CDN servers if req.Host[0] >= 'a' && req.Host[0] <= 'z' { return } @@ -216,16 +217,18 @@ func (r *HTTPClient) ensureReachableHost(req *http.Request) { } else if strings.HasSuffix(req.Host, ".cloud") { newHost = strings.Replace(req.Host, ".cloud", ".com", 1) } + // check if newHost is reachable if r.CheckIfHostIsReachable(newHost) { req.Host = newHost req.URL.Host = req.Host return } - + // just pick a random host req.Host = r.hosts[rand.Intn(len(r.hosts))] req.URL.Host = req.Host } +// CheckIfHostIsReachable checks if the given host is passed in the list of reachable hosts func (r *HTTPClient) CheckIfHostIsReachable(reqHost string) bool { for _, host := range r.hosts { if reqHost == host { diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 27a6dfc..bdd7c13 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -84,9 +84,11 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) { if expiry, ok := rd.verifiedLinks.Get(download.ID); ok && expiry > time.Now().Unix() { return download, nil } else if ok { + // if the link is expired, remove it from the verified links cache rd.verifiedLinks.Remove(download.ID) } + // check if the link is still valid (not in the cache or expired) err := rd.downloadClient.VerifyLink(download.Download) if utils.IsBytesLimitReached(err) { rd.TokenManager.SetTokenAsExpired(token, "bandwidth limit exceeded") @@ -95,6 +97,7 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) { rd.verifiedLinks.Set(download.ID, time.Now().Unix()+DOWNLOAD_LINK_EXPIRY) return download, nil } + // if verification failed, remove the link from the token map tokenMap.Remove(link) } @@ -419,8 +422,10 @@ func (rd *RealDebrid) DownloadFile(req *http.Request) (*http.Response, error) { // MonitorExpiredTokens is a permanent job for monitoring expired tokens if they are still expired func (rd *RealDebrid) MonitorExpiredTokens() { sleepPeriod := 1 * time.Minute + i := 0 rd.workerPool.Submit(func() { for { + i++ expiredTokens := rd.TokenManager.GetExpiredTokens() for _, token := range expiredTokens { tokenMap, _ := rd.UnrestrictMap.Get(token) @@ -432,7 +437,12 @@ func (rd *RealDebrid) MonitorExpiredTokens() { } err := rd.downloadClient.VerifyLink(download.Download) if err != nil { - skipAll = utils.IsBytesLimitReached(err) + if utils.IsBytesLimitReached(err) { + if i%15 == 0 { + rd.log.Debugf("Token %s is still expired", utils.MaskToken(token)) + } + skipAll = true + } return } stillExpired = false