From d03b59bb2af97339b79462e9013c1a6543044f85 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Thu, 23 May 2024 19:29:16 +0200 Subject: [PATCH] Fixers --- internal/dav/infuse.go | 19 +- internal/dav/listing.go | 2 +- internal/http/listing.go | 2 +- internal/torrent/delete.go | 9 +- internal/torrent/manager.go | 5 +- internal/torrent/refresh.go | 349 +++++++++++++++++------------- internal/torrent/repair.go | 56 +++-- internal/torrent/torrent_types.go | 51 ++--- internal/torrent/uncached.go | 4 +- pkg/realdebrid/types.go | 4 - 10 files changed, 275 insertions(+), 226 deletions(-) diff --git a/internal/dav/infuse.go b/internal/dav/infuse.go index 4357851..b7e4f79 100644 --- a/internal/dav/infuse.go +++ b/internal/dav/infuse.go @@ -15,19 +15,24 @@ import ( func ServeRootDirectoryForInfuse(torMgr *torrent.TorrentManager) ([]byte, error) { var buf bytes.Buffer buf.WriteString("") + directories := torMgr.DirectoryMap.Keys() sort.Strings(directories) + for _, directory := range directories { if strings.HasPrefix(directory, "int__") { continue } buf.WriteString(dav.BaseDirectory(directory, "")) } + if torMgr.Config.EnableDownloadMount() { buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, "")) } + _, size := version.GetFile() buf.WriteString(dav.File(version.FILE, size, "")) + buf.WriteString("") return buf.Bytes(), nil } @@ -40,15 +45,18 @@ func ServeTorrentsListForInfuse(directory string, torMgr *torrent.TorrentManager var buf bytes.Buffer buf.WriteString("") + torrentNames := torrents.Keys() sort.Strings(torrentNames) + for _, torrentName := range torrentNames { tor, ok := torrents.Get(torrentName) - if !ok || tor.AllInProgress() { + if !ok { continue } buf.WriteString(dav.BaseDirectory(torMgr.GetKey(tor), tor.Added)) } + buf.WriteString("") return buf.Bytes(), nil } @@ -58,12 +66,14 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr if !ok { return nil, fmt.Errorf("cannot find directory %s", directory) } + tor, ok := torrents.Get(torrentName) if !ok { return nil, fmt.Errorf("cannot find torrent %s", torrentName) } dirCfg := torMgr.Config.(*config.ZurgConfigV1).GetDirectoryConfig(directory) + biggestFileSize := int64(0) if dirCfg.OnlyShowTheBiggestFile { biggestFileSize = tor.ComputeBiggestFileSize() @@ -71,8 +81,10 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr var buf bytes.Buffer buf.WriteString("") + filenames := tor.SelectedFiles.Keys() sort.Strings(filenames) + for _, filename := range filenames { file, _ := tor.SelectedFiles.Get(filename) if !file.State.Is("ok_file") { @@ -89,6 +101,7 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr } buf.WriteString(dav.File(filename, file.Bytes, tor.Added)) } + buf.WriteString("") return buf.Bytes(), nil } @@ -99,9 +112,12 @@ func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error) buf.WriteString("Enable download mount in config to use this feature") return buf.Bytes(), nil } + buf.WriteString("") + filenames := torMgr.DownloadMap.Keys() sort.Strings(filenames) + for _, filename := range filenames { download, ok := torMgr.DownloadMap.Get(filename) if !ok { @@ -109,6 +125,7 @@ func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error) } buf.WriteString(dav.File(download.Filename, download.Filesize, download.Generated)) } + buf.WriteString("") return buf.Bytes(), nil } diff --git a/internal/dav/listing.go b/internal/dav/listing.go index 9e3c25b..679c88c 100644 --- a/internal/dav/listing.go +++ b/internal/dav/listing.go @@ -47,7 +47,7 @@ func ServeTorrentsList(directory string, torMgr *torrent.TorrentManager) ([]byte sort.Strings(torrentNames) for _, torrentName := range torrentNames { tor, ok := torrents.Get(torrentName) - if !ok || tor.AllInProgress() { + if !ok { continue } buf.WriteString(dav.Directory(torMgr.GetKey(tor), tor.Added)) diff --git a/internal/http/listing.go b/internal/http/listing.go index 668f98c..c4bb809 100644 --- a/internal/http/listing.go +++ b/internal/http/listing.go @@ -45,7 +45,7 @@ func ServeTorrentsList(directory string, torMgr *torrent.TorrentManager) ([]byte sort.Strings(torrentNames) for _, torrentName := range torrentNames { tor, ok := torrents.Get(torrentName) - if !ok || tor.AllInProgress() { + if !ok { continue } buf.WriteString(fmt.Sprintf("
  • %s
    • ", filepath.Join(directory, url.PathEscape(torMgr.GetKey(tor))), torMgr.GetKey(tor))) diff --git a/internal/torrent/delete.go b/internal/torrent/delete.go index 3c08494..d77afa4 100644 --- a/internal/torrent/delete.go +++ b/internal/torrent/delete.go @@ -1,6 +1,9 @@ package torrent -import cmap "github.com/orcaman/concurrent-map/v2" +import ( + "github.com/debridmediamanager/zurg/pkg/realdebrid" + cmap "github.com/orcaman/concurrent-map/v2" +) // CheckDeletedStatus checks if all files in a torrent are marked as deleted, if so, it returns true func (t *TorrentManager) CheckDeletedStatus(torrent *Torrent) bool { @@ -24,11 +27,11 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) { if torrent, ok := allTorrents.Get(accessKey); ok { hash = torrent.Hash if deleteInRD { - for torrentID := range torrent.Components { + torrent.Components.IterCb(func(torrentID string, _ *realdebrid.TorrentInfo) { t.log.Debugf("Deleting torrent %s (id=%s) in RD", accessKey, torrentID) t.api.DeleteTorrent(torrentID) t.deleteInfoFile(torrentID) - } + }) } } t.log.Infof("Removing torrent %s from zurg database (not real-debrid)", accessKey) diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 192de02..9d37a6b 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -168,7 +168,7 @@ func (t *TorrentManager) writeTorrentToFile(torrent *Torrent) { return } - t.log.Debugf("Saved torrent %s to file", torrent.Hash) + t.log.Debugf("Saved torrent %s (hash=%s) to file", t.GetKey(torrent), torrent.Hash) } func (t *TorrentManager) readTorrentFromFile(hash string) *Torrent { @@ -189,9 +189,6 @@ func (t *TorrentManager) readTorrentFromFile(hash string) *Torrent { if err := json.Unmarshal(jsonData, &torrent); err != nil { return nil } - if len(torrent.Components) == 0 { - t.log.Fatal("Torrent has no downloaded or in progress ids") - } if torrent.Version != t.requiredVersion { return nil } diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go index ef8e7a4..9598060 100644 --- a/internal/torrent/refresh.go +++ b/internal/torrent/refresh.go @@ -22,99 +22,112 @@ func (t *TorrentManager) refreshTorrents() []string { return nil } t.log.Infof("Fetched %d torrents", len(instances)) - torChan := make(chan *Torrent, len(instances)) - var wg sync.WaitGroup + var wg sync.WaitGroup + var mergeChan = make(chan *Torrent, len(instances)) + + updatedPaths := mapset.NewSet[string]() + + allTorrents, _ := t.DirectoryMap.Get(INT_ALL) + + freshHashes := mapset.NewSet[string]() + freshIDs := mapset.NewSet[string]() + freshAccessKeys := mapset.NewSet[string]() + + counter := 0 for i := range instances { - if t.trashBin.Contains(instances[i].ID) { - t.api.DeleteTorrent(instances[i].ID) - t.log.Infof("Skipping trashed torrent %s", instances[i].Name) - torChan <- nil - } - idx := i wg.Add(1) + idx := i _ = t.workerPool.Submit(func() { defer wg.Done() - torChan <- t.getMoreInfo(instances[idx]) + if t.trashBin.Contains(instances[idx].ID) { + t.api.DeleteTorrent(instances[idx].ID) + t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID) + counter++ + mergeChan <- nil + return + } + + if instances[idx].Progress != 100 { + t.log.Debugf("Skipping incomplete torrent %s (id=%s)", instances[idx].Name, instances[idx].ID) + counter++ + mergeChan <- nil + return + } + + freshHashes.Add(instances[idx].Hash) + freshIDs.Add(instances[idx].ID) + + tInfo := t.getMoreInfo(instances[idx]) + torrent := t.convertToTorrent(tInfo) + accessKey := t.GetKey(torrent) + freshAccessKeys.Add(accessKey) + + var forMerging *Torrent + mainTorrent, exists := allTorrents.Get(accessKey) + if !exists { + allTorrents.Set(accessKey, torrent) + + t.assignDirectory(torrent, func(directory string) { + listing, _ := t.DirectoryMap.Get(directory) + listing.Set(accessKey, torrent) + + updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey)) + }) + } else if !mainTorrent.Components.Has(tInfo.ID) { + forMerging = torrent + } + + counter++ + mergeChan <- forMerging }) } wg.Wait() - close(torChan) - t.log.Infof("Fetched info for %d torrents", len(instances)) + close(mergeChan) + t.log.Infof("Finished fetching info for %d torrents, proceeding to merge", counter) - var updatedPaths []string - noInfoCount := 0 - - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) - freshAccessKeys := mapset.NewSet[string]() - - allHashes := mapset.NewSet[string]() - t.getTorrentFiles().Each(func(path string) bool { - path = filepath.Base(path) - hash := strings.TrimSuffix(path, ".torrent_zurg") - allHashes.Add(hash) - return false - }) - freshHashes := mapset.NewSet[string]() - - allIDs := mapset.NewSet[string]() - t.getInfoFiles().Each(func(path string) bool { - path = filepath.Base(path) - torrentID := strings.TrimSuffix(path, ".info_zurg") - allIDs.Add(torrentID) - return false - }) - freshIDs := mapset.NewSet[string]() - - for torrent := range torChan { + for torrent := range mergeChan { if torrent == nil { - noInfoCount++ continue } - // there's only 1 component torrent at this point, let's get it - var tInfo *realdebrid.TorrentInfo - for _, tInfo = range torrent.Components { - break - } + t.log.Debugf("Merging %s", t.GetKey(torrent)) + accessKey := t.GetKey(torrent) - freshAccessKeys.Add(accessKey) - freshHashes.Add(torrent.Hash) - freshIDs.Add(tInfo.ID) - - // update allTorrents - isNewID := false - mainTorrent, exists := allTorrents.Get(accessKey) - if !exists { - allTorrents.Set(accessKey, torrent) - mainTorrent = torrent - isNewID = true - } else if _, ok := mainTorrent.Components[tInfo.ID]; !ok { - merged := t.mergeToMain(mainTorrent, torrent) - allTorrents.Set(accessKey, merged) - mainTorrent = merged - isNewID = true + existing, ok := allTorrents.Get(accessKey) + if !ok { + t.log.Warnf("Cannot merge %s", accessKey) + continue } + mainTorrent := t.mergeTorrents(existing, torrent) + allTorrents.Set(accessKey, mainTorrent) + t.writeTorrentToFile(mainTorrent) - if isNewID && tInfo.Progress == 100 { - // assign to directory - t.assignedDirectoryCb(mainTorrent, func(directory string) { - listing, _ := t.DirectoryMap.Get(directory) - listing.Set(accessKey, mainTorrent) - - updatedPaths = append(updatedPaths, fmt.Sprintf("%s/%s", directory, accessKey)) - }) - // write torrent to file - if !allHashes.Contains(mainTorrent.Hash) { - t.writeTorrentToFile(mainTorrent) + t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) { + if strings.HasPrefix(directory, "int__") { + return } - } + torrents.Remove(accessKey) + }) + + t.assignDirectory(mainTorrent, func(directory string) { + listing, _ := t.DirectoryMap.Get(directory) + listing.Set(accessKey, mainTorrent) + + updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey)) + }) + + t.log.Debugf("Merging %s done!", t.GetKey(existing)) } + t.log.Infof("Fetched info for %d torrents", len(instances)) + + noInfoCount := 0 + // removed torrents - allAccessKeys := mapset.NewSet[string](allTorrents.Keys()...) - allAccessKeys.Difference(freshAccessKeys).Each(func(accessKey string) bool { + oldPlusNewKeys := mapset.NewSet[string](allTorrents.Keys()...) + oldPlusNewKeys.Difference(freshAccessKeys).Each(func(accessKey string) bool { t.Delete(accessKey, false) return false }) @@ -122,18 +135,32 @@ func (t *TorrentManager) refreshTorrents() []string { t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount) // data directory cleanup - allHashes.Difference(freshHashes).Each(func(hash string) bool { + existingHashes := mapset.NewSet[string]() + t.getTorrentFiles().Each(func(path string) bool { + path = filepath.Base(path) + hash := strings.TrimSuffix(path, ".torrent_zurg") + existingHashes.Add(hash) + return false + }) + existingHashes.Difference(freshHashes).Each(func(hash string) bool { t.log.Infof("Deleting stale torrent file %s", hash) t.deleteTorrentFile(hash) return false }) - allIDs.Difference(freshIDs).Each(func(id string) bool { + existingIDs := mapset.NewSet[string]() + t.getInfoFiles().Each(func(path string) bool { + path = filepath.Base(path) + torrentID := strings.TrimSuffix(path, ".info_zurg") + existingIDs.Add(torrentID) + return false + }) + existingIDs.Difference(freshIDs).Each(func(id string) bool { t.log.Infof("Deleting stale info file %s", id) t.deleteInfoFile(id) return false }) - return updatedPaths + return updatedPaths.ToSlice() } // StartRefreshJob periodically refreshes the torrents @@ -165,16 +192,7 @@ func (t *TorrentManager) StartRefreshJob() { }() } -// getMoreInfo gets original name, size and files for a torrent -func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { - diskTor := t.readTorrentFromFile(rdTorrent.Hash) - if diskTor != nil { - if diskInfo, ok := diskTor.Components[rdTorrent.ID]; ok && diskInfo.Progress == 100 { - diskTor.Components = map[string]*realdebrid.TorrentInfo{rdTorrent.ID: diskInfo} - return diskTor - } - } - +func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *realdebrid.TorrentInfo { info := t.readInfoFromFile(rdTorrent.ID) if info == nil { var err error @@ -183,25 +201,40 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { t.log.Warnf("Cannot get info for id=%s: %v", rdTorrent.ID, err) return nil } - if info.Progress == 100 { - t.writeInfoToFile(info) - } + t.writeInfoToFile(info) + } + return info +} + +func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent { + torrent := t.readTorrentFromFile(info.Hash) + if torrent != nil && torrent.Components.Has(info.ID) { + return torrent } - torrent := Torrent{ + torrent = &Torrent{ Name: info.Name, OriginalName: info.OriginalName, Added: info.Added, Hash: info.Hash, State: NewTorrentState("broken_torrent"), + Components: cmap.New[*realdebrid.TorrentInfo](), } // SelectedFiles is a subset of Files with only the selected ones // it also has a Link field, which can be empty // if it is empty, it means the file is no longer available // Files+Links together are the same as SelectedFiles + allFilenames := mapset.NewSet[string]() + dupeFilenames := mapset.NewSet[string]() var selectedFiles []*File for _, file := range info.Files { + filename := filepath.Base(file.Path) + if allFilenames.Contains(filename) { + dupeFilenames.Add(filename) + } else { + allFilenames.Add(filename) + } if file.Selected == 0 { continue } @@ -216,6 +249,9 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { // all links are still intact! good! for i, file := range selectedFiles { file.Link = info.Links[i] + if strings.HasPrefix(file.Link, "https://real-debrid.com/d/") { + file.Link = file.Link[0:39] + } if err := file.State.Event(context.Background(), "repair_file"); err != nil { t.log.Errorf("Cannot repair file %s: %v", file.Path, err) } @@ -225,47 +261,34 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { t.log.Errorf("Cannot repair torrent %s: %v", torrent.Hash, err) } } else { - torrent.UnassignedLinks = mapset.NewSet[string](info.Links...) + torrent.UnassignedLinks = mapset.NewSet[string]() + for _, link := range info.Links { + if strings.HasPrefix(link, "https://real-debrid.com/d/") { + link = link[0:39] + } + torrent.UnassignedLinks.Add(link) + } } torrent.SelectedFiles = cmap.New[*File]() for _, file := range selectedFiles { - filename := t.GetPath(file) + baseFilename := t.GetPath(file) // todo better handling of duplicate filenames - if torrent.SelectedFiles.Has(filename) { - oldName := filename - ext := filepath.Ext(oldName) - noExtension := strings.TrimSuffix(oldName, ext) - newName := fmt.Sprintf("%s (%d)%s", noExtension, file.ID, ext) + if dupeFilenames.Contains(baseFilename) { + extension := filepath.Ext(baseFilename) + filenameNoExt := strings.TrimSuffix(baseFilename, extension) + newName := fmt.Sprintf("%s (%d)%s", filenameNoExt, file.ID, extension) torrent.SelectedFiles.Set(newName, file) } else { - torrent.SelectedFiles.Set(filename, file) + torrent.SelectedFiles.Set(baseFilename, file) } } - torrent.Components = map[string]*realdebrid.TorrentInfo{rdTorrent.ID: info} - return &torrent + torrent.Components.Set(info.ID, info) + + return torrent } -// ResetSelectedFiles resets the selected files for a torrent -func (t *TorrentManager) ResetSelectedFiles(torrent *Torrent) { - // reset selected files - newSelectedFiles := cmap.New[*File]() - torrent.SelectedFiles.IterCb(func(_ string, file *File) { - filename := t.GetPath(file) - if newSelectedFiles.Has(filename) { - oldName := filename - ext := filepath.Ext(oldName) - noExtension := strings.TrimSuffix(oldName, ext) - newName := fmt.Sprintf("%s (%d)%s", noExtension, file.ID, ext) - newSelectedFiles.Set(newName, file) - } else { - newSelectedFiles.Set(filename, file) - } - }) - torrent.SelectedFiles = newSelectedFiles -} - -func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) *Torrent { +func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent { var newer, older *Torrent if existing.Added < toMerge.Added { newer = toMerge @@ -275,21 +298,25 @@ func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) *Torrent { older = toMerge } - mergedComponents := map[string]*realdebrid.TorrentInfo{} - for k, v := range older.Components { - mergedComponents[k] = v - } - for k, v := range newer.Components { - mergedComponents[k] = v - } + // components + numComponents := 0 + mergedComponents := cmap.New[*realdebrid.TorrentInfo]() + older.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) { + numComponents++ + mergedComponents.Set(torrentID, info) + }) + newer.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) { + numComponents++ + mergedComponents.Set(torrentID, info) + }) - // build the main torrent - mainTorrent := Torrent{ - Name: newer.Name, - OriginalName: newer.OriginalName, - Rename: newer.Rename, - Hash: newer.Hash, - Added: newer.Added, + // base of the merged torrent + mergedTorrent := Torrent{ + Name: older.Name, + OriginalName: older.OriginalName, + Rename: older.Rename, + Hash: older.Hash, + Added: older.Added, Components: mergedComponents, State: older.State, @@ -299,44 +326,58 @@ func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) *Torrent { reasons := mapset.NewSet[string]() reasons.Add(older.UnrepairableReason) reasons.Add(newer.UnrepairableReason) - mainTorrent.UnrepairableReason = strings.Join(reasons.ToSlice(), ", ") + mergedTorrent.UnrepairableReason = strings.Join(reasons.ToSlice(), ", ") - mainTorrent.SelectedFiles = cmap.New[*File]() - older.SelectedFiles.IterCb(func(key string, olderFile *File) { - mainTorrent.SelectedFiles.Set(key, olderFile) - }) - newer.SelectedFiles.IterCb(func(key string, newerFile *File) { - if f, ok := mainTorrent.SelectedFiles.Get(key); ok && f.State.Is("deleted_file") { - return + // selected files + mergedTorrent.SelectedFiles = cmap.New[*File]() + newer.SelectedFiles.IterCb(func(key string, file *File) { + if f, ok := mergedTorrent.SelectedFiles.Get(key); !ok || !f.State.Is("ok_file") { + mergedTorrent.SelectedFiles.Set(key, file) + } + }) + older.SelectedFiles.IterCb(func(key string, file *File) { + if f, ok := mergedTorrent.SelectedFiles.Get(key); !ok || !f.State.Is("ok_file") { + mergedTorrent.SelectedFiles.Set(key, file) } - mainTorrent.SelectedFiles.Set(key, newerFile) }) - t.CheckDeletedStatus(&mainTorrent) // unassigned links - mainTorrent.UnassignedLinks = mapset.NewSet[string]() - newer.UnassignedLinks.Union(older.UnassignedLinks).Each(func(link string) bool { + mergedTorrent.UnassignedLinks = mapset.NewSet[string]() + links := newer.UnassignedLinks.Union(older.UnassignedLinks) + links.Each(func(link string) bool { found := false - mainTorrent.SelectedFiles.IterCb(func(key string, file *File) { + mergedTorrent.SelectedFiles.IterCb(func(key string, file *File) { if !found && file.Link == link { found = true - return } }) if !found { - mainTorrent.UnassignedLinks.Add(link) + mergedTorrent.UnassignedLinks.Add(link) } return false }) - return &mainTorrent + brokenCount := 0 + okCount := 0 + wtfCount := 0 + mergedTorrent.SelectedFiles.IterCb(func(key string, file *File) { + if file.State.Is("broken_file") { + brokenCount++ + } else if file.State.Is("ok_file") { + okCount++ + } else { + wtfCount++ + } + }) + + // todo + t.log.Debugf("Merging %s (%d comps) - selected files: %d ; unassigned: %d ; broken: %d ; ok %d ; wtf %d", t.GetKey(&mergedTorrent), numComponents, brokenCount+okCount+wtfCount, mergedTorrent.UnassignedLinks.Cardinality(), brokenCount, okCount, wtfCount) + + return &mergedTorrent } -func (t *TorrentManager) assignedDirectoryCb(tor *Torrent, cb func(string)) { - torrentIDs := []string{} - for id := range tor.Components { - torrentIDs = append(torrentIDs, id) - } +func (t *TorrentManager) assignDirectory(tor *Torrent, cb func(string)) { + torrentIDs := tor.Components.Keys() // get filenames needed for directory conditions var filenames []string var fileSizes []int64 diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index d063c5f..4f0766a 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -150,10 +150,7 @@ func (t *TorrentManager) Repair(torrent *Torrent, wg *sync.WaitGroup) { } func (t *TorrentManager) repair(torrent *Torrent) { - torrentIDs := []string{} - for id := range torrent.Components { - torrentIDs = append(torrentIDs, id) - } + torrentIDs := torrent.Components.Keys() t.log.Infof("Started repair process for torrent %s (ids=%v)", t.GetKey(torrent), torrentIDs) // handle torrents with incomplete links for selected files @@ -177,12 +174,11 @@ func (t *TorrentManager) repair(torrent *Torrent) { return } // delete old torrents - for id := range torrent.Components { - if id == info.ID { - continue + torrent.Components.IterCb(func(id string, _ *realdebrid.TorrentInfo) { + if id != info.ID { + t.api.DeleteTorrent(id) } - t.api.DeleteTorrent(id) - } + }) t.log.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent)) return } else if info != nil && info.Progress != 100 { @@ -210,18 +206,18 @@ func (t *TorrentManager) repair(torrent *Torrent) { return } - t.log.Infof("Repairing by downloading %d batches of the %d broken files of torrent %s", int(math.Ceil(float64(len(brokenFiles))/130)), len(brokenFiles), t.GetKey(torrent)) + t.log.Infof("Repairing by downloading %d batches of the %d broken files of torrent %s", int(math.Ceil(float64(len(brokenFiles))/100)), len(brokenFiles), t.GetKey(torrent)) newlyDownloadedIds := make([]string, 0) - group := make([]*File, 0) batchNum := 1 - for _, file := range brokenFiles { - group = append(group, file) - if len(group) >= 130 { + brokenFileIDs := getFileIDs(brokenFiles) + var group []string + for _, fileIDStr := range brokenFileIDs { + group = append(group, fileIDStr) + if len(group) >= 100 { t.log.Debugf("Downloading batch %d of broken files of torrent %s", batchNum, t.GetKey(torrent)) batchNum++ - brokenFileIDs := getFileIDs(group) - redownloadedInfo, err := t.redownloadTorrent(torrent, brokenFileIDs) + redownloadedInfo, err := t.redownloadTorrent(torrent, group) if err != nil { t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error()) for _, newId := range newlyDownloadedIds { @@ -230,15 +226,14 @@ func (t *TorrentManager) repair(torrent *Torrent) { return } newlyDownloadedIds = append(newlyDownloadedIds, redownloadedInfo.ID) - group = make([]*File, 0) + group = make([]string, 0) } } t.log.Debugf("Downloading last batch of broken files of torrent %s", t.GetKey(torrent)) if len(group) > 0 { - brokenFileIDs := getFileIDs(group) - _, err := t.redownloadTorrent(torrent, brokenFileIDs) + _, err := t.redownloadTorrent(torrent, group) if err != nil { t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error()) for _, newId := range newlyDownloadedIds { @@ -276,6 +271,9 @@ func (t *TorrentManager) assignUnassignedLinks(torrent *Torrent) bool { // base it on size because why not? if !assigned && file.State.Is("broken_file") && file.Bytes == unrestrict.Filesize && strings.HasSuffix(strings.ToLower(file.Path), strings.ToLower(unrestrict.Filename)) { file.Link = link + if strings.HasPrefix(file.Link, "https://real-debrid.com/d/") { + file.Link = file.Link[0:39] + } if err := file.State.Event(context.Background(), "repair_file"); err != nil { t.log.Errorf("Failed to mark file %s as repaired: %v", file.Path, err) return @@ -351,17 +349,14 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) // broken files means broken links // if brokenFiles is not provided, we will redownload all files finalSelection := strings.Join(selection, ",") - selectionCount := len(selection) - if selectionCount == 0 { - tmpSelection := "" + if len(selection) == 0 { torrent.SelectedFiles.IterCb(func(_ string, file *File) { - tmpSelection += fmt.Sprintf("%d,", file.ID) // select all files - selectionCount++ + selection = append(selection, fmt.Sprintf("%d", file.ID)) }) - if tmpSelection == "" { - return nil, nil // nothing to repair + if len(selection) == 0 { + return nil, nil } else { - finalSelection = tmpSelection[:len(tmpSelection)-1] + finalSelection = strings.Join(selection, ",") } } @@ -434,9 +429,9 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) } // check if incorrect number of links - if info.Progress == 100 && len(info.Links) != selectionCount { + if info.Progress == 100 && len(info.Links) != len(selection) { t.trash(newTorrentID) - return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), selectionCount) + return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), len(selection)) } t.log.Infof("Redownloading torrent %s successful (id=%s, progress=%d)", t.GetKey(torrent), info.ID, info.Progress) @@ -544,6 +539,9 @@ func (t *TorrentManager) isStillBroken(info *realdebrid.TorrentInfo, brokenFiles // all links are still intact! good! for i, file := range selectedFiles { file.Link = info.Links[i] + if strings.HasPrefix(file.Link, "https://real-debrid.com/d/") { + file.Link = file.Link[0:39] + } if err := file.State.Event(context.Background(), "repair_file"); err != nil { t.log.Errorf("Failed to mark file %s as repaired: %v", file.Path, err) return true diff --git a/internal/torrent/torrent_types.go b/internal/torrent/torrent_types.go index 98b3dc8..1520cd3 100644 --- a/internal/torrent/torrent_types.go +++ b/internal/torrent/torrent_types.go @@ -14,17 +14,17 @@ import ( var json = jsoniter.ConfigCompatibleWithStandardLibrary type Torrent struct { - Name string `json:"Name"` - OriginalName string `json:"OriginalName"` - Hash string `json:"Hash"` - Added string `json:"Added"` - Components map[string]*realdebrid.TorrentInfo `json:"Components"` + Name string `json:"Name"` + OriginalName string `json:"OriginalName"` + Hash string `json:"Hash"` + Added string `json:"Added"` + Components cmap.ConcurrentMap[string, *realdebrid.TorrentInfo] `json:"-"` UnassignedLinks mapset.Set[string] `json:"UnassignedLinks"` // when links are not complete, we cannot assign them to a file so we store them here until it's fixed - Rename string `json:"Rename"` // modified over time - SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` // modified over time - UnrepairableReason string `json:"Unfixable"` // modified over time + Rename string `json:"Rename"` + SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` + UnrepairableReason string `json:"Unfixable"` State *fsm.FSM `json:"-"` Version string `json:"Version"` // only used for files @@ -33,6 +33,7 @@ type Torrent struct { func (t *Torrent) MarshalJSON() ([]byte, error) { type Alias Torrent temp := &struct { + ComponentsJson stdjson.RawMessage `json:"Components"` SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"` StateJson stdjson.RawMessage `json:"State"` @@ -54,6 +55,12 @@ func (t *Torrent) MarshalJSON() ([]byte, error) { temp.UnassignedLinksJson = []byte(unassignedLinksStr) } + componentsJson, err := t.Components.MarshalJSON() + if err != nil { + return nil, err + } + temp.ComponentsJson = componentsJson + temp.StateJson = []byte(`"` + t.State.Current() + `"`) return json.Marshal(temp) @@ -62,6 +69,7 @@ func (t *Torrent) MarshalJSON() ([]byte, error) { func (t *Torrent) UnmarshalJSON(data []byte) error { type Alias Torrent temp := &struct { + ComponentsJson stdjson.RawMessage `json:"Components"` SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"` StateJson string `json:"State"` @@ -73,6 +81,13 @@ func (t *Torrent) UnmarshalJSON(data []byte) error { return err } + t.Components = cmap.New[*realdebrid.TorrentInfo]() + if len(temp.ComponentsJson) > 0 { + if err := t.Components.UnmarshalJSON(temp.ComponentsJson); err != nil { + return err + } + } + t.SelectedFiles = cmap.New[*File]() if len(temp.SelectedFilesJson) > 0 { if err := t.SelectedFiles.UnmarshalJSON(temp.SelectedFilesJson); err != nil { @@ -87,29 +102,11 @@ func (t *Torrent) UnmarshalJSON(data []byte) error { t.UnassignedLinks = mapset.NewSet[string]() } - t.State = NewFileState(temp.StateJson) + t.State = NewTorrentState(temp.StateJson) return nil } -func (t *Torrent) AnyInProgress() bool { - for _, info := range t.Components { - if info.Progress != 100 { - return true - } - } - return false -} - -func (t *Torrent) AllInProgress() bool { - for _, info := range t.Components { - if info.Progress == 100 { - return false - } - } - return true -} - func (t *Torrent) ComputeTotalSize() int64 { totalSize := int64(0) t.SelectedFiles.IterCb(func(key string, value *File) { diff --git a/internal/torrent/uncached.go b/internal/torrent/uncached.go index c2ff497..75633d8 100644 --- a/internal/torrent/uncached.go +++ b/internal/torrent/uncached.go @@ -17,7 +17,7 @@ func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) { hashGroups = append(hashGroups, currentGroup) allTorrents.IterCb(func(_ string, torrent *Torrent) { - if torrent.AnyInProgress() || torrent.AllInProgress() || torrent.UnrepairableReason != "" { + if torrent.UnrepairableReason != "" { return } @@ -46,7 +46,7 @@ func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) { var uncachedTorrents []*Torrent allTorrents.IterCb(func(_ string, torrent *Torrent) { - if torrent.AnyInProgress() || torrent.AllInProgress() || torrent.UnrepairableReason != "" { + if torrent.UnrepairableReason != "" { return } diff --git a/pkg/realdebrid/types.go b/pkg/realdebrid/types.go index 11ca99d..9fff189 100644 --- a/pkg/realdebrid/types.go +++ b/pkg/realdebrid/types.go @@ -50,10 +50,6 @@ type Torrent struct { Added string `json:"-"` } -func (i *Torrent) IsDone() bool { - return i.Progress == 100 && len(i.Links) > 0 -} - func (i *Torrent) UnmarshalJSON(data []byte) error { type Alias Torrent aux := &struct {