From 0a451cccde7be417b9a5cd6f096b5c2d9d813bf8 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Thu, 18 Jan 2024 20:54:41 +0100 Subject: [PATCH] Fix repairs --- internal/torrent/manager.go | 12 ++-- internal/torrent/refresh.go | 19 +++-- internal/torrent/repair.go | 137 +++++++++++++++++++----------------- internal/torrent/util.go | 2 +- 4 files changed, 93 insertions(+), 77 deletions(-) diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index 9899864..c4f192c 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -28,8 +28,8 @@ type TorrentManager struct { DownloadCache cmap.ConcurrentMap[string, *realdebrid.Download] DownloadMap cmap.ConcurrentMap[string, *realdebrid.Download] Repairs cmap.ConcurrentMap[string, bool] + onlyForRepair cmap.ConcurrentMap[string, *Torrent] allAccessKeys mapset.Set[string] - onlyForRepair mapset.Set[string] latestState *LibraryState requiredVersion string workerPool *ants.Pool @@ -43,8 +43,12 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p t := &TorrentManager{ Config: cfg, Api: api, + DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](), + DownloadCache: cmap.New[*realdebrid.Download](), + DownloadMap: cmap.New[*realdebrid.Download](), + onlyForRepair: cmap.New[*Torrent](), + Repairs: cmap.New[bool](), allAccessKeys: mapset.NewSet[string](), - onlyForRepair: mapset.NewSet[string](), latestState: &LibraryState{}, requiredVersion: "11.01.2024", workerPool: p, @@ -52,7 +56,6 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p } // create internal directories - t.DirectoryMap = cmap.New[cmap.ConcurrentMap[string, *Torrent]]() t.DirectoryMap.Set(INT_ALL, cmap.New[*Torrent]()) // key is GetAccessKey() t.DirectoryMap.Set(INT_INFO_CACHE, cmap.New[*Torrent]()) // key is Torrent ID // create directory maps @@ -61,9 +64,6 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p } // Fetch downloads - t.DownloadCache = cmap.New[*realdebrid.Download]() - t.DownloadMap = cmap.New[*realdebrid.Download]() - t.Repairs = cmap.New[bool]() if t.Config.EnableDownloadCache() { _ = t.workerPool.Submit(func() { page := 1 diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go index d23b052..6c64824 100644 --- a/internal/torrent/refresh.go +++ b/internal/torrent/refresh.go @@ -22,12 +22,23 @@ func (t *TorrentManager) RefreshTorrents() []string { infoChan := make(chan *Torrent, len(instances)) var wg sync.WaitGroup + allTorrents, _ := t.DirectoryMap.Get(INT_ALL) for i := range instances { idx := i wg.Add(1) _ = t.workerPool.Submit(func() { defer wg.Done() - infoChan <- t.getMoreInfo(instances[idx]) + if instances[idx].Progress == 100 && t.onlyForRepair.Has(instances[idx].ID) { + torrent, _ := t.onlyForRepair.Get(instances[idx].ID) + brokenFiles := getBrokenFiles(torrent) + info, err := t.redownloadTorrent(torrent, "") + if err == nil && info.Progress == 100 && !t.isStillBroken(info, brokenFiles) { + t.onlyForRepair.Remove(instances[idx].ID) + } + infoChan <- nil + } else { + infoChan <- t.getMoreInfo(instances[idx]) + } }) } @@ -36,19 +47,13 @@ func (t *TorrentManager) RefreshTorrents() []string { t.log.Debugf("Fetched info for %d torrents", len(instances)) freshKeys := mapset.NewSet[string]() - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) noInfoCount := 0 for info := range infoChan { if info == nil { noInfoCount++ continue } - accessKey := t.GetKey(info) - - if t.handleRepairTorrents(info) { - continue - } if !info.AnyInProgress() { freshKeys.Add(accessKey) } diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index 51c60c6..2bf9396 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -88,7 +88,7 @@ func (t *TorrentManager) repairAll() { } }) - if !isCached || hasBrokenFiles { + if !isCached || hasBrokenFiles || torrent.UnassignedLinks.Cardinality() > 0 { toRepair = append(toRepair, torrent) } }) @@ -147,45 +147,56 @@ func (t *TorrentManager) repair(torrent *Torrent) { // handle torrents with incomplete links for selected files assignedCount := 0 + + // number of rar files detected from the unrestricted links rarCount := 0 - unassignedDownloads := make([]*realdebrid.Download, 0) - assignedLinks := make([]string, 0) + + newUnassignedLinks := cmap.New[*realdebrid.Download]() + + // unrestrict each unassigned link that was filled out during torrent init torrent.UnassignedLinks.Each(func(link string) bool { unrestrict := t.UnrestrictUntilOk(link) if unrestrict == nil { + newUnassignedLinks.Set(link, nil) + // return early, no point continuing return false } - // assign to a selected file + + // try to assign to a selected file assigned := false torrent.SelectedFiles.IterCb(func(_ string, file *File) { - // if strings.HasSuffix(file.Path, unrestrict.Filename) { + // base it on size because why not? if file.Bytes == unrestrict.Filesize { - file.Link = unrestrict.Link + file.Link = link assigned = true assignedCount++ } }) + if !assigned { + // if not assigned and is a rar, likely it was rar'ed by RD if strings.HasSuffix(unrestrict.Filename, ".rar") { rarCount++ } - unassignedDownloads = append(unassignedDownloads, unrestrict) - } else { - assignedLinks = append(assignedLinks, unrestrict.Link) + newUnassignedLinks.Set(link, unrestrict) } return false }) - torrent.UnassignedLinks = torrent.UnassignedLinks.Difference(mapset.NewSet(assignedLinks...)) if assignedCount > 0 { + // if there are any assigned count t.log.Infof("Assigned %d links to selected files for torrent %s", assignedCount, t.GetKey(torrent)) } else if rarCount > 0 { + // also is assignedCount=0 // this is a rar'ed torrent, nothing we can do if t.Config.ShouldDeleteRarFiles() { t.log.Warnf("Torrent %s is rar'ed and we cannot repair it, deleting it as configured", t.GetKey(torrent)) t.Delete(t.GetKey(torrent), true) } else { - for _, unassigned := range unassignedDownloads { + newUnassignedLinks.IterCb(func(_ string, unassigned *realdebrid.Download) { + if unassigned == nil { + return + } newFile := &File{ File: realdebrid.File{ ID: 0, @@ -197,40 +208,47 @@ func (t *TorrentManager) repair(torrent *Torrent) { Link: unassigned.Link, } torrent.SelectedFiles.Set(unassigned.Filename, newFile) - } + }) + torrent.UnassignedLinks = mapset.NewSet[string]() t.markAsUnfixable(torrent) t.markAsUnplayable(torrent) } return } - // second solution: add only the broken files - var brokenFiles []File - torrent.SelectedFiles.IterCb(func(_ string, file *File) { - if !strings.HasPrefix(file.Link, "http") && file.Link != "unselect" { - brokenFiles = append(brokenFiles, *file) - } - }) + // get all broken files + brokenFiles := getBrokenFiles(torrent) t.log.Debugf("During repair, zurg found %d broken files for torrent %s", len(brokenFiles), t.GetKey(torrent)) + // first solution: reinsert and see if the broken file is now working + t.log.Debugf("Repair_try#1: Trying to redownload torrent %s to repair it", t.GetKey(torrent)) + info, err := t.redownloadTorrent(torrent, "") + if err != nil { + t.log.Warnf("Cannot repair torrent %s", t.GetKey(torrent)) + } + if info.Progress != 100 || (info.Progress == 100 && !t.isStillBroken(info, brokenFiles)) { + t.log.Infof("Successfully repaired torrent %s", t.GetKey(torrent)) + return + } + + // second solution: add only the broken files if len(brokenFiles) > 0 { - t.log.Infof("Redownloading %dof%d files for torrent %s", len(brokenFiles), torrent.SelectedFiles.Count(), t.GetKey(torrent)) + t.log.Infof("Repair_try#2: Redownloading %dof%d broken files for torrent %s", len(brokenFiles), torrent.SelectedFiles.Count(), t.GetKey(torrent)) brokenFileIDs := strings.Join(getFileIDs(brokenFiles), ",") - if t.redownloadTorrent(torrent, brokenFileIDs) { - t.log.Infof("Successfully downloaded torrent %s to repair it", t.GetKey(torrent)) - } else { + _, err := t.redownloadTorrent(torrent, brokenFileIDs) + if err != nil { t.log.Warnf("Cannot repair torrent %s", t.GetKey(torrent)) + } else { + t.log.Infof("Successfully repaired torrent %s", t.GetKey(torrent)) } } else { t.log.Warnf("Torrent %s has no broken files to repair", t.GetKey(torrent)) } } -func (t *TorrentManager) redownloadTorrent(torrent *Torrent, brokenFiles string) bool { +func (t *TorrentManager) redownloadTorrent(torrent *Torrent, brokenFiles string) (*realdebrid.TorrentInfo, error) { t.log.Debugf("Redownloading torrent %s, broken files=%s (all if empty)", t.GetKey(torrent), brokenFiles) - oldTorrentIDs := make([]string, 0) - // broken files means broken links // if brokenFiles is not provided if brokenFiles == "" { @@ -241,7 +259,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, brokenFiles string) tmpSelection += fmt.Sprintf("%d,", file.ID) // select all files }) if tmpSelection == "" { - return true // nothing to repair + return nil, nil // nothing to repair } else { brokenFiles = tmpSelection[:len(tmpSelection)-1] } @@ -250,11 +268,10 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, brokenFiles string) // redownload torrent resp, err := t.Api.AddMagnetHash(torrent.Hash) if err != nil { - t.log.Warnf("Cannot redownload torrent: %v", err) if strings.Contains(err.Error(), "infringing_file") { t.markAsUnfixable(torrent) } - return false + return nil, fmt.Errorf("cannot redownload torrent: %v", err) } time.Sleep(1 * time.Second) @@ -262,17 +279,16 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, brokenFiles string) newTorrentID := resp.ID err = t.Api.SelectTorrentFiles(newTorrentID, brokenFiles) if err != nil { - t.log.Warnf("Cannot start redownloading: %v", err) t.Api.DeleteTorrent(newTorrentID) - return false + return nil, fmt.Errorf("cannot start redownloading: %v", err) } + time.Sleep(1 * time.Second) // see if the torrent is ready info, err := t.Api.GetTorrentInfo(newTorrentID) if err != nil { - t.log.Warnf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err) t.Api.DeleteTorrent(newTorrentID) - return false + return nil, fmt.Errorf("cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err) } // documented status: magnet_error, magnet_conversion, waiting_files_selection, queued, downloading, downloaded, error, virus, compressing, uploading, dead @@ -286,41 +302,39 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, brokenFiles string) } } if !isOkStatus { - t.log.Warnf("The redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status) t.Api.DeleteTorrent(newTorrentID) - return false + return nil, fmt.Errorf("the redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status) } if info.Progress != 100 { t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion (this should fix the issue already)", info.ID) - t.onlyForRepair.Add(newTorrentID) if len(oldTorrentIDs) > 0 { + // only triggered when brokenFiles == "" for _, id := range oldTorrentIDs { t.Api.DeleteTorrent(id) } } else { - t.onlyForRepair.Add(newTorrentID) + t.onlyForRepair.Set(newTorrentID, torrent) } - return true + return info, nil } brokenCount := len(strings.Split(brokenFiles, ",")) if len(info.Links) != brokenCount { - t.log.Infof("It did not fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), brokenCount) t.Api.DeleteTorrent(newTorrentID) - return false + return nil, fmt.Errorf("it did not fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), brokenCount) } t.log.Infof("Redownload successful id=%s", newTorrentID) - t.onlyForRepair.Add(newTorrentID) if len(oldTorrentIDs) > 0 { + // only triggered when brokenFiles == "" for _, id := range oldTorrentIDs { t.Api.DeleteTorrent(id) } } else { - t.onlyForRepair.Add(newTorrentID) + t.onlyForRepair.Set(newTorrentID, torrent) } - return true + return info, nil } func (t *TorrentManager) canCapacityHandle() bool { @@ -387,29 +401,26 @@ func (t *TorrentManager) markAsUnfixable(torrent *Torrent) { }) } -func (t *TorrentManager) handleRepairTorrents(info *Torrent) bool { - allTorrents, _ := t.DirectoryMap.Get(INT_ALL) - accessKey := t.GetKey(info) - torrentIDs := info.DownloadedIDs.ToSlice() - inRepairList := false - for _, torrentID := range torrentIDs { - if t.onlyForRepair.Contains(torrentID) { - inRepairList = true - break +func getBrokenFiles(torrent *Torrent) []*File { + var brokenFiles []*File + torrent.SelectedFiles.IterCb(func(_ string, file *File) { + if !strings.HasPrefix(file.Link, "http") && file.Link != "unselect" { + brokenFiles = append(brokenFiles, file) } - } - if !info.AnyInProgress() && inRepairList { - t.log.Debugf("Newly downloaded %s (id=%s) is in repair list", info.Name, torrentIDs[0]) - torrent, stillExists := allTorrents.Get(accessKey) - if stillExists && t.redownloadTorrent(torrent, "") { - t.log.Debugf("Deleting repair temp id=%s because it has served its purpose", torrentIDs[0]) - // if it's 100% and it's a temp repair, remove it - for _, torrentID := range torrentIDs { - t.Api.DeleteTorrent(torrentID) - t.onlyForRepair.Remove(torrentID) + }) + return brokenFiles +} + +func (t *TorrentManager) isStillBroken(info *realdebrid.TorrentInfo, brokenFiles []*File) bool { + for _, oldFile := range brokenFiles { + for idx, newFile := range info.Files { + if oldFile.Path == newFile.Path { + unrestrict := t.UnrestrictUntilOk(info.Links[idx]) + if unrestrict == nil || oldFile.Bytes != unrestrict.Filesize { + return true + } } } - return true } return false } diff --git a/internal/torrent/util.go b/internal/torrent/util.go index f186680..f09fad1 100644 --- a/internal/torrent/util.go +++ b/internal/torrent/util.go @@ -4,7 +4,7 @@ import ( "fmt" ) -func getFileIDs(files []File) []string { +func getFileIDs(files []*File) []string { var fileIDs []string for _, file := range files { if file.ID != 0 {