From ac89f53896617ffd2952cc5f82bf367b43ff2221 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Mon, 27 May 2024 06:25:55 +0200 Subject: [PATCH] Bin error checks --- internal/torrent/bins.go | 46 ++++++++++++++++++-------------- internal/torrent/delete.go | 2 +- internal/torrent/refresh.go | 35 ++++++------------------ internal/torrent/repair.go | 53 ++++++++++++++----------------------- 4 files changed, 55 insertions(+), 81 deletions(-) diff --git a/internal/torrent/bins.go b/internal/torrent/bins.go index fbec0dd..52e5c02 100644 --- a/internal/torrent/bins.go +++ b/internal/torrent/bins.go @@ -1,6 +1,7 @@ package torrent import ( + "fmt" "os" "strings" @@ -64,30 +65,35 @@ func (t *TorrentManager) binImmediately(torrentId string) bool { return false } -func (t *TorrentManager) binOnceDone(torrentId string) bool { - if t.OnceDoneBin.Contains(torrentId) { - if err := t.api.DeleteTorrent(torrentId); err != nil { - t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err) - return false - } - t.OnceDoneBin.Remove(torrentId) - return true +func (t *TorrentManager) binOnceDoneErrorCheck(torrentId, status string) bool { + okStatuses := mapset.NewSet("downloading", "downloaded", "uploading", "queued", "compressing") + if !okStatuses.Contains(status) { + return t.binOnceDone(torrentId) } + return false +} - // special case: xxx-yyy means if xxx is done, delete yyy +func (t *TorrentManager) binOnceDone(torrentId string) bool { found := false - specialCases := t.OnceDoneBin.ToSlice() - for _, specialCase := range specialCases { - if strings.Contains(specialCase, "-") { - parts := strings.Split(specialCase, "-") - if parts[0] == torrentId { - if err := t.api.DeleteTorrent(parts[1]); err != nil { - t.log.Errorf("Failed to delete torrent %s: %v", parts[1], err) - continue - } - t.OnceDoneBin.Remove(specialCase) - found = true + binnedIDs := t.OnceDoneBin.ToSlice() + // special case: xxx-yyy means if xxx is done, delete yyy + specialCase := fmt.Sprintf("%s-", torrentId) + for _, entry := range binnedIDs { + if strings.Contains(entry, specialCase) { + idToDelete := strings.Split(entry, "-")[1] + if err := t.api.DeleteTorrent(idToDelete); err != nil { + t.log.Errorf("Failed to delete torrent %s: %v", idToDelete, err) + continue } + t.OnceDoneBin.Remove(entry) + found = true + } else if entry == torrentId { + if err := t.api.DeleteTorrent(torrentId); err != nil { + t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err) + return false + } + t.OnceDoneBin.Remove(torrentId) + return true } } diff --git a/internal/torrent/delete.go b/internal/torrent/delete.go index 1b43e80..7f544aa 100644 --- a/internal/torrent/delete.go +++ b/internal/torrent/delete.go @@ -35,7 +35,7 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) { }) } } - t.log.Infof("Removing torrent %s from zurg database (not real-debrid)", accessKey) + // t.log.Infof("Removing torrent %s from zurg database (not real-debrid)", accessKey) t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) { torrents.Remove(accessKey) }) diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go index 57db22c..f025b63 100644 --- a/internal/torrent/refresh.go +++ b/internal/torrent/refresh.go @@ -32,44 +32,25 @@ func (t *TorrentManager) refreshTorrents() []string { freshIDs := mapset.NewSet[string]() freshAccessKeys := mapset.NewSet[string]() - t.getTorrentFiles("dump").Each(func(filePath string) bool { + cachedAccessKeys := mapset.NewSet[string]() + t.getTorrentFiles("data").Each(func(filePath string) bool { torrent := t.readTorrentFromFile(filePath) if torrent != nil { - accessKey := t.GetKey(torrent) - if !allTorrents.Has(accessKey) { - t.log.Debugf("Loading dumped torrent %s", accessKey) - allTorrents.Set(accessKey, torrent) - t.assignDirectory(torrent, func(directory string) { - listing, _ := t.DirectoryMap.Get(directory) - listing.Set(accessKey, torrent) - // note that we're not adding it to updatedPaths - }) - } - freshAccessKeys.Add(accessKey) // to prevent being deleted + allTorrents.Set(t.GetKey(torrent), torrent) + filename := filepath.Base(filePath) + cachedAccessKeys.Add(strings.TrimSuffix(filename, ".zurgtorrent")) } return false }) - cachedAccessKeys := mapset.NewSet[string]() - t.getTorrentFiles("data").Each(func(path string) bool { - path = filepath.Base(path) - cachedAccessKeys.Add(strings.TrimSuffix(path, ".zurgtorrent")) - return false - }) - for i := range instances { wg.Add(1) idx := i _ = t.workerPool.Submit(func() { defer wg.Done() - if t.binImmediately(instances[idx].ID) { - // t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID) - mergeChan <- nil - return - } - - if instances[idx].Progress != 100 { - // t.log.Debugf("Skipping incomplete torrent %s (id=%s)", instances[idx].Name, instances[idx].ID) + if t.binImmediately(instances[idx].ID) || + t.binOnceDoneErrorCheck(instances[idx].ID, instances[idx].Status) || + instances[idx].Progress != 100 { mergeChan <- nil return } diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index fb01975..6438433 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -192,7 +192,7 @@ func (t *TorrentManager) repair(torrent *Torrent) { if info != nil && info.Progress == 100 && !t.isStillBroken(info, brokenFiles) { // successful repair torrent.State.Event(context.Background(), "mark_as_repaired") - t.repairLog.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent)) + t.repairLog.Infof("Successfully repaired torrent %s by redownloading whole torrent", t.GetKey(torrent)) // delete the torrents it replaced torrent.DownloadedIDs.Each(func(torrentID string) bool { if torrentID != info.ID { @@ -202,7 +202,7 @@ func (t *TorrentManager) repair(torrent *Torrent) { }) return } else if info != nil && info.Progress != 100 { - t.repairLog.Infof("Torrent %s is still in progress after redownloading but it should be repaired once done", t.GetKey(torrent)) + t.repairLog.Infof("Torrent %s is still in progress after redownloading whole torrent but it should be repaired once done", t.GetKey(torrent)) // once info.ID is done, we can delete the old torrent torrent.DownloadedIDs.Each(func(torrentID string) bool { if torrentID != info.ID { @@ -214,13 +214,13 @@ func (t *TorrentManager) repair(torrent *Torrent) { } if err != nil { - t.repairLog.Warnf("Cannot repair torrent %s by redownloading all files (error=%v)", t.GetKey(torrent), err) + t.repairLog.Warnf("Cannot repair torrent %s by redownloading whole torrent (error=%v)", t.GetKey(torrent), err) } else { - t.repairLog.Warnf("Cannot repair torrent %s by redownloading all files", t.GetKey(torrent)) + t.repairLog.Warnf("Cannot repair torrent %s by redownloading whole torrent (links are still broken)", t.GetKey(torrent)) } if torrent.UnrepairableReason != "" { - t.repairLog.Debugf("Torrent %s has been marked as unfixable during redownload (%s), ending repair process early", t.GetKey(torrent), torrent.UnrepairableReason) + t.repairLog.Debugf("Torrent %s has been marked as unfixable during redownloading whole torrent (%s), ending repair process early", t.GetKey(torrent), torrent.UnrepairableReason) return } @@ -233,7 +233,8 @@ func (t *TorrentManager) repair(torrent *Torrent) { return } - t.repairLog.Infof("Torrent %s will be repaired by downloading %d batches of the %d broken files", t.GetKey(torrent), int(math.Ceil(float64(len(brokenFiles))/100)), len(brokenFiles)) + totalBatches := int(math.Ceil(float64(len(brokenFiles)) / 100)) + t.repairLog.Infof("Torrent %s will be repaired by downloading %d batches of the %d broken files", t.GetKey(torrent), totalBatches, len(brokenFiles)) newlyDownloadedIds := make([]string, 0) batchNum := 1 @@ -242,7 +243,7 @@ func (t *TorrentManager) repair(torrent *Torrent) { for _, fileIDStr := range brokenFileIDs { group = append(group, fileIDStr) if len(group) >= 100 { - t.repairLog.Debugf("Downloading batch %d of broken files of torrent %s", batchNum, t.GetKey(torrent)) + t.repairLog.Debugf("Downloading batch %d/%d of broken files of torrent %s", batchNum, totalBatches, t.GetKey(torrent)) batchNum++ redownloadedInfo, err := t.redownloadTorrent(torrent, group) if err != nil { @@ -258,9 +259,8 @@ func (t *TorrentManager) repair(torrent *Torrent) { } } - t.repairLog.Debugf("Downloading last batch of broken files of torrent %s", t.GetKey(torrent)) - if len(group) > 0 { + t.repairLog.Debugf("Downloading batch %d/%d of broken files of torrent %s", batchNum, totalBatches, t.GetKey(torrent)) redownloadedInfo, err := t.redownloadTorrent(torrent, group) if err != nil { t.repairLog.Warnf("Cannot repair torrent %s by downloading broken files (error=%v) giving up", t.GetKey(torrent), err) @@ -281,7 +281,7 @@ func (t *TorrentManager) repair(torrent *Torrent) { func (t *TorrentManager) assignLinks(torrent *Torrent) bool { unassignedTotal := torrent.UnassignedLinks.Cardinality() - t.repairLog.Infof("Trying to assign %d links to the %d selected of incomplete torrent %s", unassignedTotal, torrent.SelectedFiles.Count(), t.GetKey(torrent)) + t.repairLog.Infof("Trying to assign %d links to the %d selected files of incomplete torrent %s", unassignedTotal, torrent.SelectedFiles.Count(), t.GetKey(torrent)) // handle torrents with incomplete links for selected files assignedCount := 0 @@ -336,7 +336,6 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool { return false // next unassigned link }) - // magnet:?xt=urn:btih:ba8720dde2472e650a87efbb78efb4fbcea3f7ee // empty/reset the unassigned links as we have assigned them already if unassignedTotal > 0 { @@ -352,9 +351,6 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool { } else { t.repairLog.Warnf("Torrent %s is rar'ed and we cannot repair it", t.GetKey(torrent)) newUnassignedLinks.IterCb(func(_ string, unassigned *realdebrid.Download) { - // if unassigned == nil { - // return - // } newFile := &File{ File: realdebrid.File{ ID: 0, @@ -380,10 +376,9 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool { } func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) (*realdebrid.TorrentInfo, error) { - // broken files means broken links - // if brokenFiles is not provided, we will redownload all files finalSelection := strings.Join(selection, ",") if len(selection) == 0 { + // if brokenFiles is not provided, we will redownload all files torrent.SelectedFiles.IterCb(func(_ string, file *File) { selection = append(selection, fmt.Sprintf("%d", file.ID)) }) @@ -404,6 +399,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) if prevState.Eq(newState) { return t.redownloadTorrent(torrent, selection) } + // sometimes, adding a new hash will encounter a timeout but the torrent is still added newTorrentID = t.latestState.FirstTorrentId } else { if strings.Contains(err.Error(), "infringing") { @@ -419,7 +415,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) } else if strings.Contains(err.Error(), "allowed") { t.markAsUnfixable(torrent, "torrent not allowed") } - return nil, fmt.Errorf("cannot add magnet of torrent %s (hash=%s): %v", t.GetKey(torrent), torrent.Hash, err) + return nil, fmt.Errorf("cannot add magnet: %v", err) } } @@ -435,20 +431,20 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) retries++ if retries > 10 { t.setToBinImmediately(newTorrentID) - return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): too many retries", t.GetKey(torrent), newTorrentID) + return nil, fmt.Errorf("cannot start redownloading: too many retries") } err = t.api.SelectTorrentFiles(newTorrentID, finalSelection) if err != nil { t.setToBinImmediately(newTorrentID) - return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): %v", t.GetKey(torrent), newTorrentID, err) + return nil, fmt.Errorf("cannot start redownloading: %v", err) } time.Sleep(2 * time.Second) info, err = t.api.GetTorrentInfo(newTorrentID) if err != nil { t.setToBinImmediately(newTorrentID) - return nil, fmt.Errorf("cannot get info on redownloaded torrent %s (id=%s) : %v", t.GetKey(torrent), newTorrentID, err) + return nil, fmt.Errorf("cannot get info on redownloaded : %v", err) } if info.Status == "magnet_conversion" { @@ -459,25 +455,16 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) } // documented status: magnet_error, magnet_conversion, waiting_files_selection, queued, downloading, downloaded, error, virus, compressing, uploading, dead - isOkStatus := false - okStatuses := []string{"downloading", "downloaded", "uploading", "queued", "compressing"} - // not compressing because we need playable files - for _, status := range okStatuses { - if info.Status == status { - isOkStatus = true - break - } - } - - if !isOkStatus { + okStatuses := mapset.NewSet("downloading", "downloaded", "uploading", "queued", "compressing") + if !okStatuses.Contains(info.Status) { t.setToBinImmediately(newTorrentID) - return nil, fmt.Errorf("the redownloaded torrent %s is in a non-OK state: %s", t.GetKey(torrent), info.Status) + return nil, fmt.Errorf("non-OK state: %s", info.Status) } // check if incorrect number of links if info.Progress == 100 && len(info.Links) != len(selection) { t.setToBinImmediately(newTorrentID) - return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), len(selection)) + return nil, fmt.Errorf("only got %d links but we need %d", len(info.Links), len(selection)) } t.repairLog.Infof("Redownloading torrent %s successful (id=%s, progress=%d)", t.GetKey(torrent), info.ID, info.Progress)