hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bin error checks

Browse Source
This commit is contained in:
Ben Sarmiento
2024-05-27 06:25:55 +02:00
parent fb37fd69e1
commit ac89f53896
4 changed files with 55 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/torrent/bins.go
+23 -17
View File
@@ -1,6 +1,7 @@
package torrent
import (
"fmt"
"os"
"strings"
@@ -64,8 +65,29 @@ func (t *TorrentManager) binImmediately(torrentId string) bool {
return false
}
func (t *TorrentManager) binOnceDoneErrorCheck(torrentId, status string) bool {
okStatuses := mapset.NewSet("downloading", "downloaded", "uploading", "queued", "compressing")
if !okStatuses.Contains(status) {
return t.binOnceDone(torrentId)
}
return false
}
func (t *TorrentManager) binOnceDone(torrentId string) bool {
if t.OnceDoneBin.Contains(torrentId) {
found := false
binnedIDs := t.OnceDoneBin.ToSlice()
// special case: xxx-yyy means if xxx is done, delete yyy
specialCase := fmt.Sprintf("%s-", torrentId)
for _, entry := range binnedIDs {
if strings.Contains(entry, specialCase) {
idToDelete := strings.Split(entry, "-")[1]
if err := t.api.DeleteTorrent(idToDelete); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", idToDelete, err)
continue
}
t.OnceDoneBin.Remove(entry)
found = true
} else if entry == torrentId {
if err := t.api.DeleteTorrent(torrentId); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err)
return false
@@ -73,22 +95,6 @@ func (t *TorrentManager) binOnceDone(torrentId string) bool {
t.OnceDoneBin.Remove(torrentId)
return true
}
// special case: xxx-yyy means if xxx is done, delete yyy
found := false
specialCases := t.OnceDoneBin.ToSlice()
for _, specialCase := range specialCases {
if strings.Contains(specialCase, "-") {
parts := strings.Split(specialCase, "-")
if parts[0] == torrentId {
if err := t.api.DeleteTorrent(parts[1]); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", parts[1], err)
continue
}
t.OnceDoneBin.Remove(specialCase)
found = true
}
}
}
return found
internal/torrent/delete.go
+1 -1
View File
@@ -35,7 +35,7 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) {
})
}
}
t.log.Infof("Removing torrent %s from zurg database (not real-debrid)", accessKey)
// t.log.Infof("Removing torrent %s from zurg database (not real-debrid)", accessKey)
t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
torrents.Remove(accessKey)
})
internal/torrent/refresh.go
+8 -27
View File
@@ -32,28 +32,14 @@ func (t *TorrentManager) refreshTorrents() []string {
freshIDs := mapset.NewSet[string]()
freshAccessKeys := mapset.NewSet[string]()
t.getTorrentFiles("dump").Each(func(filePath string) bool {
cachedAccessKeys := mapset.NewSet[string]()
t.getTorrentFiles("data").Each(func(filePath string) bool {
torrent := t.readTorrentFromFile(filePath)
if torrent != nil {
accessKey := t.GetKey(torrent)
if !allTorrents.Has(accessKey) {
t.log.Debugf("Loading dumped torrent %s", accessKey)
allTorrents.Set(accessKey, torrent)
t.assignDirectory(torrent, func(directory string) {
listing, _ := t.DirectoryMap.Get(directory)
listing.Set(accessKey, torrent)
// note that we're not adding it to updatedPaths
})
allTorrents.Set(t.GetKey(torrent), torrent)
filename := filepath.Base(filePath)
cachedAccessKeys.Add(strings.TrimSuffix(filename, ".zurgtorrent"))
}
freshAccessKeys.Add(accessKey) // to prevent being deleted
}
return false
})
cachedAccessKeys := mapset.NewSet[string]()
t.getTorrentFiles("data").Each(func(path string) bool {
path = filepath.Base(path)
cachedAccessKeys.Add(strings.TrimSuffix(path, ".zurgtorrent"))
return false
})
@@ -62,14 +48,9 @@ func (t *TorrentManager) refreshTorrents() []string {
idx := i
_ = t.workerPool.Submit(func() {
defer wg.Done()
if t.binImmediately(instances[idx].ID) {
// t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
mergeChan <- nil
return
}
if instances[idx].Progress != 100 {
// t.log.Debugf("Skipping incomplete torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
if t.binImmediately(instances[idx].ID) ||
t.binOnceDoneErrorCheck(instances[idx].ID, instances[idx].Status) ||
instances[idx].Progress != 100 {
mergeChan <- nil
return
}
internal/torrent/repair.go
+20 -33
View File
@@ -192,7 +192,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
if info != nil && info.Progress == 100 && !t.isStillBroken(info, brokenFiles) {
// successful repair
torrent.State.Event(context.Background(), "mark_as_repaired")
t.repairLog.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent))
t.repairLog.Infof("Successfully repaired torrent %s by redownloading whole torrent", t.GetKey(torrent))
// delete the torrents it replaced
torrent.DownloadedIDs.Each(func(torrentID string) bool {
if torrentID != info.ID {
@@ -202,7 +202,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
})
return
} else if info != nil && info.Progress != 100 {
t.repairLog.Infof("Torrent %s is still in progress after redownloading but it should be repaired once done", t.GetKey(torrent))
t.repairLog.Infof("Torrent %s is still in progress after redownloading whole torrent but it should be repaired once done", t.GetKey(torrent))
// once info.ID is done, we can delete the old torrent
torrent.DownloadedIDs.Each(func(torrentID string) bool {
if torrentID != info.ID {
@@ -214,13 +214,13 @@ func (t *TorrentManager) repair(torrent *Torrent) {
}
if err != nil {
t.repairLog.Warnf("Cannot repair torrent %s by redownloading all files (error=%v)", t.GetKey(torrent), err)
t.repairLog.Warnf("Cannot repair torrent %s by redownloading whole torrent (error=%v)", t.GetKey(torrent), err)
} else {
t.repairLog.Warnf("Cannot repair torrent %s by redownloading all files", t.GetKey(torrent))
t.repairLog.Warnf("Cannot repair torrent %s by redownloading whole torrent (links are still broken)", t.GetKey(torrent))
}
if torrent.UnrepairableReason != "" {
t.repairLog.Debugf("Torrent %s has been marked as unfixable during redownload (%s), ending repair process early", t.GetKey(torrent), torrent.UnrepairableReason)
t.repairLog.Debugf("Torrent %s has been marked as unfixable during redownloading whole torrent (%s), ending repair process early", t.GetKey(torrent), torrent.UnrepairableReason)
return
}
@@ -233,7 +233,8 @@ func (t *TorrentManager) repair(torrent *Torrent) {
return
}
t.repairLog.Infof("Torrent %s will be repaired by downloading %d batches of the %d broken files", t.GetKey(torrent), int(math.Ceil(float64(len(brokenFiles))/100)), len(brokenFiles))
totalBatches := int(math.Ceil(float64(len(brokenFiles)) / 100))
t.repairLog.Infof("Torrent %s will be repaired by downloading %d batches of the %d broken files", t.GetKey(torrent), totalBatches, len(brokenFiles))
newlyDownloadedIds := make([]string, 0)
batchNum := 1
@@ -242,7 +243,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
for _, fileIDStr := range brokenFileIDs {
group = append(group, fileIDStr)
if len(group) >= 100 {
t.repairLog.Debugf("Downloading batch %d of broken files of torrent %s", batchNum, t.GetKey(torrent))
t.repairLog.Debugf("Downloading batch %d/%d of broken files of torrent %s", batchNum, totalBatches, t.GetKey(torrent))
batchNum++
redownloadedInfo, err := t.redownloadTorrent(torrent, group)
if err != nil {
@@ -258,9 +259,8 @@ func (t *TorrentManager) repair(torrent *Torrent) {
}
}
t.repairLog.Debugf("Downloading last batch of broken files of torrent %s", t.GetKey(torrent))
if len(group) > 0 {
t.repairLog.Debugf("Downloading batch %d/%d of broken files of torrent %s", batchNum, totalBatches, t.GetKey(torrent))
redownloadedInfo, err := t.redownloadTorrent(torrent, group)
if err != nil {
t.repairLog.Warnf("Cannot repair torrent %s by downloading broken files (error=%v) giving up", t.GetKey(torrent), err)
@@ -281,7 +281,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
func (t *TorrentManager) assignLinks(torrent *Torrent) bool {
unassignedTotal := torrent.UnassignedLinks.Cardinality()
t.repairLog.Infof("Trying to assign %d links to the %d selected of incomplete torrent %s", unassignedTotal, torrent.SelectedFiles.Count(), t.GetKey(torrent))
t.repairLog.Infof("Trying to assign %d links to the %d selected files of incomplete torrent %s", unassignedTotal, torrent.SelectedFiles.Count(), t.GetKey(torrent))
// handle torrents with incomplete links for selected files
assignedCount := 0
@@ -336,7 +336,6 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool {
return false // next unassigned link
})
// magnet:?xt=urn:btih:ba8720dde2472e650a87efbb78efb4fbcea3f7ee
// empty/reset the unassigned links as we have assigned them already
if unassignedTotal > 0 {
@@ -352,9 +351,6 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool {
} else {
t.repairLog.Warnf("Torrent %s is rar'ed and we cannot repair it", t.GetKey(torrent))
newUnassignedLinks.IterCb(func(_ string, unassigned *realdebrid.Download) {
// if unassigned == nil {
// return
// }
newFile := &File{
File: realdebrid.File{
ID: 0,
@@ -380,10 +376,9 @@ func (t *TorrentManager) assignLinks(torrent *Torrent) bool {
}
func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) (*realdebrid.TorrentInfo, error) {
// broken files means broken links
// if brokenFiles is not provided, we will redownload all files
finalSelection := strings.Join(selection, ",")
if len(selection) == 0 {
// if brokenFiles is not provided, we will redownload all files
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
selection = append(selection, fmt.Sprintf("%d", file.ID))
})
@@ -404,6 +399,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
if prevState.Eq(newState) {
return t.redownloadTorrent(torrent, selection)
}
// sometimes, adding a new hash will encounter a timeout but the torrent is still added
newTorrentID = t.latestState.FirstTorrentId
} else {
if strings.Contains(err.Error(), "infringing") {
@@ -419,7 +415,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
} else if strings.Contains(err.Error(), "allowed") {
t.markAsUnfixable(torrent, "torrent not allowed")
}
return nil, fmt.Errorf("cannot add magnet of torrent %s (hash=%s): %v", t.GetKey(torrent), torrent.Hash, err)
return nil, fmt.Errorf("cannot add magnet: %v", err)
}
}
@@ -435,20 +431,20 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
retries++
if retries > 10 {
t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): too many retries", t.GetKey(torrent), newTorrentID)
return nil, fmt.Errorf("cannot start redownloading: too many retries")
}
err = t.api.SelectTorrentFiles(newTorrentID, finalSelection)
if err != nil {
t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): %v", t.GetKey(torrent), newTorrentID, err)
return nil, fmt.Errorf("cannot start redownloading: %v", err)
}
time.Sleep(2 * time.Second)
info, err = t.api.GetTorrentInfo(newTorrentID)
if err != nil {
t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("cannot get info on redownloaded torrent %s (id=%s) : %v", t.GetKey(torrent), newTorrentID, err)
return nil, fmt.Errorf("cannot get info on redownloaded : %v", err)
}
if info.Status == "magnet_conversion" {
@@ -459,25 +455,16 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
}
// documented status: magnet_error, magnet_conversion, waiting_files_selection, queued, downloading, downloaded, error, virus, compressing, uploading, dead
isOkStatus := false
okStatuses := []string{"downloading", "downloaded", "uploading", "queued", "compressing"}
// not compressing because we need playable files
for _, status := range okStatuses {
if info.Status == status {
isOkStatus = true
break
}
}
if !isOkStatus {
okStatuses := mapset.NewSet("downloading", "downloaded", "uploading", "queued", "compressing")
if !okStatuses.Contains(info.Status) {
t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("the redownloaded torrent %s is in a non-OK state: %s", t.GetKey(torrent), info.Status)
return nil, fmt.Errorf("non-OK state: %s", info.Status)
}
// check if incorrect number of links
if info.Progress == 100 && len(info.Links) != len(selection) {
t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), len(selection))
return nil, fmt.Errorf("only got %d links but we need %d", len(info.Links), len(selection))
}
t.repairLog.Infof("Redownloading torrent %s successful (id=%s, progress=%d)", t.GetKey(torrent), info.ID, info.Progress)