hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix rewrites

Browse Source
This commit is contained in:
Ben Sarmiento
2024-05-22 02:26:20 +02:00
parent a9cc689702
commit 9990bf90ca
10 changed files with 149 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
internal/torrent/repair.go
View File

@@ -95,23 +95,25 @@ func (t *TorrentManager) repairAll(torrent *Torrent) {
toRepair := mapset.NewSet[*Torrent]()
haystack.IterCb(func(_ string, torrent *Torrent) {
if torrent.AnyInProgress() || torrent.AllInProgress() || torrent.UnrepairableReason != "" {
if torrent.UnrepairableReason != "" {
return
}
// check 1: for broken files
brokenFileIDs := mapset.NewSet[int]()
brokenFileCount := 0
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
if file.State.Is("broken_file") {
brokenFileIDs.Add(file.ID)
brokenFileCount++
}
})
if brokenFileIDs.Cardinality() > 0 {
if brokenFileCount > 0 {
t.log.Debugf("Torrent %s has %d broken files, adding to repair list", t.GetKey(torrent), brokenFileCount)
toRepair.Add(torrent)
return
}
// check 2: for unassigned links (this means the torrent has started to deteriorate)
if torrent.UnassignedLinks.Cardinality() > 0 {
t.log.Debugf("Torrent %s has unassigned links, adding to repair list", t.GetKey(torrent))
unassignedCount := torrent.UnassignedLinks.Cardinality()
if unassignedCount > 0 {
t.log.Debugf("Torrent %s has %d unassigned links, adding to repair list", t.GetKey(torrent), unassignedCount)
toRepair.Add(torrent)
return
}
@@ -125,21 +127,10 @@ func (t *TorrentManager) repairAll(torrent *Torrent) {
})
wg.Wait()
t.log.Infof("Finished repairing %d broken torrents", toRepair.Cardinality())
t.log.Infof("Finished repair sequence for %d broken torrents", toRepair.Cardinality())
}
func (t *TorrentManager) Repair(torrent *Torrent, wg *sync.WaitGroup) {
if torrent.UnrepairableReason != "" {
t.log.Warnf("Torrent %s is unfixable (%s), skipping repair", t.GetKey(torrent), torrent.UnrepairableReason)
wg.Done()
return
}
if torrent.AnyInProgress() || torrent.AllInProgress() {
t.log.Infof("Torrent %s is in progress, skipping repair until download is done", t.GetKey(torrent))
wg.Done()
return
}
// blocks for approx 45 minutes if active torrents are full
if !t.canCapacityHandle() {
t.log.Error("Blocked for too long due to limit of active torrents, cannot continue with the repair")
@@ -150,6 +141,10 @@ func (t *TorrentManager) Repair(torrent *Torrent, wg *sync.WaitGroup) {
// assign to a worker
_ = t.workerPool.Submit(func() {
defer wg.Done()
if err := torrent.State.Event(context.Background(), "repair_torrent"); err != nil {
t.log.Errorf("Failed to mark torrent %s as under repair: %v", t.GetKey(torrent), err)
return
}
t.repair(torrent)
})
}
@@ -163,27 +158,36 @@ func (t *TorrentManager) repair(torrent *Torrent) {
// handle torrents with incomplete links for selected files
// torrent can be rar'ed by RD, so we need to check for that
if !t.assignUnassignedLinks(torrent) {
if torrent.UnassignedLinks.Cardinality() > 0 && !t.assignUnassignedLinks(torrent) {
return
}
// get all broken files
brokenFiles, allBroken := getBrokenFiles(torrent)
brokenFileIDs := getFileIDs(brokenFiles)
t.log.Debugf("Torrent %s has %d broken files (ids=%s; total is %d), repairing by redownloading", t.GetKey(torrent), len(brokenFiles), brokenFileIDs, torrent.SelectedFiles.Count())
// first step: redownload the whole torrent
info, err := t.redownloadTorrent(torrent, "") // reinsert the whole torrent, passing ""
if info != nil && info.Progress != 100 {
t.log.Infof("Torrent %s (files=%s) is still in progress after redownloading but it should be repaired once done", t.GetKey(torrent), brokenFileIDs)
t.log.Debugf("Torrent %s has %d broken files (out of %d), repairing by redownloading whole torrent", t.GetKey(torrent), len(brokenFiles), torrent.SelectedFiles.Count())
info, err := t.redownloadTorrent(torrent, []string{}) // reinsert the whole torrent, passing empty selection
if info != nil && info.Progress == 100 && !t.isStillBroken(info, brokenFiles) {
// successful repair
if err := torrent.State.Event(context.Background(), "mark_as_repaired"); err != nil {
t.log.Errorf("Cannot repair torrent %s: %v", torrent.Hash, err)
return
}
t.log.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent))
return
} else if info != nil && info.Progress == 100 && !t.isStillBroken(info, brokenFiles) {
t.log.Infof("Successfully repaired torrent %s (files=%s) by redownloading", t.GetKey(torrent), brokenFileIDs)
} else if info != nil && info.Progress != 100 {
t.log.Infof("Torrent %s is still in progress after redownloading but it should be repaired once done", t.GetKey(torrent))
return
}
t.log.Warnf("Cannot repair torrent %s by redownloading all files (error=%s)", t.GetKey(torrent), err.Error())
if err != nil {
t.log.Warnf("Cannot repair torrent %s by redownloading all files (error=%s)", t.GetKey(torrent), err.Error())
} else {
t.log.Warnf("Cannot repair torrent %s by redownloading all files", t.GetKey(torrent))
}
if torrent.UnrepairableReason != "" {
t.log.Debugf("Torrent %s has been marked as unfixable during redownload (%s), ending repair process early", t.GetKey(torrent), torrent.UnrepairableReason)
@@ -193,41 +197,29 @@ func (t *TorrentManager) repair(torrent *Torrent) {
// second step: download just the broken files
if len(brokenFiles) == 1 && allBroken {
// if all files are broken, we can't do anything
t.log.Warnf("Torrent %s has only 1 cached file and it's broken, marking as unfixable", t.GetKey(torrent))
// if all files are broken, we can't do anything!
t.log.Warnf("Torrent %s has only 1 cached file and it's broken; marking as unfixable (to fix, select other files)", t.GetKey(torrent))
t.markAsUnfixable(torrent, "the lone cached file is broken")
return
}
} else if len(brokenFiles) > 1 {
t.log.Infof("Repairing by downloading 2 batches of the %d broken files of torrent %s", len(brokenFiles), t.GetKey(torrent))
t.log.Infof("Repairing by downloading %d batches of the %d broken files of torrent %s", int(math.Ceil(float64(len(brokenFiles))/130)), len(brokenFiles), t.GetKey(torrent))
oldTorrentIDs := []string{}
for id := range torrent.Components {
oldTorrentIDs = append(torrentIDs, id)
}
oldTorrentIDs := []string{}
for id := range torrent.Components {
oldTorrentIDs = append(oldTorrentIDs, id)
}
newlyDownloadedIds := make([]string, 0)
group := make([]*File, 0)
for _, file := range brokenFiles {
group = append(group, file)
if len(group) >= 200 {
brokenFileIDs := getFileIDs(group)
redownloadedInfo, err := t.redownloadTorrent(torrent, brokenFileIDs)
if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds {
t.registerFixer(newId, "download_failed")
}
return
}
newlyDownloadedIds = append(newlyDownloadedIds, redownloadedInfo.ID)
group = make([]*File, 0)
}
}
if len(group) > 0 {
newlyDownloadedIds := make([]string, 0)
group := make([]*File, 0)
batchNum := 1
for _, file := range brokenFiles {
group = append(group, file)
if len(group) >= 130 {
t.log.Debugf("Downloading batch %d of broken files of torrent %s", batchNum, t.GetKey(torrent))
batchNum++
brokenFileIDs := getFileIDs(group)
_, err := t.redownloadTorrent(torrent, brokenFileIDs)
redownloadedInfo, err := t.redownloadTorrent(torrent, brokenFileIDs)
if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds {
@@ -235,13 +227,28 @@ func (t *TorrentManager) repair(torrent *Torrent) {
}
return
}
}
for _, oldId := range oldTorrentIDs {
t.registerFixer(oldId, "replaced")
newlyDownloadedIds = append(newlyDownloadedIds, redownloadedInfo.ID)
group = make([]*File, 0)
}
}
t.log.Debugf("Downloading last batch of broken files of torrent %s", t.GetKey(torrent))
if len(group) > 0 {
brokenFileIDs := getFileIDs(group)
_, err := t.redownloadTorrent(torrent, brokenFileIDs)
if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds {
t.registerFixer(newId, "download_failed")
}
return
}
}
for _, oldId := range oldTorrentIDs {
t.registerFixer(oldId, "replaced")
}
}
func (t *TorrentManager) assignUnassignedLinks(torrent *Torrent) bool {
@@ -267,10 +274,9 @@ func (t *TorrentManager) assignUnassignedLinks(torrent *Torrent) bool {
assigned := false
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
// base it on size because why not?
if !assigned && file.State.Is("broken_file") && ((unrestrict.Filesize > 1_000_000 && file.Bytes == unrestrict.Filesize) || strings.HasSuffix(strings.ToLower(file.Path), strings.ToLower(unrestrict.Filename))) {
if !assigned && file.State.Is("broken_file") && file.Bytes == unrestrict.Filesize && strings.HasSuffix(strings.ToLower(file.Path), strings.ToLower(unrestrict.Filename)) {
file.Link = link
err := file.State.Event(context.Background(), "repair_file")
if err != nil {
if err := file.State.Event(context.Background(), "repair_file"); err != nil {
t.log.Errorf("Failed to mark file %s as repaired: %v", file.Path, err)
return
}
@@ -331,6 +337,9 @@ func (t *TorrentManager) assignUnassignedLinks(torrent *Torrent) bool {
torrent.UnassignedLinks = mapset.NewSet[string]()
t.markAsUnfixable(torrent, "rar'ed by RD")
t.markAsUnplayable(torrent, "rar'ed by RD")
if err := torrent.State.Event(context.Background(), "mark_as_repaired"); err != nil {
t.log.Errorf("Cannot repair torrent %s: %v", torrent.Hash, err)
}
}
return false // end repair
}
@@ -338,11 +347,13 @@ func (t *TorrentManager) assignUnassignedLinks(torrent *Torrent) bool {
return true
}
func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection string) (*realdebrid.TorrentInfo, error) {
func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string) (*realdebrid.TorrentInfo, error) {
// broken files means broken links
// if brokenFiles is not provided, we will redownload all files
oldTorrentIDs := make([]string, 0)
if selection == "" {
finalSelection := strings.Join(selection, ",")
selectionCount := len(selection)
if selectionCount == 0 {
// only delete the old torrent if we are redownloading all files
for id := range torrent.Components {
oldTorrentIDs = append(oldTorrentIDs, id)
@@ -350,11 +361,12 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection string) (
tmpSelection := ""
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
tmpSelection += fmt.Sprintf("%d,", file.ID) // select all files
selectionCount++
})
if tmpSelection == "" {
return nil, nil // nothing to repair
} else {
selection = tmpSelection[:len(tmpSelection)-1]
finalSelection = tmpSelection[:len(tmpSelection)-1]
}
}
@@ -380,18 +392,18 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection string) (
newTorrentID := resp.ID
// sleep for 1 second to let RD process the magnet
time.Sleep(10 * time.Second)
time.Sleep(1 * time.Second)
var info *realdebrid.TorrentInfo
for {
// select files
err = t.api.SelectTorrentFiles(newTorrentID, selection)
err = t.api.SelectTorrentFiles(newTorrentID, finalSelection)
if err != nil {
t.registerFixer(newTorrentID, "download_failed")
return nil, fmt.Errorf("cannot start redownloading: %v", err)
return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): %v", t.GetKey(torrent), newTorrentID, err)
}
// sleep for 5 second to let RD process the magnet
time.Sleep(10 * time.Second)
// sleep for 2 second to let RD process the magnet
time.Sleep(2 * time.Second)
// see if the torrent is ready
info, err = t.api.GetTorrentInfo(newTorrentID)
@@ -400,9 +412,14 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection string) (
return nil, fmt.Errorf("cannot get info on redownloaded torrent %s (id=%s) : %v", t.GetKey(torrent), newTorrentID, err)
}
if info.Status != "magnet_conversion" && info.Status != "waiting_files_selection" {
break
if info.Status == "magnet_conversion" {
time.Sleep(60 * time.Second)
continue
} else if info.Status == "waiting_files_selection" {
time.Sleep(10 * time.Second)
return nil, fmt.Errorf("torrent %s (id=%s) is stuck on waiting_files_selection", t.GetKey(torrent), newTorrentID)
}
break
}
// documented status: magnet_error, magnet_conversion, waiting_files_selection, queued, downloading, downloaded, error, virus, compressing, uploading, dead
@@ -422,7 +439,6 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection string) (
}
// check if incorrect number of links
selectionCount := len(strings.Split(selection, ","))
if info.Progress == 100 && len(info.Links) != selectionCount {
t.registerFixer(newTorrentID, "download_failed")
return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), selectionCount)
@@ -484,8 +500,7 @@ func (t *TorrentManager) markAsUnplayable(torrent *Torrent, reason string) {
return
}
t.log.Warnf("Marking torrent %s as unplayable - %s", t.GetKey(torrent), reason)
err := torrent.State.Event(context.Background(), "mark_as_unplayable_torrent")
if err != nil {
if err := torrent.State.Event(context.Background(), "mark_as_unplayable_torrent"); err != nil {
t.log.Errorf("Failed to mark torrent %s as unplayable: %v", t.GetKey(torrent), err)
return
}
@@ -537,8 +552,7 @@ func (t *TorrentManager) isStillBroken(info *realdebrid.TorrentInfo, brokenFiles
// all links are still intact! good!
for i, file := range selectedFiles {
file.Link = info.Links[i]
err := file.State.Event(context.Background(), "repair_file")
if err != nil {
if err := file.State.Event(context.Background(), "repair_file"); err != nil {
t.log.Errorf("Failed to mark file %s as repaired: %v", file.Path, err)
return true
}