hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixers

Browse Source
This commit is contained in:
Ben Sarmiento
2024-05-23 19:29:16 +02:00
parent 2a5f12e37f
commit d03b59bb2a
10 changed files with 275 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
internal/dav/infuse.go
View File

@@ -15,19 +15,24 @@ import (
func ServeRootDirectoryForInfuse(torMgr *torrent.TorrentManager) ([]byte, error) {
var buf bytes.Buffer
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
directories := torMgr.DirectoryMap.Keys()
sort.Strings(directories)
for _, directory := range directories {
if strings.HasPrefix(directory, "int__") {
continue
}
buf.WriteString(dav.BaseDirectory(directory, ""))
}
if torMgr.Config.EnableDownloadMount() {
buf.WriteString(dav.BaseDirectory(config.DOWNLOADS, ""))
}
_, size := version.GetFile()
buf.WriteString(dav.File(version.FILE, size, ""))
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil
}
@@ -40,15 +45,18 @@ func ServeTorrentsListForInfuse(directory string, torMgr *torrent.TorrentManager
var buf bytes.Buffer
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
torrentNames := torrents.Keys()
sort.Strings(torrentNames)
for _, torrentName := range torrentNames {
tor, ok := torrents.Get(torrentName)
if !ok || tor.AllInProgress() {
if !ok {
continue
}
buf.WriteString(dav.BaseDirectory(torMgr.GetKey(tor), tor.Added))
}
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil
}
@@ -58,12 +66,14 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr
if !ok {
return nil, fmt.Errorf("cannot find directory %s", directory)
}
tor, ok := torrents.Get(torrentName)
if !ok {
return nil, fmt.Errorf("cannot find torrent %s", torrentName)
}
dirCfg := torMgr.Config.(*config.ZurgConfigV1).GetDirectoryConfig(directory)
biggestFileSize := int64(0)
if dirCfg.OnlyShowTheBiggestFile {
biggestFileSize = tor.ComputeBiggestFileSize()
@@ -71,8 +81,10 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr
var buf bytes.Buffer
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
filenames := tor.SelectedFiles.Keys()
sort.Strings(filenames)
for _, filename := range filenames {
file, _ := tor.SelectedFiles.Get(filename)
if !file.State.Is("ok_file") {
@@ -89,6 +101,7 @@ func ServeFilesListForInfuse(directory, torrentName string, torMgr *torrent.Torr
}
buf.WriteString(dav.File(filename, file.Bytes, tor.Added))
}
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil
}
@@ -99,9 +112,12 @@ func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error)
buf.WriteString("Enable download mount in config to use this feature")
return buf.Bytes(), nil
}
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
filenames := torMgr.DownloadMap.Keys()
sort.Strings(filenames)
for _, filename := range filenames {
download, ok := torMgr.DownloadMap.Get(filename)
if !ok {
@@ -109,6 +125,7 @@ func ServeDownloadsListForInfuse(torMgr *torrent.TorrentManager) ([]byte, error)
}
buf.WriteString(dav.File(download.Filename, download.Filesize, download.Generated))
}
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil
}

2
internal/dav/listing.go
View File

@@ -47,7 +47,7 @@ func ServeTorrentsList(directory string, torMgr *torrent.TorrentManager) ([]byte
sort.Strings(torrentNames)
for _, torrentName := range torrentNames {
tor, ok := torrents.Get(torrentName)
if !ok || tor.AllInProgress() {
if !ok {
continue
}
buf.WriteString(dav.Directory(torMgr.GetKey(tor), tor.Added))

2
internal/http/listing.go
View File

@@ -45,7 +45,7 @@ func ServeTorrentsList(directory string, torMgr *torrent.TorrentManager) ([]byte
sort.Strings(torrentNames)
for _, torrentName := range torrentNames {
tor, ok := torrents.Get(torrentName)
if !ok || tor.AllInProgress() {
if !ok {
continue
}
buf.WriteString(fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", filepath.Join(directory, url.PathEscape(torMgr.GetKey(tor))), torMgr.GetKey(tor)))

9
internal/torrent/delete.go
View File

@@ -1,6 +1,9 @@
package torrent
import cmap "github.com/orcaman/concurrent-map/v2"
import (
"github.com/debridmediamanager/zurg/pkg/realdebrid"
cmap "github.com/orcaman/concurrent-map/v2"
)
// CheckDeletedStatus checks if all files in a torrent are marked as deleted, if so, it returns true
func (t *TorrentManager) CheckDeletedStatus(torrent *Torrent) bool {
@@ -24,11 +27,11 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) {
if torrent, ok := allTorrents.Get(accessKey); ok {
hash = torrent.Hash
if deleteInRD {
for torrentID := range torrent.Components {
torrent.Components.IterCb(func(torrentID string, _ *realdebrid.TorrentInfo) {
t.log.Debugf("Deleting torrent %s (id=%s) in RD", accessKey, torrentID)
t.api.DeleteTorrent(torrentID)
t.deleteInfoFile(torrentID)
}
})
}
}
t.log.Infof("Removing torrent %s from zurg database (not real-debrid)", accessKey)

5
internal/torrent/manager.go
View File

@@ -168,7 +168,7 @@ func (t *TorrentManager) writeTorrentToFile(torrent *Torrent) {
return
}
t.log.Debugf("Saved torrent %s to file", torrent.Hash)
t.log.Debugf("Saved torrent %s (hash=%s) to file", t.GetKey(torrent), torrent.Hash)
}
func (t *TorrentManager) readTorrentFromFile(hash string) *Torrent {
@@ -189,9 +189,6 @@ func (t *TorrentManager) readTorrentFromFile(hash string) *Torrent {
if err := json.Unmarshal(jsonData, &torrent); err != nil {
return nil
}
if len(torrent.Components) == 0 {
t.log.Fatal("Torrent has no downloaded or in progress ids")
}
if torrent.Version != t.requiredVersion {
return nil
}

349
internal/torrent/refresh.go
View File

@@ -22,99 +22,112 @@ func (t *TorrentManager) refreshTorrents() []string {
return nil
}
t.log.Infof("Fetched %d torrents", len(instances))
torChan := make(chan *Torrent, len(instances))
var wg sync.WaitGroup
var wg sync.WaitGroup
var mergeChan = make(chan *Torrent, len(instances))
updatedPaths := mapset.NewSet[string]()
allTorrents, _ := t.DirectoryMap.Get(INT_ALL)
freshHashes := mapset.NewSet[string]()
freshIDs := mapset.NewSet[string]()
freshAccessKeys := mapset.NewSet[string]()
counter := 0
for i := range instances {
if t.trashBin.Contains(instances[i].ID) {
t.api.DeleteTorrent(instances[i].ID)
t.log.Infof("Skipping trashed torrent %s", instances[i].Name)
torChan <- nil
}
idx := i
wg.Add(1)
idx := i
_ = t.workerPool.Submit(func() {
defer wg.Done()
torChan <- t.getMoreInfo(instances[idx])
if t.trashBin.Contains(instances[idx].ID) {
t.api.DeleteTorrent(instances[idx].ID)
t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
counter++
mergeChan <- nil
return
}
if instances[idx].Progress != 100 {
t.log.Debugf("Skipping incomplete torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
counter++
mergeChan <- nil
return
}
freshHashes.Add(instances[idx].Hash)
freshIDs.Add(instances[idx].ID)
tInfo := t.getMoreInfo(instances[idx])
torrent := t.convertToTorrent(tInfo)
accessKey := t.GetKey(torrent)
freshAccessKeys.Add(accessKey)
var forMerging *Torrent
mainTorrent, exists := allTorrents.Get(accessKey)
if !exists {
allTorrents.Set(accessKey, torrent)
t.assignDirectory(torrent, func(directory string) {
listing, _ := t.DirectoryMap.Get(directory)
listing.Set(accessKey, torrent)
updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey))
})
} else if !mainTorrent.Components.Has(tInfo.ID) {
forMerging = torrent
}
counter++
mergeChan <- forMerging
})
}
wg.Wait()
close(torChan)
t.log.Infof("Fetched info for %d torrents", len(instances))
close(mergeChan)
t.log.Infof("Finished fetching info for %d torrents, proceeding to merge", counter)
var updatedPaths []string
noInfoCount := 0
allTorrents, _ := t.DirectoryMap.Get(INT_ALL)
freshAccessKeys := mapset.NewSet[string]()
allHashes := mapset.NewSet[string]()
t.getTorrentFiles().Each(func(path string) bool {
path = filepath.Base(path)
hash := strings.TrimSuffix(path, ".torrent_zurg")
allHashes.Add(hash)
return false
})
freshHashes := mapset.NewSet[string]()
allIDs := mapset.NewSet[string]()
t.getInfoFiles().Each(func(path string) bool {
path = filepath.Base(path)
torrentID := strings.TrimSuffix(path, ".info_zurg")
allIDs.Add(torrentID)
return false
})
freshIDs := mapset.NewSet[string]()
for torrent := range torChan {
for torrent := range mergeChan {
if torrent == nil {
noInfoCount++
continue
}
// there's only 1 component torrent at this point, let's get it
var tInfo *realdebrid.TorrentInfo
for _, tInfo = range torrent.Components {
break
}
t.log.Debugf("Merging %s", t.GetKey(torrent))
accessKey := t.GetKey(torrent)
freshAccessKeys.Add(accessKey)
freshHashes.Add(torrent.Hash)
freshIDs.Add(tInfo.ID)
// update allTorrents
isNewID := false
mainTorrent, exists := allTorrents.Get(accessKey)
if !exists {
allTorrents.Set(accessKey, torrent)
mainTorrent = torrent
isNewID = true
} else if _, ok := mainTorrent.Components[tInfo.ID]; !ok {
merged := t.mergeToMain(mainTorrent, torrent)
allTorrents.Set(accessKey, merged)
mainTorrent = merged
isNewID = true
existing, ok := allTorrents.Get(accessKey)
if !ok {
t.log.Warnf("Cannot merge %s", accessKey)
continue
}
mainTorrent := t.mergeTorrents(existing, torrent)
allTorrents.Set(accessKey, mainTorrent)
t.writeTorrentToFile(mainTorrent)
if isNewID && tInfo.Progress == 100 {
// assign to directory
t.assignedDirectoryCb(mainTorrent, func(directory string) {
listing, _ := t.DirectoryMap.Get(directory)
listing.Set(accessKey, mainTorrent)
updatedPaths = append(updatedPaths, fmt.Sprintf("%s/%s", directory, accessKey))
})
// write torrent to file
if !allHashes.Contains(mainTorrent.Hash) {
t.writeTorrentToFile(mainTorrent)
t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
if strings.HasPrefix(directory, "int__") {
return
}
}
torrents.Remove(accessKey)
})
t.assignDirectory(mainTorrent, func(directory string) {
listing, _ := t.DirectoryMap.Get(directory)
listing.Set(accessKey, mainTorrent)
updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey))
})
t.log.Debugf("Merging %s done!", t.GetKey(existing))
}
t.log.Infof("Fetched info for %d torrents", len(instances))
noInfoCount := 0
// removed torrents
allAccessKeys := mapset.NewSet[string](allTorrents.Keys()...)
allAccessKeys.Difference(freshAccessKeys).Each(func(accessKey string) bool {
oldPlusNewKeys := mapset.NewSet[string](allTorrents.Keys()...)
oldPlusNewKeys.Difference(freshAccessKeys).Each(func(accessKey string) bool {
t.Delete(accessKey, false)
return false
})
@@ -122,18 +135,32 @@ func (t *TorrentManager) refreshTorrents() []string {
t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
// data directory cleanup
allHashes.Difference(freshHashes).Each(func(hash string) bool {
existingHashes := mapset.NewSet[string]()
t.getTorrentFiles().Each(func(path string) bool {
path = filepath.Base(path)
hash := strings.TrimSuffix(path, ".torrent_zurg")
existingHashes.Add(hash)
return false
})
existingHashes.Difference(freshHashes).Each(func(hash string) bool {
t.log.Infof("Deleting stale torrent file %s", hash)
t.deleteTorrentFile(hash)
return false
})
allIDs.Difference(freshIDs).Each(func(id string) bool {
existingIDs := mapset.NewSet[string]()
t.getInfoFiles().Each(func(path string) bool {
path = filepath.Base(path)
torrentID := strings.TrimSuffix(path, ".info_zurg")
existingIDs.Add(torrentID)
return false
})
existingIDs.Difference(freshIDs).Each(func(id string) bool {
t.log.Infof("Deleting stale info file %s", id)
t.deleteInfoFile(id)
return false
})
return updatedPaths
return updatedPaths.ToSlice()
}
// StartRefreshJob periodically refreshes the torrents
@@ -165,16 +192,7 @@ func (t *TorrentManager) StartRefreshJob() {
}()
}
// getMoreInfo gets original name, size and files for a torrent
func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
diskTor := t.readTorrentFromFile(rdTorrent.Hash)
if diskTor != nil {
if diskInfo, ok := diskTor.Components[rdTorrent.ID]; ok && diskInfo.Progress == 100 {
diskTor.Components = map[string]*realdebrid.TorrentInfo{rdTorrent.ID: diskInfo}
return diskTor
}
}
func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *realdebrid.TorrentInfo {
info := t.readInfoFromFile(rdTorrent.ID)
if info == nil {
var err error
@@ -183,25 +201,40 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
t.log.Warnf("Cannot get info for id=%s: %v", rdTorrent.ID, err)
return nil
}
if info.Progress == 100 {
t.writeInfoToFile(info)
}
t.writeInfoToFile(info)
}
return info
}
func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent {
torrent := t.readTorrentFromFile(info.Hash)
if torrent != nil && torrent.Components.Has(info.ID) {
return torrent
}
torrent := Torrent{
torrent = &Torrent{
Name: info.Name,
OriginalName: info.OriginalName,
Added: info.Added,
Hash: info.Hash,
State: NewTorrentState("broken_torrent"),
Components: cmap.New[*realdebrid.TorrentInfo](),
}
// SelectedFiles is a subset of Files with only the selected ones
// it also has a Link field, which can be empty
// if it is empty, it means the file is no longer available
// Files+Links together are the same as SelectedFiles
allFilenames := mapset.NewSet[string]()
dupeFilenames := mapset.NewSet[string]()
var selectedFiles []*File
for _, file := range info.Files {
filename := filepath.Base(file.Path)
if allFilenames.Contains(filename) {
dupeFilenames.Add(filename)
} else {
allFilenames.Add(filename)
}
if file.Selected == 0 {
continue
}
@@ -216,6 +249,9 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
// all links are still intact! good!
for i, file := range selectedFiles {
file.Link = info.Links[i]
if strings.HasPrefix(file.Link, "https://real-debrid.com/d/") {
file.Link = file.Link[0:39]
}
if err := file.State.Event(context.Background(), "repair_file"); err != nil {
t.log.Errorf("Cannot repair file %s: %v", file.Path, err)
}
@@ -225,47 +261,34 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
t.log.Errorf("Cannot repair torrent %s: %v", torrent.Hash, err)
}
} else {
torrent.UnassignedLinks = mapset.NewSet[string](info.Links...)
torrent.UnassignedLinks = mapset.NewSet[string]()
for _, link := range info.Links {
if strings.HasPrefix(link, "https://real-debrid.com/d/") {
link = link[0:39]
}
torrent.UnassignedLinks.Add(link)
}
}
torrent.SelectedFiles = cmap.New[*File]()
for _, file := range selectedFiles {
filename := t.GetPath(file)
baseFilename := t.GetPath(file)
// todo better handling of duplicate filenames
if torrent.SelectedFiles.Has(filename) {
oldName := filename
ext := filepath.Ext(oldName)
noExtension := strings.TrimSuffix(oldName, ext)
newName := fmt.Sprintf("%s (%d)%s", noExtension, file.ID, ext)
if dupeFilenames.Contains(baseFilename) {
extension := filepath.Ext(baseFilename)
filenameNoExt := strings.TrimSuffix(baseFilename, extension)
newName := fmt.Sprintf("%s (%d)%s", filenameNoExt, file.ID, extension)
torrent.SelectedFiles.Set(newName, file)
} else {
torrent.SelectedFiles.Set(filename, file)
torrent.SelectedFiles.Set(baseFilename, file)
}
}
torrent.Components = map[string]*realdebrid.TorrentInfo{rdTorrent.ID: info}
return &torrent
torrent.Components.Set(info.ID, info)
return torrent
}
// ResetSelectedFiles resets the selected files for a torrent
func (t *TorrentManager) ResetSelectedFiles(torrent *Torrent) {
// reset selected files
newSelectedFiles := cmap.New[*File]()
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
filename := t.GetPath(file)
if newSelectedFiles.Has(filename) {
oldName := filename
ext := filepath.Ext(oldName)
noExtension := strings.TrimSuffix(oldName, ext)
newName := fmt.Sprintf("%s (%d)%s", noExtension, file.ID, ext)
newSelectedFiles.Set(newName, file)
} else {
newSelectedFiles.Set(filename, file)
}
})
torrent.SelectedFiles = newSelectedFiles
}
func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) *Torrent {
func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent {
var newer, older *Torrent
if existing.Added < toMerge.Added {
newer = toMerge
@@ -275,21 +298,25 @@ func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) *Torrent {
older = toMerge
}
mergedComponents := map[string]*realdebrid.TorrentInfo{}
for k, v := range older.Components {
mergedComponents[k] = v
}
for k, v := range newer.Components {
mergedComponents[k] = v
}
// components
numComponents := 0
mergedComponents := cmap.New[*realdebrid.TorrentInfo]()
older.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) {
numComponents++
mergedComponents.Set(torrentID, info)
})
newer.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) {
numComponents++
mergedComponents.Set(torrentID, info)
})
// build the main torrent
mainTorrent := Torrent{
Name: newer.Name,
OriginalName: newer.OriginalName,
Rename: newer.Rename,
Hash: newer.Hash,
Added: newer.Added,
// base of the merged torrent
mergedTorrent := Torrent{
Name: older.Name,
OriginalName: older.OriginalName,
Rename: older.Rename,
Hash: older.Hash,
Added: older.Added,
Components: mergedComponents,
State: older.State,
@@ -299,44 +326,58 @@ func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) *Torrent {
reasons := mapset.NewSet[string]()
reasons.Add(older.UnrepairableReason)
reasons.Add(newer.UnrepairableReason)
mainTorrent.UnrepairableReason = strings.Join(reasons.ToSlice(), ", ")
mergedTorrent.UnrepairableReason = strings.Join(reasons.ToSlice(), ", ")
mainTorrent.SelectedFiles = cmap.New[*File]()
older.SelectedFiles.IterCb(func(key string, olderFile *File) {
mainTorrent.SelectedFiles.Set(key, olderFile)
})
newer.SelectedFiles.IterCb(func(key string, newerFile *File) {
if f, ok := mainTorrent.SelectedFiles.Get(key); ok && f.State.Is("deleted_file") {
return
// selected files
mergedTorrent.SelectedFiles = cmap.New[*File]()
newer.SelectedFiles.IterCb(func(key string, file *File) {
if f, ok := mergedTorrent.SelectedFiles.Get(key); !ok || !f.State.Is("ok_file") {
mergedTorrent.SelectedFiles.Set(key, file)
}
})
older.SelectedFiles.IterCb(func(key string, file *File) {
if f, ok := mergedTorrent.SelectedFiles.Get(key); !ok || !f.State.Is("ok_file") {
mergedTorrent.SelectedFiles.Set(key, file)
}
mainTorrent.SelectedFiles.Set(key, newerFile)
})
t.CheckDeletedStatus(&mainTorrent)
// unassigned links
mainTorrent.UnassignedLinks = mapset.NewSet[string]()
newer.UnassignedLinks.Union(older.UnassignedLinks).Each(func(link string) bool {
mergedTorrent.UnassignedLinks = mapset.NewSet[string]()
links := newer.UnassignedLinks.Union(older.UnassignedLinks)
links.Each(func(link string) bool {
found := false
mainTorrent.SelectedFiles.IterCb(func(key string, file *File) {
mergedTorrent.SelectedFiles.IterCb(func(key string, file *File) {
if !found && file.Link == link {
found = true
return
}
})
if !found {
mainTorrent.UnassignedLinks.Add(link)
mergedTorrent.UnassignedLinks.Add(link)
}
return false
})
return &mainTorrent
brokenCount := 0
okCount := 0
wtfCount := 0
mergedTorrent.SelectedFiles.IterCb(func(key string, file *File) {
if file.State.Is("broken_file") {
brokenCount++
} else if file.State.Is("ok_file") {
okCount++
} else {
wtfCount++
}
})
// todo
t.log.Debugf("Merging %s (%d comps) - selected files: %d ; unassigned: %d ; broken: %d ; ok %d ; wtf %d", t.GetKey(&mergedTorrent), numComponents, brokenCount+okCount+wtfCount, mergedTorrent.UnassignedLinks.Cardinality(), brokenCount, okCount, wtfCount)
return &mergedTorrent
}
func (t *TorrentManager) assignedDirectoryCb(tor *Torrent, cb func(string)) {
torrentIDs := []string{}
for id := range tor.Components {
torrentIDs = append(torrentIDs, id)
}
func (t *TorrentManager) assignDirectory(tor *Torrent, cb func(string)) {
torrentIDs := tor.Components.Keys()
// get filenames needed for directory conditions
var filenames []string
var fileSizes []int64

56
internal/torrent/repair.go
View File

@@ -150,10 +150,7 @@ func (t *TorrentManager) Repair(torrent *Torrent, wg *sync.WaitGroup) {
}
func (t *TorrentManager) repair(torrent *Torrent) {
torrentIDs := []string{}
for id := range torrent.Components {
torrentIDs = append(torrentIDs, id)
}
torrentIDs := torrent.Components.Keys()
t.log.Infof("Started repair process for torrent %s (ids=%v)", t.GetKey(torrent), torrentIDs)
// handle torrents with incomplete links for selected files
@@ -177,12 +174,11 @@ func (t *TorrentManager) repair(torrent *Torrent) {
return
}
// delete old torrents
for id := range torrent.Components {
if id == info.ID {
continue
torrent.Components.IterCb(func(id string, _ *realdebrid.TorrentInfo) {
if id != info.ID {
t.api.DeleteTorrent(id)
}
t.api.DeleteTorrent(id)
}
})
t.log.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent))
return
} else if info != nil && info.Progress != 100 {
@@ -210,18 +206,18 @@ func (t *TorrentManager) repair(torrent *Torrent) {
return
}
t.log.Infof("Repairing by downloading %d batches of the %d broken files of torrent %s", int(math.Ceil(float64(len(brokenFiles))/130)), len(brokenFiles), t.GetKey(torrent))
t.log.Infof("Repairing by downloading %d batches of the %d broken files of torrent %s", int(math.Ceil(float64(len(brokenFiles))/100)), len(brokenFiles), t.GetKey(torrent))
newlyDownloadedIds := make([]string, 0)
group := make([]*File, 0)
batchNum := 1
for _, file := range brokenFiles {
group = append(group, file)
if len(group) >= 130 {
brokenFileIDs := getFileIDs(brokenFiles)
var group []string
for _, fileIDStr := range brokenFileIDs {
group = append(group, fileIDStr)
if len(group) >= 100 {
t.log.Debugf("Downloading batch %d of broken files of torrent %s", batchNum, t.GetKey(torrent))
batchNum++
brokenFileIDs := getFileIDs(group)
redownloadedInfo, err := t.redownloadTorrent(torrent, brokenFileIDs)
redownloadedInfo, err := t.redownloadTorrent(torrent, group)
if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds {
@@ -230,15 +226,14 @@ func (t *TorrentManager) repair(torrent *Torrent) {
return
}
newlyDownloadedIds = append(newlyDownloadedIds, redownloadedInfo.ID)
group = make([]*File, 0)
group = make([]string, 0)
}
}
t.log.Debugf("Downloading last batch of broken files of torrent %s", t.GetKey(torrent))
if len(group) > 0 {
brokenFileIDs := getFileIDs(group)
_, err := t.redownloadTorrent(torrent, brokenFileIDs)
_, err := t.redownloadTorrent(torrent, group)
if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds {
@@ -276,6 +271,9 @@ func (t *TorrentManager) assignUnassignedLinks(torrent *Torrent) bool {
// base it on size because why not?
if !assigned && file.State.Is("broken_file") && file.Bytes == unrestrict.Filesize && strings.HasSuffix(strings.ToLower(file.Path), strings.ToLower(unrestrict.Filename)) {
file.Link = link
if strings.HasPrefix(file.Link, "https://real-debrid.com/d/") {
file.Link = file.Link[0:39]
}
if err := file.State.Event(context.Background(), "repair_file"); err != nil {
t.log.Errorf("Failed to mark file %s as repaired: %v", file.Path, err)
return
@@ -351,17 +349,14 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
// broken files means broken links
// if brokenFiles is not provided, we will redownload all files
finalSelection := strings.Join(selection, ",")
selectionCount := len(selection)
if selectionCount == 0 {
tmpSelection := ""
if len(selection) == 0 {
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
tmpSelection += fmt.Sprintf("%d,", file.ID) // select all files
selectionCount++
selection = append(selection, fmt.Sprintf("%d", file.ID))
})
if tmpSelection == "" {
return nil, nil // nothing to repair
if len(selection) == 0 {
return nil, nil
} else {
finalSelection = tmpSelection[:len(tmpSelection)-1]
finalSelection = strings.Join(selection, ",")
}
}
@@ -434,9 +429,9 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
}
// check if incorrect number of links
if info.Progress == 100 && len(info.Links) != selectionCount {
if info.Progress == 100 && len(info.Links) != len(selection) {
t.trash(newTorrentID)
return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), selectionCount)
return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), len(selection))
}
t.log.Infof("Redownloading torrent %s successful (id=%s, progress=%d)", t.GetKey(torrent), info.ID, info.Progress)
@@ -544,6 +539,9 @@ func (t *TorrentManager) isStillBroken(info *realdebrid.TorrentInfo, brokenFiles
// all links are still intact! good!
for i, file := range selectedFiles {
file.Link = info.Links[i]
if strings.HasPrefix(file.Link, "https://real-debrid.com/d/") {
file.Link = file.Link[0:39]
}
if err := file.State.Event(context.Background(), "repair_file"); err != nil {
t.log.Errorf("Failed to mark file %s as repaired: %v", file.Path, err)
return true

51
internal/torrent/torrent_types.go
View File

@@ -14,17 +14,17 @@ import (
var json = jsoniter.ConfigCompatibleWithStandardLibrary
type Torrent struct {
Name string `json:"Name"`
OriginalName string `json:"OriginalName"`
Hash string `json:"Hash"`
Added string `json:"Added"`
Components map[string]*realdebrid.TorrentInfo `json:"Components"`
Name string `json:"Name"`
OriginalName string `json:"OriginalName"`
Hash string `json:"Hash"`
Added string `json:"Added"`
Components cmap.ConcurrentMap[string, *realdebrid.TorrentInfo] `json:"-"`
UnassignedLinks mapset.Set[string] `json:"UnassignedLinks"` // when links are not complete, we cannot assign them to a file so we store them here until it's fixed
Rename string `json:"Rename"` // modified over time
SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` // modified over time
UnrepairableReason string `json:"Unfixable"` // modified over time
Rename string `json:"Rename"`
SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"`
UnrepairableReason string `json:"Unfixable"`
State *fsm.FSM `json:"-"`
Version string `json:"Version"` // only used for files
@@ -33,6 +33,7 @@ type Torrent struct {
func (t *Torrent) MarshalJSON() ([]byte, error) {
type Alias Torrent
temp := &struct {
ComponentsJson stdjson.RawMessage `json:"Components"`
SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
StateJson stdjson.RawMessage `json:"State"`
@@ -54,6 +55,12 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
temp.UnassignedLinksJson = []byte(unassignedLinksStr)
}
componentsJson, err := t.Components.MarshalJSON()
if err != nil {
return nil, err
}
temp.ComponentsJson = componentsJson
temp.StateJson = []byte(`"` + t.State.Current() + `"`)
return json.Marshal(temp)
@@ -62,6 +69,7 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
func (t *Torrent) UnmarshalJSON(data []byte) error {
type Alias Torrent
temp := &struct {
ComponentsJson stdjson.RawMessage `json:"Components"`
SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
StateJson string `json:"State"`
@@ -73,6 +81,13 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
return err
}
t.Components = cmap.New[*realdebrid.TorrentInfo]()
if len(temp.ComponentsJson) > 0 {
if err := t.Components.UnmarshalJSON(temp.ComponentsJson); err != nil {
return err
}
}
t.SelectedFiles = cmap.New[*File]()
if len(temp.SelectedFilesJson) > 0 {
if err := t.SelectedFiles.UnmarshalJSON(temp.SelectedFilesJson); err != nil {
@@ -87,29 +102,11 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
t.UnassignedLinks = mapset.NewSet[string]()
}
t.State = NewFileState(temp.StateJson)
t.State = NewTorrentState(temp.StateJson)
return nil
}
func (t *Torrent) AnyInProgress() bool {
for _, info := range t.Components {
if info.Progress != 100 {
return true
}
}
return false
}
func (t *Torrent) AllInProgress() bool {
for _, info := range t.Components {
if info.Progress == 100 {
return false
}
}
return true
}
func (t *Torrent) ComputeTotalSize() int64 {
totalSize := int64(0)
t.SelectedFiles.IterCb(func(key string, value *File) {

4
internal/torrent/uncached.go
View File

@@ -17,7 +17,7 @@ func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) {
hashGroups = append(hashGroups, currentGroup)
allTorrents.IterCb(func(_ string, torrent *Torrent) {
if torrent.AnyInProgress() || torrent.AllInProgress() || torrent.UnrepairableReason != "" {
if torrent.UnrepairableReason != "" {
return
}
@@ -46,7 +46,7 @@ func (t *TorrentManager) GetUncachedTorrents() ([]*Torrent, error) {
var uncachedTorrents []*Torrent
allTorrents.IterCb(func(_ string, torrent *Torrent) {
if torrent.AnyInProgress() || torrent.AllInProgress() || torrent.UnrepairableReason != "" {
if torrent.UnrepairableReason != "" {
return
}

4
pkg/realdebrid/types.go
View File

@@ -50,10 +50,6 @@ type Torrent struct {
Added string `json:"-"`
}
func (i *Torrent) IsDone() bool {
return i.Progress == 100 && len(i.Links) > 0
}
func (i *Torrent) UnmarshalJSON(data []byte) error {
type Alias Torrent
aux := &struct {