hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor file links and merges

Browse Source
This commit is contained in:
Ben Sarmiento
2024-01-27 17:38:26 +01:00
parent 1913498cbe
commit 1aabcfd322
5 changed files with 59 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/torrent/manager.go
View File

@@ -54,7 +54,7 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
deleteOnceDone: mapset.NewSet[string](),
allAccessKeys: mapset.NewSet[string](),
latestState: &LibraryState{},
requiredVersion: "27.01.2024",
requiredVersion: "0.9.3-hotfix.3",
workerPool: workerPool,
repairPool: repairPool,
log: log,

81
internal/torrent/refresh.go
View File

@@ -231,7 +231,6 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
} else {
torrent.InProgressIDs.Add(info.ID)
}
torrent.BrokenLinks = mapset.NewSet[string]()
infoCache.Set(rdTorrent.ID, &torrent)
t.saveTorrentChangesToDisk(&torrent, nil)
@@ -240,61 +239,63 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
}
func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) Torrent {
var newer, older *Torrent
if existing.Added < toMerge.Added {
newer = toMerge
older = existing
} else {
newer = existing
older = toMerge
}
// build the main torrent
mainTorrent := Torrent{
Name: existing.Name,
OriginalName: existing.OriginalName,
Rename: existing.Rename,
Hash: existing.Hash,
DownloadedIDs: mapset.NewSet[string](),
InProgressIDs: mapset.NewSet[string](),
// UnassignedLinks: mapset.NewSet[string](),
UnassignedLinks: existing.UnassignedLinks.Union(toMerge.UnassignedLinks),
BrokenLinks: existing.BrokenLinks.Union(toMerge.BrokenLinks),
Name: newer.Name,
OriginalName: newer.OriginalName,
Rename: newer.Rename,
Hash: newer.Hash,
Added: newer.Added,
DownloadedIDs: newer.DownloadedIDs.Union(older.DownloadedIDs),
InProgressIDs: newer.InProgressIDs.Union(older.InProgressIDs),
UnassignedLinks: newer.UnassignedLinks.Union(older.UnassignedLinks),
SelectedFiles: newer.SelectedFiles,
}
// unrepairable reason
if existing.UnrepairableReason != "" && toMerge.UnrepairableReason != "" && existing.UnrepairableReason != toMerge.UnrepairableReason {
mainTorrent.UnrepairableReason = fmt.Sprintf("%s, %s", existing.UnrepairableReason, toMerge.UnrepairableReason)
} else if existing.UnrepairableReason != "" {
mainTorrent.UnrepairableReason = existing.UnrepairableReason
} else if toMerge.UnrepairableReason != "" {
mainTorrent.UnrepairableReason = toMerge.UnrepairableReason
if newer.UnrepairableReason != "" && older.UnrepairableReason != "" && newer.UnrepairableReason != older.UnrepairableReason {
mainTorrent.UnrepairableReason = fmt.Sprintf("%s, %s", newer.UnrepairableReason, older.UnrepairableReason)
} else if newer.UnrepairableReason != "" {
mainTorrent.UnrepairableReason = newer.UnrepairableReason
} else if older.UnrepairableReason != "" {
mainTorrent.UnrepairableReason = older.UnrepairableReason
}
// this function triggers only when we have a new DownloadedID
toMerge.DownloadedIDs.Difference(existing.DownloadedIDs).Each(func(id string) bool {
mainTorrent.DownloadedIDs.Add(id)
// update in progress ids
mainTorrent.DownloadedIDs.Each(func(id string) bool {
mainTorrent.InProgressIDs.Remove(id)
return false
})
// the link can have the following values
// 1. https://*** - the file is available
// 2. repair - the link is there but we need to repair it
// 3. unselect - the file is deleted
// 4. empty - the file is not available
mainTorrent.SelectedFiles = existing.SelectedFiles
toMerge.SelectedFiles.IterCb(func(filepath string, fileToMerge *File) {
// see if it already exists in the main torrent
if originalFile, ok := mainTorrent.SelectedFiles.Get(filepath); !ok || fileToMerge.Link == "unselect" {
// if it doesn't exist in the main torrent, add it
mainTorrent.SelectedFiles.Set(filepath, fileToMerge)
} else if originalFile.Link != "unselect" {
// if it exists, compare the Added property and the link
if existing.Added < toMerge.Added {
// && strings.HasPrefix(fileToMerge.Link, "http")
// if torrentToMerge is more recent and its file has a link, update the main torrent's file
mainTorrent.SelectedFiles.Set(filepath, fileToMerge)
// 2. unselect - the file is deleted
// 3. empty - the file is not available
mainTorrent.SelectedFiles.IterCb(func(key string, file *File) {
if file.Link == "" {
file, ok := older.SelectedFiles.Get(key)
if ok {
mainTorrent.SelectedFiles.Set(key, file)
}
// else do nothing, the main torrent's file is more recent or has a valid link
}
})
if existing.Added < toMerge.Added {
mainTorrent.Added = toMerge.Added
} else {
mainTorrent.Added = existing.Added
older.SelectedFiles.IterCb(func(key string, file *File) {
if !mainTorrent.SelectedFiles.Has(key) {
mainTorrent.SelectedFiles.Set(key, file)
} else if file.Link == "unselect" {
mainTorrent.SelectedFiles.Set(key, file)
}
})
return mainTorrent
}

35
internal/torrent/repair.go
View File

@@ -78,7 +78,7 @@ func (t *TorrentManager) repairAll(torrent *Torrent) {
allTorrents, _ = t.DirectoryMap.Get(INT_ALL)
} else {
allTorrents = cmap.New[*Torrent]()
allTorrents.Set(t.GetKey(torrent), torrent)
allTorrents.Set("", torrent)
}
// collect all torrents that need to be repaired
@@ -89,29 +89,6 @@ func (t *TorrentManager) repairAll(torrent *Torrent) {
return
}
// save the broken files to the file cache
// broken files are also added when trying to open a file
if torrent.BrokenLinks.Cardinality() > 0 {
t.saveTorrentChangesToDisk(torrent, func(info *Torrent) {
hasBrokenFiles := false
info.SelectedFiles.IterCb(func(_ string, file *File) {
torrent.BrokenLinks.Each(func(brokenLink string) bool {
if file.Link == brokenLink {
hasBrokenFiles = true
file.Link = ""
return true
}
return false
})
})
if hasBrokenFiles {
info.BrokenLinks = torrent.BrokenLinks
} else {
info.BrokenLinks = mapset.NewSet[string]()
}
})
}
// check 2: for broken files
brokenFileIDs := mapset.NewSet[int]()
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
@@ -204,10 +181,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
file.Link = info.Links[ix]
ix++
})
torrent.BrokenLinks = mapset.NewSet[string]()
t.saveTorrentChangesToDisk(torrent, func(info *Torrent) {
info.BrokenLinks = mapset.NewSet[string]()
})
t.saveTorrentChangesToDisk(torrent, nil)
t.log.Infof("Successfully repaired torrent %s using repair_method#1", t.GetKey(torrent))
return
}
@@ -554,10 +528,7 @@ func (t *TorrentManager) handleFixers(fixer realdebrid.Torrent) *Torrent {
}
}
})
torrent.BrokenLinks = mapset.NewSet[string]()
t.saveTorrentChangesToDisk(torrent, func(info *Torrent) {
info.BrokenLinks = mapset.NewSet[string]()
})
t.saveTorrentChangesToDisk(torrent, nil)
t.log.Infof("Successfully repaired torrent %s using repair_method#2", t.GetKey(torrent))
} else {
t.log.Warnf("repair_method#2: Fixer is done but torrent %s is still broken; let's keep the fixer", t.GetKey(torrent))

17
internal/torrent/types.go
View File

@@ -24,7 +24,6 @@ type Torrent struct {
Rename string `json:"Rename"` // modified over time
SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` // modified over time
UnrepairableReason string `json:"Unfixable"` // modified over time
BrokenLinks mapset.Set[string] `json:"BrokenLinks"` // only relevant on repair
Version string `json:"Version"` // only used for files
}
@@ -36,7 +35,6 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"`
InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
BrokenLinksJson stdjson.RawMessage `json:"BrokenLinks"`
*Alias
}{
Alias: (*Alias)(t),
@@ -69,13 +67,6 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
temp.UnassignedLinksJson = []byte(unassignedLinksStr)
}
if t.BrokenLinks.IsEmpty() {
temp.BrokenLinksJson = []byte(`""`)
} else {
brokenLinksStr := `"` + strings.Join(t.BrokenLinks.ToSlice(), ",") + `"`
temp.BrokenLinksJson = []byte(brokenLinksStr)
}
return json.Marshal(temp)
}
@@ -86,7 +77,6 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"`
InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
BrokenLinksJson stdjson.RawMessage `json:"BrokenLinks"`
*Alias
}{
Alias: (*Alias)(t),
@@ -123,13 +113,6 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
t.UnassignedLinks = mapset.NewSet[string]()
}
if len(temp.BrokenLinksJson) > 2 {
brokenLinks := strings.Split(strings.ReplaceAll(string(temp.BrokenLinksJson), `"`, ""), ",")
t.BrokenLinks = mapset.NewSet[string](brokenLinks...)
} else {
t.BrokenLinks = mapset.NewSet[string]()
}
return nil
}

26
internal/universal/downloader.go
View File

@@ -49,17 +49,17 @@ func (dl *Downloader) DownloadFile(directory, torrentName, fileName string, resp
http.Error(resp, "File is not available", http.StatusNotFound)
return
}
link := file.Link
unrestrict := torMgr.UnrestrictUntilOk(link)
log.Debugf("Opening file %s from torrent %s (%s)", fileName, torMgr.GetKey(torrent), file.Link)
unrestrict := torMgr.UnrestrictUntilOk(file.Link)
if unrestrict == nil {
log.Warnf("File %s cannot be unrestricted (link=%s)", fileName, link)
torrent.BrokenLinks.Add(file.Link)
// file.Link = "repair"
log.Warnf("File %s cannot be unrestricted (link=%s)", fileName, file.Link)
file.Link = ""
if cfg.EnableRepair() {
torMgr.TriggerRepair(torrent)
} else {
log.Debugf("Repair is disabled, skipping repair for unavailable file %s (link=%s)", fileName, link)
log.Debugf("Repair is disabled, skipping repair for unavailable file %s (link=%s)", fileName, file.Link)
}
http.Error(resp, "File is not available", http.StatusNotFound)
return
@@ -100,6 +100,8 @@ func (dl *Downloader) DownloadLink(fileName, link string, resp http.ResponseWrit
return
}
log.Debugf("Opening file %s (%s)", fileName, link)
unrestrict := torMgr.UnrestrictUntilOk(link)
if unrestrict == nil {
log.Warnf("File %s cannot be unrestricted (link=%s)", fileName, link)
@@ -155,17 +157,16 @@ func (dl *Downloader) streamFileToResponse(torrent *intTor.Torrent, file *intTor
}
if torrent != nil {
log.Debugf("Opening file %s from torrent %s (%s)%s", unrestrict.Download, torMgr.GetKey(torrent), unrestrict.Link, rangeLog)
log.Debugf("Downloading unrestricted link %s from torrent %s (%s)%s", unrestrict.Download, torMgr.GetKey(torrent), unrestrict.Link, rangeLog)
} else {
log.Debugf("Opening file %s (%s)%s", unrestrict.Download, unrestrict.Link, rangeLog)
log.Debugf("Downloading unrestricted link %s (%s)%s", unrestrict.Download, unrestrict.Link, rangeLog)
}
download, err := dl.client.Do(dlReq)
if err != nil {
log.Warnf("Cannot download file %s: %v", unrestrict.Download, err)
if file != nil && unrestrict.Streamable == 1 {
torrent.BrokenLinks.Add(file.Link)
// file.Link = "repair"
file.Link = ""
if cfg.EnableRepair() && torrent != nil {
torMgr.TriggerRepair(torrent)
} else {
@@ -181,8 +182,7 @@ func (dl *Downloader) streamFileToResponse(torrent *intTor.Torrent, file *intTor
if download.StatusCode/100 != 2 {
if file != nil && unrestrict.Streamable == 1 {
log.Warnf("Received a %s status code for file %s", download.Status, file.Path)
torrent.BrokenLinks.Add(file.Link)
// file.Link = "repair"
file.Link = ""
if cfg.EnableRepair() && torrent != nil {
torMgr.TriggerRepair(torrent)
} else {
@@ -199,7 +199,7 @@ func (dl *Downloader) streamFileToResponse(torrent *intTor.Torrent, file *intTor
}
}
log.Debugf("Started serving file %s%s", unrestrict.Filename, rangeLog)
log.Debugf("Serving file %s%s", unrestrict.Download, rangeLog)
buf := make([]byte, cfg.GetNetworkBufferSize())
io.CopyBuffer(resp, download.Body, buf)