hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Repair rework for rar files

Browse Source
This commit is contained in:
Ben Sarmiento
2024-01-07 14:24:21 +01:00
parent e5f0f4d7bd
commit 72f7b0f1e5
5 changed files with 97 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/torrent/manager.go
View File

@@ -44,7 +44,7 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p
Api: api,
allAccessKeys: mapset.NewSet[string](),
latestState: &initialSate,
requiredVersion: "06.12.2023",
requiredVersion: "07.01.2024",
workerPool: p,
log: log,
}
@@ -109,7 +109,14 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p
// proxy
func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.Download {
return t.Api.UnrestrictUntilOk(link, t.Config.ShouldServeFromRclone())
if download, exists := t.DownloadCache.Get(link); exists {
return download
}
ret := t.Api.UnrestrictUntilOk(link, t.Config.ShouldServeFromRclone())
if ret != nil {
t.DownloadCache.Set(link, ret)
}
return ret
}
func (t *TorrentManager) TriggerHookOnLibraryUpdate(updatedPaths []string) {

5
internal/torrent/refresh.go
View File

@@ -149,7 +149,6 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
}
selectedFiles = append(selectedFiles, &File{
File: file,
Added: info.Added,
Ended: info.Ended,
Link: "", // no link yet
})
@@ -159,6 +158,9 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
for i, file := range selectedFiles {
file.Link = info.Links[i]
}
torrent.UnassignedLinks = mapset.NewSet[string]()
} else {
torrent.UnassignedLinks = mapset.NewSet[string](info.Links...)
}
torrent.SelectedFiles = cmap.New[*File]()
for _, file := range selectedFiles {
@@ -195,6 +197,7 @@ func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) Torrent {
mainTorrent.DownloadedIDs = mapset.NewSet[string]()
mainTorrent.InProgressIDs = mapset.NewSet[string]()
mainTorrent.Unfixable = existing.Unfixable || toMerge.Unfixable
mainTorrent.UnassignedLinks = existing.UnassignedLinks.Union(toMerge.UnassignedLinks)
// this function triggers only when we have a new DownloadedID
toMerge.DownloadedIDs.Difference(existing.DownloadedIDs).Each(func(id string) bool {

56
internal/torrent/repair.go
View File

@@ -5,6 +5,8 @@ import (
"math"
"strings"
"time"
"github.com/debridmediamanager/zurg/pkg/realdebrid"
)
const EXPIRED_LINK_TOLERANCE_HOURS = 24
@@ -82,6 +84,57 @@ func (t *TorrentManager) repair(torrent *Torrent) {
// sleep for 30 seconds to let the torrent accumulate more broken files if scanning
time.Sleep(30 * time.Second)
// handle rar'ed torrents
assignedCount := 0
rarCount := 0
unassignedDownloads := make([]*realdebrid.Download, 0)
torrent.UnassignedLinks.Each(func(link string) bool {
unrestrict := t.UnrestrictUntilOk(link)
if unrestrict != nil && unrestrict.Link != "" {
// assign to a selected file
assigned := false
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
// if strings.HasSuffix(file.Path, unrestrict.Filename) {
if file.Bytes == unrestrict.Filesize {
file.Link = unrestrict.Link
assigned = true
assignedCount++
}
})
if !assigned {
if strings.HasSuffix(unrestrict.Filename, ".rar") {
rarCount++
}
unassignedDownloads = append(unassignedDownloads, unrestrict)
}
}
return false
})
if assignedCount > 0 {
t.log.Infof("Assigned %d links to selected files for torrent %s", assignedCount, torrent.AccessKey)
} else if rarCount > 0 {
if t.Config.ShouldDeleteRarFiles() {
t.Delete(torrent.AccessKey, true)
} else {
for _, unassigned := range unassignedDownloads {
newFile := &File{
File: realdebrid.File{
ID: 0,
Path: unassigned.Filename,
Bytes: unassigned.Filesize,
Selected: 1,
},
Ended: torrent.LatestAdded,
Link: unassigned.Link,
}
torrent.SelectedFiles.Set(unassigned.Filename, newFile)
}
}
return
}
// second solution: add only the broken files
var brokenFiles []File
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
@@ -94,8 +147,6 @@ func (t *TorrentManager) repair(torrent *Torrent) {
// todo: to verify removed logic when there's only 1 selected file selected and it's broken
// todo: handle rar'ed torrents
if len(brokenFiles) == 1 && torrent.SelectedFiles.Count() > 1 {
// if we download a single file, it will be named differently
// so we need to download 1 extra file to preserve the name
@@ -109,6 +160,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
}
}
}
if len(brokenFiles) > 0 {
t.log.Infof("Redownloading the %d broken files for torrent %s", len(brokenFiles), torrent.AccessKey)
brokenFileIDs := strings.Join(getFileIDs(brokenFiles), ",")

18
internal/torrent/types.go
View File

@@ -21,6 +21,7 @@ type Torrent struct {
Unfixable bool `json:"Unfixable"`
DownloadedIDs mapset.Set[string] `json:"DownloadedIDs"`
InProgressIDs mapset.Set[string] `json:"InProgressIDs"`
UnassignedLinks mapset.Set[string] `json:"UnassignedLinks"`
Version string `json:"Version"` // only used for files
}
@@ -31,6 +32,7 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"`
DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"`
InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
*Alias
}{
Alias: (*Alias)(t),
@@ -56,6 +58,13 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
temp.InProgressIDsJson = []byte(inProgressIDsStr)
}
if t.UnassignedLinks.IsEmpty() {
temp.UnassignedLinksJson = []byte(`""`)
} else {
unassignedLinksStr := `"` + strings.Join(t.UnassignedLinks.ToSlice(), ",") + `"`
temp.UnassignedLinksJson = []byte(unassignedLinksStr)
}
return json.Marshal(temp)
}
@@ -65,6 +74,7 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"`
DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"`
InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
*Alias
}{
Alias: (*Alias)(t),
@@ -94,6 +104,13 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
t.InProgressIDs = mapset.NewSet[string]()
}
if len(temp.UnassignedLinksJson) > 2 {
unassignedLinks := strings.Split(strings.ReplaceAll(string(temp.UnassignedLinksJson), `"`, ""), ",")
t.UnassignedLinks = mapset.NewSet[string](unassignedLinks...)
} else {
t.UnassignedLinks = mapset.NewSet[string]()
}
return nil
}
@@ -133,7 +150,6 @@ func (t *Torrent) OlderThanDuration(duration time.Duration) bool {
type File struct {
realdebrid.File
Added string `json:"Added"`
Ended string `json:"Ended"`
Link string `json:"Link"`
}

54
internal/universal/get.go
View File

@@ -1,7 +1,6 @@
package universal
import (
"fmt"
"io"
"net/http"
"net/url"
@@ -53,25 +52,6 @@ func (gf *GetFile) ServeFile(directory, torrentName, fileName string, resp http.
}
link := file.Link
if download, exists := torMgr.DownloadCache.Get(link); exists {
if cfg.ShouldServeFromRclone() {
if cfg.ShouldVerifyDownloadLink() {
if torMgr.Api.CanFetchFirstByte(download.Download) {
redirect(resp, req, download.Download, cfg)
return
}
} else {
redirect(resp, req, download.Download, cfg)
return
}
} else {
err := gf.streamCachedLinkToResponse(download.Download, resp, req, torMgr, cfg, log)
if err == nil {
return
}
}
}
log.Debugf("Opening file %s from torrent %s (%s)", fileName, torrentName, link)
unrestrict := torMgr.UnrestrictUntilOk(link)
if unrestrict == nil || unrestrict.Link == "" {
@@ -98,7 +78,6 @@ func (gf *GetFile) ServeFile(directory, torrentName, fileName string, resp http.
log.Warnf("Filename mismatch: %s and %s", fileName, unrestrict.Filename)
}
}
torMgr.DownloadCache.Set(link, unrestrict)
if cfg.ShouldServeFromRclone() {
if cfg.ShouldVerifyDownloadLink() {
if !torMgr.Api.CanFetchFirstByte(unrestrict.Download) {
@@ -115,39 +94,6 @@ func (gf *GetFile) ServeFile(directory, torrentName, fileName string, resp http.
}
}
func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) error {
// Create a new dlReq for the file download.
dlReq, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return fmt.Errorf("file is not available")
}
// copy range header if it exists
if req.Header.Get("Range") != "" {
dlReq.Header.Add("Range", req.Header.Get("Range"))
}
download, err := gf.client.Do(dlReq)
if err != nil {
return fmt.Errorf("file is not available")
}
defer download.Body.Close()
if download.StatusCode != http.StatusOK && download.StatusCode != http.StatusPartialContent {
return fmt.Errorf("file is not available")
}
for k, vv := range download.Header {
for _, v := range vv {
resp.Header().Add(k, v)
}
}
buf := make([]byte, cfg.GetNetworkBufferSize())
io.CopyBuffer(resp, download.Body, buf)
return nil
}
func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) {
// Create a new request for the file download.
dlReq, err := http.NewRequest(http.MethodGet, unrestrict.Download, nil)