From 72f7b0f1e57348ba0d85783e4957b8aa93b7e3b0 Mon Sep 17 00:00:00 2001 From: Ben Sarmiento Date: Sun, 7 Jan 2024 14:24:21 +0100 Subject: [PATCH] Repair rework for rar files --- internal/torrent/manager.go | 11 ++++++-- internal/torrent/refresh.go | 5 +++- internal/torrent/repair.go | 56 +++++++++++++++++++++++++++++++++++-- internal/torrent/types.go | 44 +++++++++++++++++++---------- internal/universal/get.go | 54 ----------------------------------- 5 files changed, 97 insertions(+), 73 deletions(-) diff --git a/internal/torrent/manager.go b/internal/torrent/manager.go index de6ee18..faafe43 100644 --- a/internal/torrent/manager.go +++ b/internal/torrent/manager.go @@ -44,7 +44,7 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p Api: api, allAccessKeys: mapset.NewSet[string](), latestState: &initialSate, - requiredVersion: "06.12.2023", + requiredVersion: "07.01.2024", workerPool: p, log: log, } @@ -109,7 +109,14 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p // proxy func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.Download { - return t.Api.UnrestrictUntilOk(link, t.Config.ShouldServeFromRclone()) + if download, exists := t.DownloadCache.Get(link); exists { + return download + } + ret := t.Api.UnrestrictUntilOk(link, t.Config.ShouldServeFromRclone()) + if ret != nil { + t.DownloadCache.Set(link, ret) + } + return ret } func (t *TorrentManager) TriggerHookOnLibraryUpdate(updatedPaths []string) { diff --git a/internal/torrent/refresh.go b/internal/torrent/refresh.go index dd430c7..7140811 100644 --- a/internal/torrent/refresh.go +++ b/internal/torrent/refresh.go @@ -149,7 +149,6 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { } selectedFiles = append(selectedFiles, &File{ File: file, - Added: info.Added, Ended: info.Ended, Link: "", // no link yet }) @@ -159,6 +158,9 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent { for i, file := range selectedFiles { file.Link = info.Links[i] } + torrent.UnassignedLinks = mapset.NewSet[string]() + } else { + torrent.UnassignedLinks = mapset.NewSet[string](info.Links...) } torrent.SelectedFiles = cmap.New[*File]() for _, file := range selectedFiles { @@ -195,6 +197,7 @@ func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) Torrent { mainTorrent.DownloadedIDs = mapset.NewSet[string]() mainTorrent.InProgressIDs = mapset.NewSet[string]() mainTorrent.Unfixable = existing.Unfixable || toMerge.Unfixable + mainTorrent.UnassignedLinks = existing.UnassignedLinks.Union(toMerge.UnassignedLinks) // this function triggers only when we have a new DownloadedID toMerge.DownloadedIDs.Difference(existing.DownloadedIDs).Each(func(id string) bool { diff --git a/internal/torrent/repair.go b/internal/torrent/repair.go index cccc5a8..4f2e260 100644 --- a/internal/torrent/repair.go +++ b/internal/torrent/repair.go @@ -5,6 +5,8 @@ import ( "math" "strings" "time" + + "github.com/debridmediamanager/zurg/pkg/realdebrid" ) const EXPIRED_LINK_TOLERANCE_HOURS = 24 @@ -82,6 +84,57 @@ func (t *TorrentManager) repair(torrent *Torrent) { // sleep for 30 seconds to let the torrent accumulate more broken files if scanning time.Sleep(30 * time.Second) + + // handle rar'ed torrents + assignedCount := 0 + rarCount := 0 + unassignedDownloads := make([]*realdebrid.Download, 0) + torrent.UnassignedLinks.Each(func(link string) bool { + unrestrict := t.UnrestrictUntilOk(link) + if unrestrict != nil && unrestrict.Link != "" { + // assign to a selected file + assigned := false + torrent.SelectedFiles.IterCb(func(_ string, file *File) { + // if strings.HasSuffix(file.Path, unrestrict.Filename) { + if file.Bytes == unrestrict.Filesize { + file.Link = unrestrict.Link + assigned = true + assignedCount++ + } + }) + if !assigned { + if strings.HasSuffix(unrestrict.Filename, ".rar") { + rarCount++ + } + unassignedDownloads = append(unassignedDownloads, unrestrict) + } + } + return false + }) + + if assignedCount > 0 { + t.log.Infof("Assigned %d links to selected files for torrent %s", assignedCount, torrent.AccessKey) + } else if rarCount > 0 { + if t.Config.ShouldDeleteRarFiles() { + t.Delete(torrent.AccessKey, true) + } else { + for _, unassigned := range unassignedDownloads { + newFile := &File{ + File: realdebrid.File{ + ID: 0, + Path: unassigned.Filename, + Bytes: unassigned.Filesize, + Selected: 1, + }, + Ended: torrent.LatestAdded, + Link: unassigned.Link, + } + torrent.SelectedFiles.Set(unassigned.Filename, newFile) + } + } + return + } + // second solution: add only the broken files var brokenFiles []File torrent.SelectedFiles.IterCb(func(_ string, file *File) { @@ -94,8 +147,6 @@ func (t *TorrentManager) repair(torrent *Torrent) { // todo: to verify removed logic when there's only 1 selected file selected and it's broken - // todo: handle rar'ed torrents - if len(brokenFiles) == 1 && torrent.SelectedFiles.Count() > 1 { // if we download a single file, it will be named differently // so we need to download 1 extra file to preserve the name @@ -109,6 +160,7 @@ func (t *TorrentManager) repair(torrent *Torrent) { } } } + if len(brokenFiles) > 0 { t.log.Infof("Redownloading the %d broken files for torrent %s", len(brokenFiles), torrent.AccessKey) brokenFileIDs := strings.Join(getFileIDs(brokenFiles), ",") diff --git a/internal/torrent/types.go b/internal/torrent/types.go index 1a42437..dcafb46 100644 --- a/internal/torrent/types.go +++ b/internal/torrent/types.go @@ -14,13 +14,14 @@ import ( var json = jsoniter.ConfigCompatibleWithStandardLibrary type Torrent struct { - AccessKey string `json:"AccessKey"` - Hash string `json:"Hash"` - SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` - LatestAdded string `json:"LatestAdded"` - Unfixable bool `json:"Unfixable"` - DownloadedIDs mapset.Set[string] `json:"DownloadedIDs"` - InProgressIDs mapset.Set[string] `json:"InProgressIDs"` + AccessKey string `json:"AccessKey"` + Hash string `json:"Hash"` + SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` + LatestAdded string `json:"LatestAdded"` + Unfixable bool `json:"Unfixable"` + DownloadedIDs mapset.Set[string] `json:"DownloadedIDs"` + InProgressIDs mapset.Set[string] `json:"InProgressIDs"` + UnassignedLinks mapset.Set[string] `json:"UnassignedLinks"` Version string `json:"Version"` // only used for files } @@ -28,9 +29,10 @@ type Torrent struct { func (t *Torrent) MarshalJSON() ([]byte, error) { type Alias Torrent temp := &struct { - SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` - DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"` - InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"` + SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` + DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"` + InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"` + UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"` *Alias }{ Alias: (*Alias)(t), @@ -56,15 +58,23 @@ func (t *Torrent) MarshalJSON() ([]byte, error) { temp.InProgressIDsJson = []byte(inProgressIDsStr) } + if t.UnassignedLinks.IsEmpty() { + temp.UnassignedLinksJson = []byte(`""`) + } else { + unassignedLinksStr := `"` + strings.Join(t.UnassignedLinks.ToSlice(), ",") + `"` + temp.UnassignedLinksJson = []byte(unassignedLinksStr) + } + return json.Marshal(temp) } func (t *Torrent) UnmarshalJSON(data []byte) error { type Alias Torrent temp := &struct { - SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` - DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"` - InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"` + SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` + DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"` + InProgressIDsJson stdjson.RawMessage `json:"InProgressIDs"` + UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"` *Alias }{ Alias: (*Alias)(t), @@ -94,6 +104,13 @@ func (t *Torrent) UnmarshalJSON(data []byte) error { t.InProgressIDs = mapset.NewSet[string]() } + if len(temp.UnassignedLinksJson) > 2 { + unassignedLinks := strings.Split(strings.ReplaceAll(string(temp.UnassignedLinksJson), `"`, ""), ",") + t.UnassignedLinks = mapset.NewSet[string](unassignedLinks...) + } else { + t.UnassignedLinks = mapset.NewSet[string]() + } + return nil } @@ -133,7 +150,6 @@ func (t *Torrent) OlderThanDuration(duration time.Duration) bool { type File struct { realdebrid.File - Added string `json:"Added"` Ended string `json:"Ended"` Link string `json:"Link"` } diff --git a/internal/universal/get.go b/internal/universal/get.go index 9c8d037..9a463cb 100644 --- a/internal/universal/get.go +++ b/internal/universal/get.go @@ -1,7 +1,6 @@ package universal import ( - "fmt" "io" "net/http" "net/url" @@ -53,25 +52,6 @@ func (gf *GetFile) ServeFile(directory, torrentName, fileName string, resp http. } link := file.Link - if download, exists := torMgr.DownloadCache.Get(link); exists { - if cfg.ShouldServeFromRclone() { - if cfg.ShouldVerifyDownloadLink() { - if torMgr.Api.CanFetchFirstByte(download.Download) { - redirect(resp, req, download.Download, cfg) - return - } - } else { - redirect(resp, req, download.Download, cfg) - return - } - } else { - err := gf.streamCachedLinkToResponse(download.Download, resp, req, torMgr, cfg, log) - if err == nil { - return - } - } - } - log.Debugf("Opening file %s from torrent %s (%s)", fileName, torrentName, link) unrestrict := torMgr.UnrestrictUntilOk(link) if unrestrict == nil || unrestrict.Link == "" { @@ -98,7 +78,6 @@ func (gf *GetFile) ServeFile(directory, torrentName, fileName string, resp http. log.Warnf("Filename mismatch: %s and %s", fileName, unrestrict.Filename) } } - torMgr.DownloadCache.Set(link, unrestrict) if cfg.ShouldServeFromRclone() { if cfg.ShouldVerifyDownloadLink() { if !torMgr.Api.CanFetchFirstByte(unrestrict.Download) { @@ -115,39 +94,6 @@ func (gf *GetFile) ServeFile(directory, torrentName, fileName string, resp http. } } -func (gf *GetFile) streamCachedLinkToResponse(url string, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) error { - // Create a new dlReq for the file download. - dlReq, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return fmt.Errorf("file is not available") - } - - // copy range header if it exists - if req.Header.Get("Range") != "" { - dlReq.Header.Add("Range", req.Header.Get("Range")) - } - - download, err := gf.client.Do(dlReq) - if err != nil { - return fmt.Errorf("file is not available") - } - defer download.Body.Close() - - if download.StatusCode != http.StatusOK && download.StatusCode != http.StatusPartialContent { - return fmt.Errorf("file is not available") - } - - for k, vv := range download.Header { - for _, v := range vv { - resp.Header().Add(k, v) - } - } - - buf := make([]byte, cfg.GetNetworkBufferSize()) - io.CopyBuffer(resp, download.Body, buf) - return nil -} - func (gf *GetFile) streamFileToResponse(torrent *intTor.Torrent, file *intTor.File, unrestrict *realdebrid.Download, resp http.ResponseWriter, req *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *logutil.Logger) { // Create a new request for the file download. dlReq, err := http.NewRequest(http.MethodGet, unrestrict.Download, nil)