Remove components, do downloaded ids ; support dumps

This commit is contained in:
Ben Sarmiento
2024-05-24 02:24:26 +02:00
parent beba993364
commit 9ecbb5d892
8 changed files with 188 additions and 107 deletions

3
.gitignore vendored
View File

@@ -43,3 +43,6 @@ pkg/anidb/
logs/ logs/
*.log *.log
*.zurgtorrent
dump/

View File

@@ -93,6 +93,7 @@ func MainApp(configPath string) {
defer workerPool.Release() defer workerPool.Release()
utils.EnsureDirExists("data") // Ensure the data directory exists utils.EnsureDirExists("data") // Ensure the data directory exists
utils.EnsureDirExists("dump") // dump is a new directory for "torrent" files
torrentMgr := torrent.NewTorrentManager( torrentMgr := torrent.NewTorrentManager(
config, config,
api, api,
@@ -113,6 +114,7 @@ func MainApp(configPath string) {
log.Named("router"), log.Named("router"),
) )
//// pprof
// go func() { // go func() {
// if err := netHttp.ListenAndServe(":6060", nil); err != nil && err != netHttp.ErrServerClosed { // if err := netHttp.ListenAndServe(":6060", nil); err != nil && err != netHttp.ErrServerClosed {
// zurglog.Errorf("Failed to start pprof: %v", err) // zurglog.Errorf("Failed to start pprof: %v", err)

View File

@@ -1,7 +1,6 @@
package torrent package torrent
import ( import (
"github.com/debridmediamanager/zurg/pkg/realdebrid"
cmap "github.com/orcaman/concurrent-map/v2" cmap "github.com/orcaman/concurrent-map/v2"
) )
@@ -27,10 +26,12 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) {
if torrent, ok := allTorrents.Get(accessKey); ok { if torrent, ok := allTorrents.Get(accessKey); ok {
hash = torrent.Hash hash = torrent.Hash
if deleteInRD { if deleteInRD {
torrent.Components.IterCb(func(torrentID string, _ *realdebrid.TorrentInfo) { torrent.DownloadedIDs.Each(func(torrentID string) bool {
t.log.Debugf("Deleting torrent %s (id=%s) in RD", accessKey, torrentID) t.log.Debugf("Deleting torrent %s (id=%s) in RD", accessKey, torrentID)
t.api.DeleteTorrent(torrentID) t.api.DeleteTorrent(torrentID)
t.deleteInfoFile(torrentID) t.deleteInfoFile(torrentID)
return false
}) })
} }
} }

View File

@@ -45,8 +45,8 @@ type TorrentManager struct {
repairRunning bool repairRunning bool
repairRunningMu sync.Mutex repairRunningMu sync.Mutex
trashBin mapset.Set[string] immediateBin mapset.Set[string]
repairBin mapset.Set[string] // same as trash bin, but only if the torrent has been downloaded onceDoneBin mapset.Set[string]
} }
// NewTorrentManager creates a new torrent manager // NewTorrentManager creates a new torrent manager
@@ -73,8 +73,8 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, w
latestState: &LibraryState{log: log}, latestState: &LibraryState{log: log},
} }
t.trashBin = mapset.NewSet[string]() t.initializeBins()
t.initializeDirectories() t.initializeDirectoryMaps()
t.workerPool.Submit(func() { t.workerPool.Submit(func() {
t.refreshTorrents() t.refreshTorrents()
t.setNewLatestState(t.getCurrentState()) t.setNewLatestState(t.getCurrentState())
@@ -139,17 +139,17 @@ func (t *TorrentManager) GetPath(file *File) string {
/// torrent functions /// torrent functions
func (t *TorrentManager) getTorrentFiles() mapset.Set[string] { func (t *TorrentManager) getTorrentFiles(parentDir string) mapset.Set[string] {
files, err := filepath.Glob("data/*.torrent_zurg") files, err := filepath.Glob(parentDir + "/*.zurgtorrent")
if err != nil { if err != nil {
t.log.Warnf("Cannot get files in data directory: %v", err) t.log.Warnf("Cannot get files in %s directory: %v", parentDir, err)
return nil return nil
} }
return mapset.NewSet[string](files...) return mapset.NewSet[string](files...)
} }
func (t *TorrentManager) writeTorrentToFile(torrent *Torrent) { func (t *TorrentManager) writeTorrentToFile(torrent *Torrent) {
filePath := "data/" + torrent.Hash + ".torrent_zurg" filePath := "data/" + torrent.Hash + ".zurgtorrent"
file, err := os.Create(filePath) file, err := os.Create(filePath)
if err != nil { if err != nil {
t.log.Warnf("Cannot create file %s: %v", filePath, err) t.log.Warnf("Cannot create file %s: %v", filePath, err)
@@ -173,8 +173,7 @@ func (t *TorrentManager) writeTorrentToFile(torrent *Torrent) {
t.log.Debugf("Saved torrent %s (hash=%s) to file", t.GetKey(torrent), torrent.Hash) t.log.Debugf("Saved torrent %s (hash=%s) to file", t.GetKey(torrent), torrent.Hash)
} }
func (t *TorrentManager) readTorrentFromFile(hash string) *Torrent { func (t *TorrentManager) readTorrentFromFile(filePath string) *Torrent {
filePath := "data/" + hash + ".torrent_zurg"
file, err := os.Open(filePath) file, err := os.Open(filePath)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
@@ -198,7 +197,7 @@ func (t *TorrentManager) readTorrentFromFile(hash string) *Torrent {
} }
func (t *TorrentManager) deleteTorrentFile(hash string) { func (t *TorrentManager) deleteTorrentFile(hash string) {
filePath := "data/" + hash + ".torrent_zurg" filePath := "data/" + hash + ".zurgtorrent"
_ = os.Remove(filePath) _ = os.Remove(filePath)
} }
@@ -310,7 +309,7 @@ func (t *TorrentManager) StartDownloadsJob() {
}) })
} }
func (t *TorrentManager) initializeDirectories() { func (t *TorrentManager) initializeDirectoryMaps() {
// create internal directories // create internal directories
t.DirectoryMap.Set(INT_ALL, cmap.New[*Torrent]()) // key is GetAccessKey() t.DirectoryMap.Set(INT_ALL, cmap.New[*Torrent]()) // key is GetAccessKey()
// create directory maps // create directory maps

View File

@@ -30,18 +30,40 @@ func (t *TorrentManager) refreshTorrents() []string {
allTorrents, _ := t.DirectoryMap.Get(INT_ALL) allTorrents, _ := t.DirectoryMap.Get(INT_ALL)
freshHashes := mapset.NewSet[string]()
freshIDs := mapset.NewSet[string]() freshIDs := mapset.NewSet[string]()
freshAccessKeys := mapset.NewSet[string]() freshAccessKeys := mapset.NewSet[string]()
t.getTorrentFiles("dump").Each(func(filePath string) bool {
torrent := t.readTorrentFromFile(filePath)
if torrent != nil {
accessKey := t.GetKey(torrent)
t.log.Debugf("Adding dumped torrent %s", accessKey)
allTorrents.Set(accessKey, torrent)
t.assignDirectory(torrent, func(directory string) {
listing, _ := t.DirectoryMap.Get(directory)
listing.Set(accessKey, torrent)
// note that we're not adding it to updatedPaths
})
freshAccessKeys.Add(accessKey) // to prevent being deleted
}
return false
})
existingHashes := mapset.NewSet[string]()
t.getTorrentFiles("data").Each(func(path string) bool {
path = filepath.Base(path)
hash := strings.TrimSuffix(path, ".zurgtorrent")
existingHashes.Add(hash)
return false
})
for i := range instances { for i := range instances {
wg.Add(1) wg.Add(1)
idx := i idx := i
_ = t.workerPool.Submit(func() { _ = t.workerPool.Submit(func() {
defer wg.Done() defer wg.Done()
if t.trashBin.Contains(instances[idx].ID) { if t.binImmediately(instances[idx].ID) {
t.api.DeleteTorrent(instances[idx].ID) // t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
mergeChan <- nil mergeChan <- nil
return return
} }
@@ -52,7 +74,6 @@ func (t *TorrentManager) refreshTorrents() []string {
return return
} }
freshHashes.Add(instances[idx].Hash)
freshIDs.Add(instances[idx].ID) freshIDs.Add(instances[idx].ID)
tInfo := t.getMoreInfo(instances[idx]) tInfo := t.getMoreInfo(instances[idx])
@@ -71,10 +92,15 @@ func (t *TorrentManager) refreshTorrents() []string {
updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey)) updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey))
}) })
} else if !mainTorrent.Components.Has(tInfo.ID) { } else if !mainTorrent.DownloadedIDs.Contains(tInfo.ID) {
forMerging = torrent forMerging = torrent
} }
// write to file if it is a new torrent
if forMerging == nil && !existingHashes.Contains(tInfo.Hash) {
t.writeTorrentToFile(torrent)
}
mergeChan <- forMerging mergeChan <- forMerging
}) })
} }
@@ -123,24 +149,13 @@ func (t *TorrentManager) refreshTorrents() []string {
t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount) t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
// data directory cleanup
// existingHashes := mapset.NewSet[string]()
// t.getTorrentFiles().Each(func(path string) bool {
// path = filepath.Base(path)
// hash := strings.TrimSuffix(path, ".torrent_zurg")
// existingHashes.Add(hash)
// return false
// })
// existingHashes.Difference(freshHashes).Each(func(hash string) bool {
// t.log.Infof("Deleting stale torrent file %s", hash)
// t.deleteTorrentFile(hash)
// return false
// })
existingIDs := mapset.NewSet[string]() existingIDs := mapset.NewSet[string]()
t.getInfoFiles().Each(func(path string) bool { t.getInfoFiles().Each(func(path string) bool {
path = filepath.Base(path) path = filepath.Base(path)
torrentID := strings.TrimSuffix(path, ".info_zurg") torrentID := strings.TrimSuffix(path, ".info_zurg")
if !t.binOnceDone(torrentID) {
existingIDs.Add(torrentID) existingIDs.Add(torrentID)
}
return false return false
}) })
existingIDs.Difference(freshIDs).Each(func(id string) bool { existingIDs.Difference(freshIDs).Each(func(id string) bool {
@@ -149,6 +164,8 @@ func (t *TorrentManager) refreshTorrents() []string {
return false return false
}) })
t.cleanupBins(freshIDs)
return updatedPaths.ToSlice() return updatedPaths.ToSlice()
} }
@@ -195,8 +212,8 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *realdebrid.T
} }
func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent { func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent {
torrent := t.readTorrentFromFile(info.Hash) torrent := t.readTorrentFromFile("data/" + info.Hash + ".zurgtorrent")
if torrent != nil && torrent.Components.Has(info.ID) { if torrent != nil && torrent.DownloadedIDs.Contains(info.ID) {
return torrent return torrent
} }
@@ -206,7 +223,7 @@ func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent
Added: info.Added, Added: info.Added,
Hash: info.Hash, Hash: info.Hash,
State: NewTorrentState("broken_torrent"), State: NewTorrentState("broken_torrent"),
Components: cmap.New[*realdebrid.TorrentInfo](), DownloadedIDs: mapset.NewSet[string](),
} }
// SelectedFiles is a subset of Files with only the selected ones // SelectedFiles is a subset of Files with only the selected ones
@@ -271,7 +288,7 @@ func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent
} }
} }
torrent.Components.Set(info.ID, info) torrent.DownloadedIDs.Add(info.ID)
return torrent return torrent
} }
@@ -286,18 +303,6 @@ func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent {
older = toMerge older = toMerge
} }
// components
numComponents := 0
mergedComponents := cmap.New[*realdebrid.TorrentInfo]()
older.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) {
numComponents++
mergedComponents.Set(torrentID, info)
})
newer.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) {
numComponents++
mergedComponents.Set(torrentID, info)
})
// base of the merged torrent // base of the merged torrent
mergedTorrent := &Torrent{ mergedTorrent := &Torrent{
Name: older.Name, Name: older.Name,
@@ -305,7 +310,7 @@ func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent {
Rename: older.Rename, Rename: older.Rename,
Hash: older.Hash, Hash: older.Hash,
Added: older.Added, Added: older.Added,
Components: mergedComponents, DownloadedIDs: older.DownloadedIDs.Union(newer.DownloadedIDs),
State: older.State, State: older.State,
} }
@@ -368,7 +373,7 @@ func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent {
} }
func (t *TorrentManager) assignDirectory(tor *Torrent, cb func(string)) { func (t *TorrentManager) assignDirectory(tor *Torrent, cb func(string)) {
torrentIDs := tor.Components.Keys() torrentIDs := tor.DownloadedIDs.ToSlice()
// get filenames needed for directory conditions // get filenames needed for directory conditions
var filenames []string var filenames []string
var fileSizes []int64 var fileSizes []int64
@@ -412,20 +417,78 @@ func (t *TorrentManager) IsPlayable(filePath string) bool {
return false return false
} }
func (t *TorrentManager) trash(torrentId string) { // initializeBins reads from bins.json and assigns values to t.trashBin and t.repairBin
t.log.Debugf("Trash: %s", torrentId) func (t *TorrentManager) initializeBins() {
t.trashBin.Add(torrentId) if _, err := os.Stat("data/bins.json"); os.IsNotExist(err) {
t.log.Warn("data/bins.json does not exist. Initializing empty bins.")
t.immediateBin = mapset.NewSet[string]()
t.onceDoneBin = mapset.NewSet[string]()
return
} }
func (t *TorrentManager) trashOnceCompleted(torrentId string) { fileData, err := os.ReadFile("data/bins.json")
t.log.Debugf("Trash once completed: %s", torrentId) if err != nil {
t.repairBin.Add(torrentId) t.log.Errorf("Failed to read bins.json file: %v", err)
t.immediateBin = mapset.NewSet[string]()
t.onceDoneBin = mapset.NewSet[string]()
return
} }
func (t *TorrentManager) saveToBinFile() { data := map[string][]string{}
err = json.Unmarshal(fileData, &data)
if err != nil {
t.log.Errorf("Failed to unmarshal bin data: %v", err)
return
}
t.immediateBin = mapset.NewSet[string](data["trash_bin"]...)
t.onceDoneBin = mapset.NewSet[string](data["repair_bin"]...)
t.log.Debug("Successfully read bins from bins.json")
t.log.Debugf("Bin immediately: %v", t.immediateBin.ToSlice())
t.log.Debugf("Bin once done: %v", t.onceDoneBin.ToSlice())
}
func (t *TorrentManager) setToBinImmediately(torrentId string) {
t.log.Debugf("Set to delete immediately: %s", torrentId)
t.immediateBin.Add(torrentId)
t.persistBins()
}
func (t *TorrentManager) setToBinOnceDone(torrentId string) {
t.log.Debugf("Set to delete once completed: %s", torrentId)
t.onceDoneBin.Add(torrentId)
t.persistBins()
}
func (t *TorrentManager) binImmediately(torrentId string) bool {
if t.immediateBin.Contains(torrentId) {
if err := t.api.DeleteTorrent(torrentId); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err)
return false
}
t.immediateBin.Remove(torrentId)
return true
}
return false
}
func (t *TorrentManager) binOnceDone(torrentId string) bool {
if t.onceDoneBin.Contains(torrentId) {
if err := t.api.DeleteTorrent(torrentId); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err)
return false
}
t.onceDoneBin.Remove(torrentId)
return true
}
return false
}
func (t *TorrentManager) persistBins() {
data := map[string]interface{}{ data := map[string]interface{}{
"trash_bin": t.trashBin.ToSlice(), // Assuming trashBin is a mapset.Set[string] "trash_bin": t.immediateBin.ToSlice(), // Assuming trashBin is a mapset.Set[string]
"repair_bin": t.repairBin.ToSlice(), // Assuming repairBin is a mapset.Set[string] "repair_bin": t.onceDoneBin.ToSlice(), // Assuming repairBin is a mapset.Set[string]
} }
jsonData, err := json.Marshal(data) jsonData, err := json.Marshal(data)
@@ -434,17 +497,29 @@ func (t *TorrentManager) saveToBinFile() {
return return
} }
file, err := os.Create("trash_bin") file, err := os.Create("data/bins.json")
if err != nil { if err != nil {
t.log.Errorf("Failed to create trash_bin file: %v", err) t.log.Errorf("Failed to create bins.json file: %v", err)
return return
} }
defer file.Close() defer file.Close()
_, err = file.Write(jsonData) _, err = file.Write(jsonData)
if err != nil { if err != nil {
t.log.Errorf("Failed to write to trash_bin file: %v", err) t.log.Errorf("Failed to write to bins.json file: %v", err)
} else { } else {
t.log.Debug("Successfully saved bins to file") t.log.Debug("Successfully saved bins to file")
} }
} }
func (t *TorrentManager) cleanupBins(freshIDs mapset.Set[string]) {
t.immediateBin.Difference(freshIDs).Each(func(id string) bool {
t.immediateBin.Remove(id)
return false
})
t.onceDoneBin.Difference(freshIDs).Each(func(id string) bool {
t.onceDoneBin.Remove(id)
return false
})
t.persistBins()
}

View File

@@ -156,7 +156,7 @@ func (t *TorrentManager) Repair(torrent *Torrent, wg *sync.WaitGroup) {
} }
func (t *TorrentManager) repair(torrent *Torrent) { func (t *TorrentManager) repair(torrent *Torrent) {
torrentIDs := torrent.Components.Keys() torrentIDs := torrent.DownloadedIDs.ToSlice()
t.log.Infof("Started repair process for torrent %s (ids=%v)", t.GetKey(torrent), torrentIDs) t.log.Infof("Started repair process for torrent %s (ids=%v)", t.GetKey(torrent), torrentIDs)
// handle torrents with incomplete links for selected files // handle torrents with incomplete links for selected files
@@ -180,10 +180,11 @@ func (t *TorrentManager) repair(torrent *Torrent) {
return return
} }
// delete old torrents // delete old torrents
torrent.Components.IterCb(func(id string, _ *realdebrid.TorrentInfo) { torrent.DownloadedIDs.Each(func(torrentID string) bool {
if id != info.ID { if torrentID != info.ID {
t.api.DeleteTorrent(id) t.setToBinOnceDone(torrentID)
} }
return false
}) })
t.log.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent)) t.log.Infof("Successfully repaired torrent %s by redownloading all files", t.GetKey(torrent))
return return
@@ -212,7 +213,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
return return
} }
t.log.Infof("Torrent %s will be repaired by downloading %d batches of the %d broken files", int(math.Ceil(float64(len(brokenFiles))/100)), len(brokenFiles), t.GetKey(torrent)) t.log.Infof("Torrent %s will be repaired by downloading %d batches of the %d broken files", t.GetKey(torrent), int(math.Ceil(float64(len(brokenFiles))/100)), len(brokenFiles))
newlyDownloadedIds := make([]string, 0) newlyDownloadedIds := make([]string, 0)
batchNum := 1 batchNum := 1
@@ -227,7 +228,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
if err != nil { if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error()) t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds { for _, newId := range newlyDownloadedIds {
t.trash(newId) t.setToBinImmediately(newId)
} }
return return
} }
@@ -243,7 +244,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
if err != nil { if err != nil {
t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error()) t.log.Warnf("Cannot repair torrent %s by downloading broken files (error=%s) giving up", t.GetKey(torrent), err.Error())
for _, newId := range newlyDownloadedIds { for _, newId := range newlyDownloadedIds {
t.trash(newId) t.setToBinImmediately(newId)
} }
return return
} }
@@ -251,7 +252,7 @@ func (t *TorrentManager) repair(torrent *Torrent) {
} }
for _, newId := range newlyDownloadedIds { for _, newId := range newlyDownloadedIds {
t.trashOnceCompleted(newId) t.setToBinOnceDone(newId)
} }
} }
@@ -398,7 +399,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
// select files // select files
err = t.api.SelectTorrentFiles(newTorrentID, finalSelection) err = t.api.SelectTorrentFiles(newTorrentID, finalSelection)
if err != nil { if err != nil {
t.trash(newTorrentID) t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): %v", t.GetKey(torrent), newTorrentID, err) return nil, fmt.Errorf("cannot start redownloading torrent %s (id=%s): %v", t.GetKey(torrent), newTorrentID, err)
} }
// sleep for 2 second to let RD process the magnet // sleep for 2 second to let RD process the magnet
@@ -407,7 +408,7 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
// see if the torrent is ready // see if the torrent is ready
info, err = t.api.GetTorrentInfo(newTorrentID) info, err = t.api.GetTorrentInfo(newTorrentID)
if err != nil { if err != nil {
t.trash(newTorrentID) t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("cannot get info on redownloaded torrent %s (id=%s) : %v", t.GetKey(torrent), newTorrentID, err) return nil, fmt.Errorf("cannot get info on redownloaded torrent %s (id=%s) : %v", t.GetKey(torrent), newTorrentID, err)
} }
@@ -433,13 +434,13 @@ func (t *TorrentManager) redownloadTorrent(torrent *Torrent, selection []string)
} }
if !isOkStatus { if !isOkStatus {
t.trash(info.ID) t.setToBinImmediately(info.ID)
return nil, fmt.Errorf("the redownloaded torrent %s is in a non-OK state: %s", t.GetKey(torrent), info.Status) return nil, fmt.Errorf("the redownloaded torrent %s is in a non-OK state: %s", t.GetKey(torrent), info.Status)
} }
// check if incorrect number of links // check if incorrect number of links
if info.Progress == 100 && len(info.Links) != len(selection) { if info.Progress == 100 && len(info.Links) != len(selection) {
t.trash(newTorrentID) t.setToBinImmediately(newTorrentID)
return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), len(selection)) return nil, fmt.Errorf("torrent %s only got %d links but we need %d", t.GetKey(torrent), len(info.Links), len(selection))
} }

View File

@@ -4,7 +4,6 @@ import (
stdjson "encoding/json" stdjson "encoding/json"
"strings" "strings"
"github.com/debridmediamanager/zurg/pkg/realdebrid"
mapset "github.com/deckarep/golang-set/v2" mapset "github.com/deckarep/golang-set/v2"
jsoniter "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go"
"github.com/looplab/fsm" "github.com/looplab/fsm"
@@ -18,9 +17,9 @@ type Torrent struct {
OriginalName string `json:"OriginalName"` OriginalName string `json:"OriginalName"`
Hash string `json:"Hash"` Hash string `json:"Hash"`
Added string `json:"Added"` Added string `json:"Added"`
Components cmap.ConcurrentMap[string, *realdebrid.TorrentInfo] `json:"-"` DownloadedIDs mapset.Set[string] `json:"DownloadedIDs"` // used for keeping track of downloaded files
UnassignedLinks mapset.Set[string] `json:"UnassignedLinks"` // when links are not complete, we cannot assign them to a file so we store them here until it's fixed UnassignedLinks mapset.Set[string] `json:"-"` // when links are not complete, we cannot assign them to a file so we store them here until it's fixed
Rename string `json:"Rename"` Rename string `json:"Rename"`
SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"` SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"`
@@ -33,7 +32,7 @@ type Torrent struct {
func (t *Torrent) MarshalJSON() ([]byte, error) { func (t *Torrent) MarshalJSON() ([]byte, error) {
type Alias Torrent type Alias Torrent
temp := &struct { temp := &struct {
ComponentsJson stdjson.RawMessage `json:"Components"` DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"`
SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"` UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
StateJson stdjson.RawMessage `json:"State"` StateJson stdjson.RawMessage `json:"State"`
@@ -55,11 +54,12 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
temp.UnassignedLinksJson = []byte(unassignedLinksStr) temp.UnassignedLinksJson = []byte(unassignedLinksStr)
} }
componentsJson, err := t.Components.MarshalJSON() if t.DownloadedIDs.IsEmpty() {
if err != nil { temp.DownloadedIDsJson = []byte(`""`)
return nil, err } else {
DownloadedIDsStr := `"` + strings.Join(t.DownloadedIDs.ToSlice(), ",") + `"`
temp.DownloadedIDsJson = []byte(DownloadedIDsStr)
} }
temp.ComponentsJson = componentsJson
temp.StateJson = []byte(`"` + t.State.Current() + `"`) temp.StateJson = []byte(`"` + t.State.Current() + `"`)
@@ -69,7 +69,7 @@ func (t *Torrent) MarshalJSON() ([]byte, error) {
func (t *Torrent) UnmarshalJSON(data []byte) error { func (t *Torrent) UnmarshalJSON(data []byte) error {
type Alias Torrent type Alias Torrent
temp := &struct { temp := &struct {
ComponentsJson stdjson.RawMessage `json:"Components"` DownloadedIDsJson stdjson.RawMessage `json:"DownloadedIDs"`
SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"` SelectedFilesJson stdjson.RawMessage `json:"SelectedFiles"`
UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"` UnassignedLinksJson stdjson.RawMessage `json:"UnassignedLinks"`
StateJson string `json:"State"` StateJson string `json:"State"`
@@ -81,13 +81,6 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
return err return err
} }
t.Components = cmap.New[*realdebrid.TorrentInfo]()
if len(temp.ComponentsJson) > 0 {
if err := t.Components.UnmarshalJSON(temp.ComponentsJson); err != nil {
return err
}
}
t.SelectedFiles = cmap.New[*File]() t.SelectedFiles = cmap.New[*File]()
if len(temp.SelectedFilesJson) > 0 { if len(temp.SelectedFilesJson) > 0 {
if err := t.SelectedFiles.UnmarshalJSON(temp.SelectedFilesJson); err != nil { if err := t.SelectedFiles.UnmarshalJSON(temp.SelectedFilesJson); err != nil {
@@ -95,6 +88,13 @@ func (t *Torrent) UnmarshalJSON(data []byte) error {
} }
} }
if len(temp.DownloadedIDsJson) > 2 {
downloadedIDs := strings.Split(strings.ReplaceAll(string(temp.DownloadedIDsJson), `"`, ""), ",")
t.DownloadedIDs = mapset.NewSet[string](downloadedIDs...)
} else {
t.DownloadedIDs = mapset.NewSet[string]()
}
if len(temp.UnassignedLinksJson) > 2 { if len(temp.UnassignedLinksJson) > 2 {
unassignedLinks := strings.Split(strings.ReplaceAll(string(temp.UnassignedLinksJson), `"`, ""), ",") unassignedLinks := strings.Split(strings.ReplaceAll(string(temp.UnassignedLinksJson), `"`, ""), ",")
t.UnassignedLinks = mapset.NewSet[string](unassignedLinks...) t.UnassignedLinks = mapset.NewSet[string](unassignedLinks...)

View File

@@ -146,7 +146,7 @@ func (dl *Downloader) streamFileToResponse(
// Add the range header if it exists // Add the range header if it exists
if req.Header.Get("Range") != "" { if req.Header.Get("Range") != "" {
dlReq.Header.Add("Range", req.Header.Get("Range")) dlReq.Header.Add("Range", req.Header.Get("Range"))
log.Debugf("Range request for file %s: %s", unrestrict.Download, req.Header.Get("Range")) // log.Debugf("Range request for file %s: %s", unrestrict.Download, req.Header.Get("Range"))
} }
// Perform the request // Perform the request