hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove components, do downloaded ids ; support dumps

Browse Source
This commit is contained in:
Ben Sarmiento
2024-05-24 02:24:26 +02:00
parent beba993364
commit 9ecbb5d892
8 changed files with 188 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
internal/torrent/refresh.go
View File

@@ -30,18 +30,40 @@ func (t *TorrentManager) refreshTorrents() []string {
allTorrents, _ := t.DirectoryMap.Get(INT_ALL)
freshHashes := mapset.NewSet[string]()
freshIDs := mapset.NewSet[string]()
freshAccessKeys := mapset.NewSet[string]()
t.getTorrentFiles("dump").Each(func(filePath string) bool {
torrent := t.readTorrentFromFile(filePath)
if torrent != nil {
accessKey := t.GetKey(torrent)
t.log.Debugf("Adding dumped torrent %s", accessKey)
allTorrents.Set(accessKey, torrent)
t.assignDirectory(torrent, func(directory string) {
listing, _ := t.DirectoryMap.Get(directory)
listing.Set(accessKey, torrent)
// note that we're not adding it to updatedPaths
})
freshAccessKeys.Add(accessKey) // to prevent being deleted
}
return false
})
existingHashes := mapset.NewSet[string]()
t.getTorrentFiles("data").Each(func(path string) bool {
path = filepath.Base(path)
hash := strings.TrimSuffix(path, ".zurgtorrent")
existingHashes.Add(hash)
return false
})
for i := range instances {
wg.Add(1)
idx := i
_ = t.workerPool.Submit(func() {
defer wg.Done()
if t.trashBin.Contains(instances[idx].ID) {
t.api.DeleteTorrent(instances[idx].ID)
t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
if t.binImmediately(instances[idx].ID) {
// t.log.Debugf("Skipping trashed torrent %s (id=%s)", instances[idx].Name, instances[idx].ID)
mergeChan <- nil
return
}
@@ -52,7 +74,6 @@ func (t *TorrentManager) refreshTorrents() []string {
return
}
freshHashes.Add(instances[idx].Hash)
freshIDs.Add(instances[idx].ID)
tInfo := t.getMoreInfo(instances[idx])
@@ -71,10 +92,15 @@ func (t *TorrentManager) refreshTorrents() []string {
updatedPaths.Add(fmt.Sprintf("%s/%s", directory, accessKey))
})
} else if !mainTorrent.Components.Has(tInfo.ID) {
} else if !mainTorrent.DownloadedIDs.Contains(tInfo.ID) {
forMerging = torrent
}
// write to file if it is a new torrent
if forMerging == nil && !existingHashes.Contains(tInfo.Hash) {
t.writeTorrentToFile(torrent)
}
mergeChan <- forMerging
})
}
@@ -123,24 +149,13 @@ func (t *TorrentManager) refreshTorrents() []string {
t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
// data directory cleanup
// existingHashes := mapset.NewSet[string]()
// t.getTorrentFiles().Each(func(path string) bool {
// path = filepath.Base(path)
// hash := strings.TrimSuffix(path, ".torrent_zurg")
// existingHashes.Add(hash)
// return false
// })
// existingHashes.Difference(freshHashes).Each(func(hash string) bool {
// t.log.Infof("Deleting stale torrent file %s", hash)
// t.deleteTorrentFile(hash)
// return false
// })
existingIDs := mapset.NewSet[string]()
t.getInfoFiles().Each(func(path string) bool {
path = filepath.Base(path)
torrentID := strings.TrimSuffix(path, ".info_zurg")
existingIDs.Add(torrentID)
if !t.binOnceDone(torrentID) {
existingIDs.Add(torrentID)
}
return false
})
existingIDs.Difference(freshIDs).Each(func(id string) bool {
@@ -149,6 +164,8 @@ func (t *TorrentManager) refreshTorrents() []string {
return false
})
t.cleanupBins(freshIDs)
return updatedPaths.ToSlice()
}
@@ -195,18 +212,18 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *realdebrid.T
}
func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent {
torrent := t.readTorrentFromFile(info.Hash)
if torrent != nil && torrent.Components.Has(info.ID) {
torrent := t.readTorrentFromFile("data/" + info.Hash + ".zurgtorrent")
if torrent != nil && torrent.DownloadedIDs.Contains(info.ID) {
return torrent
}
torrent = &Torrent{
Name: info.Name,
OriginalName: info.OriginalName,
Added: info.Added,
Hash: info.Hash,
State: NewTorrentState("broken_torrent"),
Components: cmap.New[*realdebrid.TorrentInfo](),
Name: info.Name,
OriginalName: info.OriginalName,
Added: info.Added,
Hash: info.Hash,
State: NewTorrentState("broken_torrent"),
DownloadedIDs: mapset.NewSet[string](),
}
// SelectedFiles is a subset of Files with only the selected ones
@@ -271,7 +288,7 @@ func (t *TorrentManager) convertToTorrent(info *realdebrid.TorrentInfo) *Torrent
}
}
torrent.Components.Set(info.ID, info)
torrent.DownloadedIDs.Add(info.ID)
return torrent
}
@@ -286,26 +303,14 @@ func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent {
older = toMerge
}
// components
numComponents := 0
mergedComponents := cmap.New[*realdebrid.TorrentInfo]()
older.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) {
numComponents++
mergedComponents.Set(torrentID, info)
})
newer.Components.IterCb(func(torrentID string, info *realdebrid.TorrentInfo) {
numComponents++
mergedComponents.Set(torrentID, info)
})
// base of the merged torrent
mergedTorrent := &Torrent{
Name: older.Name,
OriginalName: older.OriginalName,
Rename: older.Rename,
Hash: older.Hash,
Added: older.Added,
Components: mergedComponents,
Name: older.Name,
OriginalName: older.OriginalName,
Rename: older.Rename,
Hash: older.Hash,
Added: older.Added,
DownloadedIDs: older.DownloadedIDs.Union(newer.DownloadedIDs),
State: older.State,
}
@@ -368,7 +373,7 @@ func (t *TorrentManager) mergeTorrents(existing, toMerge *Torrent) *Torrent {
}
func (t *TorrentManager) assignDirectory(tor *Torrent, cb func(string)) {
torrentIDs := tor.Components.Keys()
torrentIDs := tor.DownloadedIDs.ToSlice()
// get filenames needed for directory conditions
var filenames []string
var fileSizes []int64
@@ -412,20 +417,78 @@ func (t *TorrentManager) IsPlayable(filePath string) bool {
return false
}
func (t *TorrentManager) trash(torrentId string) {
t.log.Debugf("Trash: %s", torrentId)
t.trashBin.Add(torrentId)
// initializeBins reads from bins.json and assigns values to t.trashBin and t.repairBin
func (t *TorrentManager) initializeBins() {
if _, err := os.Stat("data/bins.json"); os.IsNotExist(err) {
t.log.Warn("data/bins.json does not exist. Initializing empty bins.")
t.immediateBin = mapset.NewSet[string]()
t.onceDoneBin = mapset.NewSet[string]()
return
}
fileData, err := os.ReadFile("data/bins.json")
if err != nil {
t.log.Errorf("Failed to read bins.json file: %v", err)
t.immediateBin = mapset.NewSet[string]()
t.onceDoneBin = mapset.NewSet[string]()
return
}
data := map[string][]string{}
err = json.Unmarshal(fileData, &data)
if err != nil {
t.log.Errorf("Failed to unmarshal bin data: %v", err)
return
}
t.immediateBin = mapset.NewSet[string](data["trash_bin"]...)
t.onceDoneBin = mapset.NewSet[string](data["repair_bin"]...)
t.log.Debug("Successfully read bins from bins.json")
t.log.Debugf("Bin immediately: %v", t.immediateBin.ToSlice())
t.log.Debugf("Bin once done: %v", t.onceDoneBin.ToSlice())
}
func (t *TorrentManager) setToBinImmediately(torrentId string) {
t.log.Debugf("Set to delete immediately: %s", torrentId)
t.immediateBin.Add(torrentId)
t.persistBins()
}
func (t *TorrentManager) trashOnceCompleted(torrentId string) {
t.log.Debugf("Trash once completed: %s", torrentId)
t.repairBin.Add(torrentId)
func (t *TorrentManager) setToBinOnceDone(torrentId string) {
t.log.Debugf("Set to delete once completed: %s", torrentId)
t.onceDoneBin.Add(torrentId)
t.persistBins()
}
func (t *TorrentManager) saveToBinFile() {
func (t *TorrentManager) binImmediately(torrentId string) bool {
if t.immediateBin.Contains(torrentId) {
if err := t.api.DeleteTorrent(torrentId); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err)
return false
}
t.immediateBin.Remove(torrentId)
return true
}
return false
}
func (t *TorrentManager) binOnceDone(torrentId string) bool {
if t.onceDoneBin.Contains(torrentId) {
if err := t.api.DeleteTorrent(torrentId); err != nil {
t.log.Errorf("Failed to delete torrent %s: %v", torrentId, err)
return false
}
t.onceDoneBin.Remove(torrentId)
return true
}
return false
}
func (t *TorrentManager) persistBins() {
data := map[string]interface{}{
"trash_bin": t.trashBin.ToSlice(), // Assuming trashBin is a mapset.Set[string]
"repair_bin": t.repairBin.ToSlice(), // Assuming repairBin is a mapset.Set[string]
"trash_bin": t.immediateBin.ToSlice(), // Assuming trashBin is a mapset.Set[string]
"repair_bin": t.onceDoneBin.ToSlice(), // Assuming repairBin is a mapset.Set[string]
}
jsonData, err := json.Marshal(data)
@@ -434,17 +497,29 @@ func (t *TorrentManager) saveToBinFile() {
return
}
file, err := os.Create("trash_bin")
file, err := os.Create("data/bins.json")
if err != nil {
t.log.Errorf("Failed to create trash_bin file: %v", err)
t.log.Errorf("Failed to create bins.json file: %v", err)
return
}
defer file.Close()
_, err = file.Write(jsonData)
if err != nil {
t.log.Errorf("Failed to write to trash_bin file: %v", err)
t.log.Errorf("Failed to write to bins.json file: %v", err)
} else {
t.log.Debug("Successfully saved bins to file")
}
}
func (t *TorrentManager) cleanupBins(freshIDs mapset.Set[string]) {
t.immediateBin.Difference(freshIDs).Each(func(id string) bool {
t.immediateBin.Remove(id)
return false
})
t.onceDoneBin.Difference(freshIDs).Each(func(id string) bool {
t.onceDoneBin.Remove(id)
return false
})
t.persistBins()
}