hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Access key computation without clearing data

Browse Source
This commit is contained in:
Ben Sarmiento
2024-01-10 02:36:12 +01:00
parent 91f6d35831
commit ed87c2bbcc
12 changed files with 146 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/config/types.go
View File

@@ -19,6 +19,7 @@ type ConfigInterface interface {
GetNetworkBufferSize() int
EnableRetainFolderNameExtension() bool
EnableRetainRDTorrentName() bool
ShouldAllowRenames() bool
GetRandomPreferredHost() string
ShouldServeFromRclone() bool
ShouldVerifyDownloadLink() bool
@@ -41,6 +42,7 @@ type ZurgConfig struct {
NumOfWorkers int `yaml:"concurrent_workers" json:"concurrent_workers"`
RefreshEverySeconds int `yaml:"check_for_changes_every_secs" json:"check_for_changes_every_secs"`
AllowRenames bool `yaml:"allow_renames" json:"allow_renames"`
RetainRDTorrentName bool `yaml:"retain_rd_torrent_name" json:"retain_rd_torrent_name"`
RetainFolderNameExtension bool `yaml:"retain_folder_name_extension" json:"retain_folder_name_extension"`
@@ -127,6 +129,10 @@ func (z *ZurgConfig) EnableRetainRDTorrentName() bool {
return z.RetainRDTorrentName
}
func (z *ZurgConfig) ShouldAllowRenames() bool {
return z.AllowRenames
}
func (z *ZurgConfig) GetRandomPreferredHost() string {
if len(z.PreferredHosts) == 0 {
return ""

2
internal/dav/infuse.go
View File

@@ -47,7 +47,7 @@ func ServeTorrentsListForInfuse(directory string, torMgr *torrent.TorrentManager
if !ok || tor.AllInProgress() {
continue
}
buf.WriteString(dav.BaseDirectory(tor.AccessKey, tor.LatestAdded))
buf.WriteString(dav.BaseDirectory(torMgr.GetKey(tor), tor.LatestAdded))
}
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil

6
internal/dav/listing.go
View File

@@ -50,7 +50,7 @@ func ServeTorrentsList(directory string, torMgr *torrent.TorrentManager) ([]byte
if !ok || tor.AllInProgress() {
continue
}
buf.WriteString(dav.Directory(tor.AccessKey, tor.LatestAdded))
buf.WriteString(dav.Directory(torMgr.GetKey(tor), tor.LatestAdded))
}
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil
@@ -74,7 +74,7 @@ func ServeFilesList(directory, torrentName string, torMgr *torrent.TorrentManage
var buf bytes.Buffer
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
buf.WriteString(dav.BaseDirectory(filepath.Join(directory, tor.AccessKey), tor.LatestAdded))
buf.WriteString(dav.BaseDirectory(filepath.Join(directory, torMgr.GetKey(tor)), tor.LatestAdded))
filenames := tor.SelectedFiles.Keys()
sort.Strings(filenames)
for _, filename := range filenames {
@@ -113,7 +113,7 @@ func HandleSingleFile(directory, torrentName, fileName string, torMgr *torrent.T
var buf bytes.Buffer
buf.WriteString("<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
buf.WriteString(dav.BaseDirectory(filepath.Join(directory, tor.AccessKey), tor.LatestAdded))
buf.WriteString(dav.BaseDirectory(filepath.Join(directory, torMgr.GetKey(tor)), tor.LatestAdded))
buf.WriteString(dav.File(fileName, file.Bytes, file.Ended))
buf.WriteString("</d:multistatus>")
return buf.Bytes(), nil

2
internal/dav/rename.go
View File

@@ -17,7 +17,7 @@ func HandleRenameTorrent(directory, torrentName, newName string, torMgr *torrent
}
torrents.Remove(torrentName)
torrents.Set(newName, torrent)
torrent.AccessKey = newName
torrent.Rename = newName
return nil
}

2
internal/http/listing.go
View File

@@ -48,7 +48,7 @@ func ServeTorrentsList(directory string, torMgr *torrent.TorrentManager) ([]byte
if !ok || tor.AllInProgress() {
continue
}
buf.WriteString(fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", filepath.Join(directory, url.PathEscape(tor.AccessKey)), tor.AccessKey))
buf.WriteString(fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", filepath.Join(directory, url.PathEscape(torMgr.GetKey(tor))), torMgr.GetKey(tor)))
}
return buf.Bytes(), nil
}

2
internal/torrent/delete.go
View File

@@ -15,7 +15,7 @@ func (t *TorrentManager) CheckDeletedStatus(torrent *Torrent) bool {
infoCache, _ := t.DirectoryMap.Get(INT_INFO_CACHE)
torrent.DownloadedIDs.Each(func(id string) bool {
infoCache.Set(id, torrent)
t.writeTorrentToFile(id, torrent)
t.writeTorrentToFile(id, torrent, false)
return false
})
}

26
internal/torrent/manager.go
View File

@@ -51,7 +51,7 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p
// create internal directories
t.DirectoryMap = cmap.New[cmap.ConcurrentMap[string, *Torrent]]()
t.DirectoryMap.Set(INT_ALL, cmap.New[*Torrent]()) // key is AccessKey
t.DirectoryMap.Set(INT_ALL, cmap.New[*Torrent]()) // key is GetAccessKey()
t.DirectoryMap.Set(INT_INFO_CACHE, cmap.New[*Torrent]()) // key is Torrent ID
// create directory maps
for _, directory := range cfg.GetDirectories() {
@@ -165,7 +165,7 @@ func (t *TorrentManager) assignedDirectoryCb(tor *Torrent, cb func(string)) {
cb(config.UNPLAYABLE_TORRENTS)
break
}
if t.Config.MeetsConditions(directory, tor.AccessKey, tor.ComputeTotalSize(), torrentIDs, filenames, fileSizes) {
if t.Config.MeetsConditions(directory, t.GetKey(tor), tor.ComputeTotalSize(), torrentIDs, filenames, fileSizes) {
cb(directory)
break
}
@@ -174,21 +174,24 @@ func (t *TorrentManager) assignedDirectoryCb(tor *Torrent, cb func(string)) {
}
}
func (t *TorrentManager) computeAccessKey(name, originalName string) string {
func (t *TorrentManager) GetKey(torrent *Torrent) string {
if t.Config.ShouldAllowRenames() && torrent.Rename != "" {
return torrent.Rename
}
if t.Config.EnableRetainRDTorrentName() {
return name
return torrent.Name
}
// drop the extension from the name
if t.Config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
return name
if t.Config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
return torrent.Name
} else {
ret := strings.TrimSuffix(originalName, ".mp4")
ret := strings.TrimSuffix(torrent.OriginalName, ".mp4")
ret = strings.TrimSuffix(ret, ".mkv")
return ret
}
}
func (t *TorrentManager) writeTorrentToFile(instanceID string, torrent *Torrent) {
func (t *TorrentManager) writeTorrentToFile(instanceID string, torrent *Torrent, overwriteNames bool) {
filePath := "data/" + instanceID + ".json"
file, err := os.Create(filePath)
if err != nil {
@@ -197,6 +200,13 @@ func (t *TorrentManager) writeTorrentToFile(instanceID string, torrent *Torrent)
}
defer file.Close()
if !overwriteNames {
infoCache, _ := t.DirectoryMap.Get(INT_INFO_CACHE)
if cachedTorrent, exists := infoCache.Get(instanceID); exists {
torrent.Name = cachedTorrent.Name
torrent.OriginalName = cachedTorrent.OriginalName
}
}
torrent.Version = t.requiredVersion
jsonData, err := json.Marshal(torrent)

35
internal/torrent/refresh.go
View File

@@ -44,13 +44,13 @@ func (t *TorrentManager) RefreshTorrents() []string {
continue
}
if !info.AnyInProgress() {
freshKeys.Add(info.AccessKey)
freshKeys.Add(t.GetKey(info))
}
if torrent, exists := allTorrents.Get(info.AccessKey); !exists {
allTorrents.Set(info.AccessKey, info)
if torrent, exists := allTorrents.Get(t.GetKey(info)); !exists {
allTorrents.Set(t.GetKey(info), info)
} else if !info.DownloadedIDs.Difference(torrent.DownloadedIDs).IsEmpty() {
mainTorrent := t.mergeToMain(torrent, info)
allTorrents.Set(info.AccessKey, &mainTorrent)
allTorrents.Set(t.GetKey(info), &mainTorrent)
}
}
t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
@@ -130,9 +130,10 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
}
torrent := Torrent{
AccessKey: t.computeAccessKey(info.Name, info.OriginalName),
LatestAdded: info.Added,
Hash: info.Hash,
Name: info.Name,
OriginalName: info.OriginalName,
LatestAdded: info.Added,
Hash: info.Hash,
}
// SelectedFiles is a subset of Files with only the selected ones
// it also has a Link field, which can be empty
@@ -180,21 +181,23 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
torrent.InProgressIDs.Add(info.ID)
}
t.writeTorrentToFile(rdTorrent.ID, &torrent)
t.writeTorrentToFile(rdTorrent.ID, &torrent, true)
infoCache.Set(rdTorrent.ID, &torrent)
return &torrent
}
func (t *TorrentManager) mergeToMain(existing, toMerge *Torrent) Torrent {
mainTorrent := Torrent{}
mainTorrent.AccessKey = existing.AccessKey
mainTorrent.Hash = existing.Hash
mainTorrent.DownloadedIDs = mapset.NewSet[string]()
mainTorrent.InProgressIDs = mapset.NewSet[string]()
mainTorrent.Unfixable = existing.Unfixable || toMerge.Unfixable
mainTorrent.UnassignedLinks = existing.UnassignedLinks.Union(toMerge.UnassignedLinks)
mainTorrent := Torrent{
Name: existing.Name,
OriginalName: existing.OriginalName,
Rename: existing.Rename,
Hash: existing.Hash,
DownloadedIDs: mapset.NewSet[string](),
InProgressIDs: mapset.NewSet[string](),
Unfixable: existing.Unfixable || toMerge.Unfixable,
UnassignedLinks: existing.UnassignedLinks.Union(toMerge.UnassignedLinks),
}
// this function triggers only when we have a new DownloadedID
toMerge.DownloadedIDs.Difference(existing.DownloadedIDs).Each(func(id string) bool {

48
internal/torrent/repair.go
View File

@@ -96,7 +96,7 @@ func (t *TorrentManager) repairAll() {
t.log.Debugf("Found %d broken torrents to repair in total", len(toRepair))
for i := range toRepair {
torrent := toRepair[i]
t.log.Infof("Repairing %s", torrent.AccessKey)
t.log.Infof("Repairing %s", t.GetKey(torrent))
t.repair(torrent)
}
}
@@ -105,19 +105,19 @@ func (t *TorrentManager) Repair(torrent *Torrent) {
infoCache, _ := t.DirectoryMap.Get(INT_INFO_CACHE)
torrent.DownloadedIDs.Each(func(id string) bool {
infoCache.Set(id, torrent)
t.writeTorrentToFile(id, torrent)
t.writeTorrentToFile(id, torrent, false)
return false
})
_ = t.repairWorker.Submit(func() {
t.log.Infof("Repairing torrent %s", torrent.AccessKey)
t.log.Infof("Repairing torrent %s", t.GetKey(torrent))
t.repair(torrent)
t.log.Infof("Finished repairing torrent %s", torrent.AccessKey)
t.log.Infof("Finished repairing torrent %s", t.GetKey(torrent))
})
}
func (t *TorrentManager) repair(torrent *Torrent) {
if torrent.AnyInProgress() {
t.log.Infof("Torrent %s is in progress, skipping repair until download is done", torrent.AccessKey)
t.log.Infof("Torrent %s is in progress, skipping repair until download is done", t.GetKey(torrent))
return
}
@@ -129,18 +129,18 @@ func (t *TorrentManager) repair(torrent *Torrent) {
if torrent.OlderThanDuration(EXPIRED_LINK_TOLERANCE_HOURS * time.Hour) {
// first solution: reinsert with same selection
t.log.Infof("Torrent %s is older than %d hours, reinserting it", torrent.AccessKey, EXPIRED_LINK_TOLERANCE_HOURS)
t.log.Infof("Torrent %s is older than %d hours, reinserting it", t.GetKey(torrent), EXPIRED_LINK_TOLERANCE_HOURS)
if t.reinsertTorrent(torrent, "") {
t.log.Infof("Successfully downloaded torrent %s to repair it", torrent.AccessKey)
t.log.Infof("Successfully downloaded torrent %s to repair it", t.GetKey(torrent))
return
} else if torrent.Unfixable {
t.log.Warnf("Cannot repair torrent %s", torrent.AccessKey)
t.log.Warnf("Cannot repair torrent %s", t.GetKey(torrent))
return
} else {
t.log.Warnf("Failed to repair by reinserting torrent %s, will only redownload broken files...", torrent.AccessKey)
t.log.Warnf("Failed to repair by reinserting torrent %s, will only redownload broken files...", t.GetKey(torrent))
}
} else {
t.log.Warnf("Torrent %s is not older than %d hours to be repaired by reinsertion, will only redownload broken files...", torrent.AccessKey, EXPIRED_LINK_TOLERANCE_HOURS)
t.log.Warnf("Torrent %s is not older than %d hours to be repaired by reinsertion, will only redownload broken files...", t.GetKey(torrent), EXPIRED_LINK_TOLERANCE_HOURS)
}
// sleep for 30 seconds to let the torrent accumulate more broken files if scanning
@@ -174,12 +174,12 @@ func (t *TorrentManager) repair(torrent *Torrent) {
})
if assignedCount > 0 {
t.log.Infof("Assigned %d links to selected files for torrent %s", assignedCount, torrent.AccessKey)
t.log.Infof("Assigned %d links to selected files for torrent %s", assignedCount, t.GetKey(torrent))
} else if rarCount > 0 {
// this is a rar'ed torrent, nothing we can do
if t.Config.ShouldDeleteRarFiles() {
t.log.Warnf("Torrent %s is rar'ed and we cannot repair it, deleting it as configured", torrent.AccessKey)
t.Delete(torrent.AccessKey, true)
t.log.Warnf("Torrent %s is rar'ed and we cannot repair it, deleting it as configured", t.GetKey(torrent))
t.Delete(t.GetKey(torrent), true)
} else {
for _, unassigned := range unassignedDownloads {
newFile := &File{
@@ -208,14 +208,14 @@ func (t *TorrentManager) repair(torrent *Torrent) {
file.Link = "repairing"
}
})
t.log.Debugf("During repair, zurg found %d broken files for torrent %s", len(brokenFiles), torrent.AccessKey)
t.log.Debugf("During repair, zurg found %d broken files for torrent %s", len(brokenFiles), t.GetKey(torrent))
if len(brokenFiles) == 1 && torrent.SelectedFiles.Count() > 2 {
// if we download a single file, it will be named differently
// so we need to download 1 extra file to preserve the name
// this is only relevant if we enable retain_rd_torrent_name
// add the first file link encountered with a prefix of http
t.log.Debugf("Torrent %s has only 1 broken file, adding 1 extra file to preserve the name", torrent.AccessKey)
t.log.Debugf("Torrent %s has only 1 broken file, adding 1 extra file to preserve the name", t.GetKey(torrent))
for _, file := range torrent.SelectedFiles.Items() {
if strings.HasPrefix(file.Link, "http") {
brokenFiles = append(brokenFiles, *file)
@@ -225,15 +225,15 @@ func (t *TorrentManager) repair(torrent *Torrent) {
}
if len(brokenFiles) > 0 {
t.log.Infof("Redownloading %dof%d files for torrent %s", len(brokenFiles), torrent.SelectedFiles.Count(), torrent.AccessKey)
t.log.Infof("Redownloading %dof%d files for torrent %s", len(brokenFiles), torrent.SelectedFiles.Count(), t.GetKey(torrent))
brokenFileIDs := strings.Join(getFileIDs(brokenFiles), ",")
if t.reinsertTorrent(torrent, brokenFileIDs) {
t.log.Infof("Successfully downloaded torrent %s to repair it", torrent.AccessKey)
t.log.Infof("Successfully downloaded torrent %s to repair it", t.GetKey(torrent))
} else {
t.log.Warnf("Cannot repair torrent %s", torrent.AccessKey)
t.log.Warnf("Cannot repair torrent %s", t.GetKey(torrent))
}
} else {
t.log.Warnf("Torrent %s has no broken files to repair", torrent.AccessKey)
t.log.Warnf("Torrent %s has no broken files to repair", t.GetKey(torrent))
}
}
@@ -356,22 +356,22 @@ func (t *TorrentManager) canCapacityHandle() bool {
}
func (t *TorrentManager) markAsUnplayable(torrent *Torrent) {
t.log.Warnf("Marking torrent %s as unplayable", torrent.AccessKey)
t.log.Warnf("Marking torrent %s as unplayable", t.GetKey(torrent))
t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
torrents.Remove(torrent.AccessKey)
torrents.Remove(t.GetKey(torrent))
})
torrents, _ := t.DirectoryMap.Get(config.UNPLAYABLE_TORRENTS)
torrents.Set(torrent.AccessKey, torrent)
torrents.Set(t.GetKey(torrent), torrent)
}
func (t *TorrentManager) markAsUnfixable(torrent *Torrent) {
t.log.Warnf("Marking torrent %s as unfixable", torrent.AccessKey)
t.log.Warnf("Marking torrent %s as unfixable", t.GetKey(torrent))
torrent.Unfixable = true
infoCache, _ := t.DirectoryMap.Get(INT_INFO_CACHE)
torrent.DownloadedIDs.Each(func(id string) bool {
info, _ := infoCache.Get(id)
info.Unfixable = true
t.writeTorrentToFile(id, torrent)
t.writeTorrentToFile(id, torrent, false)
return false
})
}

4
internal/torrent/types.go
View File

@@ -14,7 +14,9 @@ import (
var json = jsoniter.ConfigCompatibleWithStandardLibrary
type Torrent struct {
AccessKey string `json:"AccessKey"`
Name string `json:"Name"`
OriginalName string `json:"OriginalName"`
Rename string `json:"Rename"`
Hash string `json:"Hash"`
SelectedFiles cmap.ConcurrentMap[string, *File] `json:"-"`
LatestAdded string `json:"LatestAdded"`