hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Create a torrentMap

Browse Source
This commit is contained in:
Ben Sarmiento
2023-11-08 20:20:52 +01:00
parent 17cd7ae1f4
commit 9dfd6c32d5
16 changed files with 405 additions and 324 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
internal/torrent/manager.go
View File

@@ -18,7 +18,9 @@ import (
type TorrentManager struct {
requiredVersion string
rd *realdebrid.RealDebrid
torrents []Torrent
torrentMap map[string]*Torrent
inProgress []string
checksum string
config config.ConfigInterface
@@ -35,7 +37,9 @@ type TorrentManager struct {
// and store them in-memory; it is called only once at startup
func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[string, string]) *TorrentManager {
t := &TorrentManager{
requiredVersion: fmt.Sprintf("4.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
requiredVersion: fmt.Sprintf("8.11.2023 - retain:%v", config.EnableRetainFolderNameExtension()),
rd: realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")),
torrentMap: make(map[string]*Torrent),
config: config,
cache: cache,
workerPool: make(chan bool, config.GetNumOfWorkers()),
@@ -45,32 +49,49 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
log: logutil.NewLogger().Named("manager"),
}
// Initialize torrents for the first time
// start with a clean slate
t.mu.Lock()
t.torrents = t.getFreshListFromAPI()
t.checksum = t.getChecksum()
t.mu.Unlock()
t.cache.Purge()
t.torrents = nil
// log.Println("First checksum", t.checksum)
newTorrents, _, err := t.rd.GetTorrents(0)
if err != nil {
t.log.Fatalf("Cannot get torrents: %v\n", err)
}
torrentsChan := make(chan *Torrent, len(newTorrents))
var wg sync.WaitGroup
for i := range t.torrents {
for i := range newTorrents {
wg.Add(1)
go func(idx int) {
defer wg.Done()
t.workerPool <- true
t.addMoreInfo(&t.torrents[idx])
torrentsChan <- t.getMoreInfo(newTorrents[idx])
<-t.workerPool
}(i)
}
wg.Wait()
close(torrentsChan)
for newTorrent := range torrentsChan {
if newTorrent == nil {
continue
}
t.torrents = append(t.torrents, *newTorrent)
if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
} else {
t.torrentMap[newTorrent.AccessKey] = newTorrent
}
}
t.checksum = t.getChecksum()
t.mu.Unlock()
if t.config.EnableRepair() {
go t.repairAll(&wg)
go t.repairAll()
}
wg.Wait()
t.mapToDirectories()
go t.startRefreshJob()
return t
@@ -80,7 +101,7 @@ func NewTorrentManager(config config.ConfigInterface, cache *expirable.LRU[strin
func (t *TorrentManager) GetByDirectory(directory string) []Torrent {
var torrents []Torrent
for i := range t.torrents {
for _, dir := range t.directoryMap[t.torrents[i].Name] {
for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
if dir == directory {
torrents = append(torrents, t.torrents[i])
}
@@ -100,13 +121,18 @@ func (t *TorrentManager) FindAllTorrentsWithName(directory, torrentName string)
var matchingTorrents []Torrent
torrents := t.GetByDirectory(directory)
for i := range torrents {
if torrents[i].Name == torrentName || strings.HasPrefix(torrents[i].Name, torrentName) {
if torrents[i].AccessKey == torrentName || strings.Contains(torrents[i].AccessKey, torrentName) {
matchingTorrents = append(matchingTorrents, torrents[i])
}
}
return matchingTorrents
}
// proxy
func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.UnrestrictResponse {
return t.rd.UnrestrictUntilOk(link)
}
// findAllDownloadedFilesFromHash finds all files that were with a given hash
func (t *TorrentManager) findAllDownloadedFilesFromHash(hash string) []File {
var files []File
@@ -134,7 +160,7 @@ func (t *TorrentManager) getChecksum() string {
// GetTorrents request
go func() {
torrents, totalCount, err := realdebrid.GetTorrents(t.config.GetToken(), 1)
torrents, totalCount, err := t.rd.GetTorrents(1)
if err != nil {
errChan <- err
return
@@ -144,7 +170,7 @@ func (t *TorrentManager) getChecksum() string {
// GetActiveTorrentCount request
go func() {
count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
count, err := t.rd.GetActiveTorrentCount()
if err != nil {
errChan <- err
return
@@ -186,92 +212,79 @@ func (t *TorrentManager) startRefreshJob() {
if checksum == t.checksum {
continue
}
t.mu.Lock()
t.cache.Purge()
t.torrents = nil
newTorrents := t.getFreshListFromAPI()
newTorrents, _, err := t.rd.GetTorrents(0)
if err != nil {
t.log.Errorf("Cannot get torrents: %v\n", err)
continue
}
torrentsChan := make(chan *Torrent)
var wg sync.WaitGroup
for i := range newTorrents {
wg.Add(1)
go func(idx int) {
defer wg.Done()
t.log.Debug(newTorrents[idx].ID)
t.workerPool <- true
t.addMoreInfo(&newTorrents[idx])
torrentsChan <- t.getMoreInfo(newTorrents[idx])
<-t.workerPool
}(i)
}
wg.Wait()
// apply side effects
t.mu.Lock()
t.torrents = newTorrents
close(torrentsChan)
for newTorrent := range torrentsChan {
if newTorrent == nil {
continue
}
t.torrents = append(t.torrents, *newTorrent)
if _, exists := t.torrentMap[newTorrent.AccessKey]; exists {
t.torrentMap[newTorrent.AccessKey].Files = newTorrent.Files
t.torrentMap[newTorrent.AccessKey].Links = newTorrent.Links
t.torrentMap[newTorrent.AccessKey].SelectedFiles = newTorrent.SelectedFiles
t.torrentMap[newTorrent.AccessKey].ForRepair = newTorrent.ForRepair
} else {
t.torrentMap[newTorrent.AccessKey] = newTorrent
}
}
t.checksum = t.getChecksum()
t.mu.Unlock()
// log.Println("Checksum changed", t.checksum)
if t.config.EnableRepair() {
go t.repairAll(&wg)
go t.repairAll()
}
go t.mapToDirectories()
go OnLibraryUpdateHook(t.config)
}
}
// getFreshListFromAPI returns all torrents
func (t *TorrentManager) getFreshListFromAPI() []Torrent {
torrents, _, err := realdebrid.GetTorrents(t.config.GetToken(), 0)
if err != nil {
t.log.Errorf("Cannot get torrents: %v\n", err)
return nil
}
// convert to own internal type without SelectedFiles yet
// populate inProgress
var torrentsV2 []Torrent
t.inProgress = t.inProgress[:0] // reset
for _, torrent := range torrents {
torrent.Name = strings.TrimSuffix(torrent.Name, "/")
torrentV2 := Torrent{
Torrent: torrent,
SelectedFiles: nil,
ForRepair: false,
lock: &sync.Mutex{},
}
torrentsV2 = append(torrentsV2, torrentV2)
if torrent.Progress != 100 {
t.inProgress = append(t.inProgress, torrent.Hash)
}
}
t.log.Infof("Fetched %d torrents", len(torrentsV2))
return torrentsV2
}
// addMoreInfo updates the selected files for a torrent
func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
// getMoreInfo updates the selected files for a torrent
func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
t.log.Info("Getting more info for", rdTorrent.ID)
// file cache
torrentFromFile := t.readFromFile(torrent.ID)
torrentFromFile := t.readFromFile(rdTorrent.ID)
if torrentFromFile != nil {
// see if api data and file data still match
// then it means data is still usable
if len(torrentFromFile.Links) == len(torrent.Links) {
torrent.Name = t.getName(torrentFromFile)
torrent.ForRepair = torrentFromFile.ForRepair
torrent.SelectedFiles = torrentFromFile.SelectedFiles[:]
return
if len(torrentFromFile.Links) == len(rdTorrent.Links) {
return torrentFromFile
}
}
// no file data yet as it is still downloading
if torrent.Progress != 100 {
return
t.log.Debug("Getting info for", rdTorrent.ID)
info, err := t.rd.GetTorrentInfo(rdTorrent.ID)
if err != nil {
t.log.Errorf("Cannot get info for id=%s: %v\n", rdTorrent.ID, err)
return nil
}
// t.log.Println("Getting info for", torrent.ID)
info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), torrent.ID)
if err != nil {
t.log.Errorf("Cannot get info: %v\n", err)
return
torrent := Torrent{
Version: t.requiredVersion,
Torrent: *info,
SelectedFiles: nil,
ForRepair: false,
}
// SelectedFiles is a subset of Files with only the selected ones
@@ -321,23 +334,25 @@ func (t *TorrentManager) addMoreInfo(torrent *Torrent) {
selectedFiles[i].Link = link
}
}
torrent.ForRepair = forRepair
torrent.SelectedFiles = selectedFiles
torrent.AccessKey = t.getName(info.Name, info.OriginalName)
// update file cache
torrent.OriginalName = info.OriginalName
torrent.Name = t.getName(torrent)
if len(selectedFiles) > 0 {
// update the torrent with more data!
torrent.SelectedFiles = selectedFiles
torrent.ForRepair = forRepair
t.writeToFile(torrent)
t.writeToFile(&torrent)
}
t.log.Debugf("Got info for %s %s", torrent.ID, torrent.AccessKey)
return &torrent
}
func (t *TorrentManager) getName(torrent *Torrent) string {
func (t *TorrentManager) getName(name, originalName string) string {
// drop the extension from the name
if t.config.EnableRetainFolderNameExtension() && strings.Contains(torrent.Name, torrent.OriginalName) {
return torrent.Name
if t.config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
return name
} else {
ret := strings.TrimSuffix(torrent.OriginalName, ".mp4")
ret := strings.TrimSuffix(originalName, ".mp4")
ret = strings.TrimSuffix(ret, ".mkv")
return ret
}
@@ -357,7 +372,7 @@ func (t *TorrentManager) mapToDirectories() {
for i := range t.torrents {
// don't process torrents that are already mapped if it is not the first run
alreadyMappedToGroup := false
for _, mappedGroup := range t.processedTorrents[t.torrents[i].Name] {
for _, mappedGroup := range t.processedTorrents[t.torrents[i].AccessKey] {
if mappedGroup == group {
alreadyMappedToGroup = true
}
@@ -371,10 +386,10 @@ func (t *TorrentManager) mapToDirectories() {
for _, file := range t.torrents[i].SelectedFiles {
filenames = append(filenames, file.Path)
}
if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].Name, filenames) {
if configV1.MeetsConditions(directory, t.torrents[i].ID, t.torrents[i].AccessKey, filenames) {
found := false
// check if it is already mapped to this directory
for _, dir := range t.directoryMap[t.torrents[i].Name] {
for _, dir := range t.directoryMap[t.torrents[i].AccessKey] {
if dir == directory {
found = true
break // it is already mapped to this directory
@@ -383,14 +398,14 @@ func (t *TorrentManager) mapToDirectories() {
if !found {
counter[directory]++
t.mu.Lock()
t.directoryMap[t.torrents[i].Name] = append(t.directoryMap[t.torrents[i].Name], directory)
t.directoryMap[t.torrents[i].AccessKey] = append(t.directoryMap[t.torrents[i].AccessKey], directory)
t.mu.Unlock()
break // we found a directory for this torrent, so we can stop looking for more
}
}
}
t.mu.Lock()
t.processedTorrents[t.torrents[i].Name] = append(t.processedTorrents[t.torrents[i].Name], group)
t.processedTorrents[t.torrents[i].AccessKey] = append(t.processedTorrents[t.torrents[i].AccessKey], group)
t.mu.Unlock()
}
sum := 0
@@ -408,6 +423,33 @@ func (t *TorrentManager) mapToDirectories() {
}
}
func (t *TorrentManager) getDirectories(torrent *Torrent) []string {
var ret []string
// Map torrents to directories
switch t.config.GetVersion() {
case "v1":
configV1 := t.config.(*config.ZurgConfigV1)
groupMap := configV1.GetGroupMap()
// for every group, iterate over every torrent
// and then sprinkle/distribute the torrents to the directories of the group
for _, directories := range groupMap {
for _, directory := range directories {
var filenames []string
for _, file := range torrent.SelectedFiles {
filenames = append(filenames, file.Path)
}
if configV1.MeetsConditions(directory, torrent.ID, torrent.AccessKey, filenames) {
ret = append(ret, directory)
break // we found a directory for this torrent for this group, so we can stop looking for more
}
}
}
default:
t.log.Error("Unknown config version")
}
return ret
}
// getByID returns a torrent by its ID
func (t *TorrentManager) getByID(torrentID string) *Torrent {
for i := range t.torrents {
@@ -463,19 +505,18 @@ func (t *TorrentManager) readFromFile(torrentID string) *Torrent {
return &torrent
}
func (t *TorrentManager) repairAll(wg *sync.WaitGroup) {
wg.Wait()
func (t *TorrentManager) repairAll() {
for _, torrent := range t.torrents {
if torrent.ForRepair {
t.log.Infof("Issues were detected on %s %s; fixing...", torrent.ID, torrent.Name)
t.log.Infof("There were less links than was expected on %s %s; fixing...", torrent.ID, torrent.AccessKey)
t.repair(torrent.ID, torrent.SelectedFiles, true)
}
if len(torrent.Links) == 0 && torrent.Progress == 100 {
// If the torrent has no links
// and already processing repair
// delete it!
t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.Name)
realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
t.log.Infof("Deleting broken torrent %s %s as it doesn't contain any files", torrent.ID, torrent.AccessKey)
t.rd.DeleteTorrent(torrent.ID)
}
}
}
@@ -518,12 +559,12 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
}
}
if len(missingFiles) == 0 {
t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.Name)
t.log.Infof("Torrent %s %s is already repaired", torrent.ID, torrent.AccessKey)
return
}
// then we repair it!
t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.Name)
t.log.Infof("Repairing torrent %s %s", torrent.ID, torrent.AccessKey)
// check if we can still add more downloads
proceed := t.canCapacityHandle()
if !proceed {
@@ -573,7 +614,7 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
t.log.Info("No other missing files left to reinsert")
}
} else {
t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.Name)
t.log.Infof("Torrent %s %s is unfixable as the only link cached in RD is already broken", torrent.ID, torrent.AccessKey)
t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", torrent.Hash)
return
}
@@ -584,7 +625,7 @@ func (t *TorrentManager) repair(torrentID string, selectedFiles []File, tryReins
func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string, deleteIfFailed bool) bool {
// if missingFiles is not provided, look for missing files
if missingFiles == "" {
t.log.Info("Redownloading whole torrent", torrent.Name)
t.log.Info("Redownloading whole torrent", torrent.AccessKey)
var selection string
for _, file := range torrent.SelectedFiles {
selection += fmt.Sprintf("%d,", file.ID)
@@ -598,29 +639,29 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string,
}
// redownload torrent
resp, err := realdebrid.AddMagnetHash(t.config.GetToken(), torrent.Hash)
resp, err := t.rd.AddMagnetHash(torrent.Hash)
if err != nil {
t.log.Errorf("Cannot redownload torrent: %v", err)
return false
}
newTorrentID := resp.ID
err = realdebrid.SelectTorrentFiles(t.config.GetToken(), newTorrentID, missingFiles)
err = t.rd.SelectTorrentFiles(newTorrentID, missingFiles)
if err != nil {
t.log.Errorf("Cannot start redownloading: %v", err)
}
if deleteIfFailed {
if err != nil {
realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
t.rd.DeleteTorrent(newTorrentID)
return false
}
time.Sleep(1 * time.Second)
// see if the torrent is ready
info, err := realdebrid.GetTorrentInfo(t.config.GetToken(), newTorrentID)
info, err := t.rd.GetTorrentInfo(newTorrentID)
if err != nil {
t.log.Errorf("Cannot get info on redownloaded torrent: %v", err)
if deleteIfFailed {
realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
t.rd.DeleteTorrent(newTorrentID)
}
return false
}
@@ -628,17 +669,17 @@ func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string,
if info.Progress != 100 {
t.log.Infof("Torrent is not cached anymore so we have to wait until completion, currently %d%%", info.Progress)
realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
t.rd.DeleteTorrent(newTorrentID)
return false
}
if len(info.Links) != len(torrent.SelectedFiles) {
t.log.Infof("It didn't fix the issue, only got %d files but we need %d, undoing", len(info.Links), len(torrent.SelectedFiles))
realdebrid.DeleteTorrent(t.config.GetToken(), newTorrentID)
t.rd.DeleteTorrent(newTorrentID)
return false
}
t.log.Info("Redownload successful, deleting old torrent")
realdebrid.DeleteTorrent(t.config.GetToken(), torrent.ID)
t.rd.DeleteTorrent(torrent.ID)
}
return true
}
@@ -652,43 +693,43 @@ func (t *TorrentManager) organizeChaos(info *realdebrid.Torrent, selectedFiles [
var wg sync.WaitGroup
// Limit concurrency
sem := make(chan struct{}, t.config.GetNumOfWorkers())
sem := make(chan bool, t.config.GetNumOfWorkers())
for _, link := range info.Links {
wg.Add(1)
sem <- struct{}{}
sem <- true
go func(lnk string) {
defer wg.Done()
defer func() { <-sem }()
unrestrictFn := func() (*realdebrid.UnrestrictResponse, error) {
return realdebrid.UnrestrictCheck(t.config.GetToken(), lnk)
}
resp := realdebrid.RetryUntilOk(unrestrictFn)
if resp != nil {
resultsChan <- Result{Response: resp}
}
resp := t.rd.UnrestrictUntilOk(lnk)
resultsChan <- Result{Response: resp}
}(link)
}
go func() {
t.log.Debugf("Checking %d link(s) for problematic torrent id=%s", len(info.Links), info.ID)
wg.Wait()
close(sem)
close(resultsChan)
t.log.Debugf("Closing results channel for torrent id=%s, checking...", info.ID)
}()
isChaotic := false
for result := range resultsChan {
if result.Response == nil {
continue
}
found := false
for i := range selectedFiles {
if strings.HasSuffix(selectedFiles[i].Path, result.Response.Filename) {
if strings.Contains(selectedFiles[i].Path, result.Response.Filename) {
t.log.Debugf("Found a file that is in the selection for torrent id=%s: %s", info.ID, result.Response.Filename)
selectedFiles[i].Link = result.Response.Link
found = true
}
}
if !found {
// "chaos" file, we don't know where it belongs
isChaotic = !isStreamable(result.Response.Filename)
isChaotic = result.Response.Streamable == 0
t.log.Debugf("Found a file that is not in the selection for torrent id=%s: %s %v", info.ID, result.Response.Filename, result.Response.Streamable)
selectedFiles = append(selectedFiles, File{
File: realdebrid.File{
Path: result.Response.Filename,
@@ -710,7 +751,7 @@ func (t *TorrentManager) canCapacityHandle() bool {
const maxDelay = 60 * time.Second
retryCount := 0
for {
count, err := realdebrid.GetActiveTorrentCount(t.config.GetToken())
count, err := t.rd.GetActiveTorrentCount()
if err != nil {
t.log.Errorf("Cannot get active downloads count: %v", err)
if retryCount >= maxRetries {