hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Big refactor

Browse Source
This commit is contained in:
Ben Sarmiento
2023-11-14 01:55:00 +01:00
parent 5e723a9e8a
commit 16505ec33f
26 changed files with 1602 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
internal/torrent/manager.go
View File

@@ -1,8 +1,10 @@
package torrent
import (
"bufio"
"encoding/gob"
"fmt"
"hash/fnv"
"math"
"os"
"path/filepath"
@@ -19,6 +21,7 @@ import (
type TorrentManager struct {
config config.ConfigInterface
DirectoryMap *orderedmap.OrderedMap[string, *orderedmap.OrderedMap[string, *Torrent]]
TorrentMap *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
repairMap *orderedmap.OrderedMap[string, time.Time] // accessKey -> time last repaired
requiredVersion string
@@ -35,6 +38,7 @@ type TorrentManager struct {
func NewTorrentManager(config config.ConfigInterface, api *realdebrid.RealDebrid) *TorrentManager {
t := &TorrentManager{
config: config,
DirectoryMap: orderedmap.NewOrderedMap[string, *orderedmap.OrderedMap[string, *Torrent]](),
TorrentMap: orderedmap.NewOrderedMap[string, *Torrent](),
repairMap: orderedmap.NewOrderedMap[string, time.Time](),
requiredVersion: "10.11.2023",
@@ -44,8 +48,6 @@ func NewTorrentManager(config config.ConfigInterface, api *realdebrid.RealDebrid
log: logutil.NewLogger().Named("manager"),
}
t.mu.Lock()
newTorrents, _, err := t.api.GetTorrents(0)
if err != nil {
t.log.Fatalf("Cannot get torrents: %v\n", err)
@@ -62,28 +64,34 @@ func NewTorrentManager(config config.ConfigInterface, api *realdebrid.RealDebrid
<-t.workerPool
}(i)
}
t.log.Infof("Got %d torrents", len(newTorrents))
t.log.Infof("Received %d torrents", len(newTorrents))
wg.Wait()
t.log.Infof("Fetched info for %d torrents", len(newTorrents))
close(torrentsChan)
count := 0
for newTorrent := range torrentsChan {
if newTorrent == nil {
count++
continue
}
torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
if torrent != nil {
t.mu.Lock()
t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
t.mu.Unlock()
} else {
t.mu.Lock()
t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
t.mu.Unlock()
}
}
t.log.Infof("Compiled to %d unique movies and shows", t.TorrentMap.Len())
t.log.Infof("Compiled all torrents to %d unique movies and shows, %d were missing info", t.TorrentMap.Len(), count)
t.checksum = t.getChecksum()
t.mu.Unlock()
if t.config.EnableRepair() {
go t.repairAll()
}
go t.startRefreshJob()
// go t.startRefreshJob()
return t
}
@@ -194,8 +202,6 @@ func (t *TorrentManager) startRefreshJob() {
continue
}
t.mu.Lock()
newTorrents, _, err := t.api.GetTorrents(0)
if err != nil {
t.log.Warnf("Cannot get torrents: %v\n", err)
@@ -231,6 +237,15 @@ func (t *TorrentManager) startRefreshJob() {
}
for _, accessKey := range toDelete {
t.TorrentMap.Delete(accessKey)
for el := t.DirectoryMap.Front(); el != nil; el = el.Next() {
torrents := el.Value
for el2 := torrents.Front(); el2 != nil; el2 = el2.Next() {
if el2.Key == accessKey {
torrents.Delete(accessKey)
break
}
}
}
}
wg.Wait()
@@ -241,13 +256,16 @@ func (t *TorrentManager) startRefreshJob() {
}
torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
if torrent != nil {
t.mu.Lock()
t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
t.mu.Unlock()
} else {
t.mu.Lock()
t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
t.mu.Unlock()
}
}
t.checksum = t.getChecksum()
t.mu.Unlock()
if t.config.EnableRepair() {
go t.repairAll()
@@ -293,9 +311,10 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
continue
}
selectedFiles.Set(filepath.Base(file.Path), &File{
File: file,
Added: info.Added,
Link: "", // no link yet
File: file,
Added: info.Added,
Link: "", // no link yet
ZurgFS: hashStringToFh(file.Path + info.Hash),
})
}
if selectedFiles.Len() > len(info.Links) && info.Progress == 100 {
@@ -304,7 +323,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
var isChaotic bool
selectedFiles, isChaotic = t.organizeChaos(info.Links, selectedFiles)
if isChaotic {
// t.log.Warnf("Torrent id=%s %s is unfixable, it is always returning an unstreamable link (it is no longer shown in your directories)", info.ID, info.Name)
t.log.Warnf("Torrent id=%s %s is unrepairable, it is always returning a rar file (it will no longer show up in your directories)", info.ID, info.Name)
// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
return nil
} else {
@@ -316,7 +335,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
t.log.Infof("Torrent id=%s %s marked for repair", info.ID, info.Name)
forRepair = true
} else {
// t.log.Warnf("Torrent id=%s %s is unfixable, the lone streamable link has expired (it is no longer shown in your directories)", info.ID, info.Name)
t.log.Warnf("Torrent id=%s %s is unrepairable, the lone streamable link has expired (it will no longer show up in your directories)", info.ID, info.Name)
// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
return nil
}
@@ -344,12 +363,31 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
InProgress: info.Progress != 100,
Instances: []realdebrid.TorrentInfo{*info},
}
for _, directory := range torrent.Directories {
if _, ok := t.DirectoryMap.Get(directory); !ok {
newMap := orderedmap.NewOrderedMap[string, *Torrent]()
t.mu.Lock()
t.DirectoryMap.Set(directory, newMap)
t.mu.Unlock()
} else {
torrents, _ := t.DirectoryMap.Get(directory)
t.mu.Lock()
torrents.Set(torrent.AccessKey, &torrent)
t.mu.Unlock()
}
}
if selectedFiles.Len() > 0 && torrentFromFile == nil {
t.writeToFile(info) // only when there are selected files, else it's useless
}
return &torrent
}
func hashStringToFh(s string) (fh uint64) {
hasher := fnv.New64a()
_, _ = hasher.Write([]byte(s)) // Write the string to the hasher; ignoring error as it never returns an error
return hasher.Sum64() // Returns a 64-bit hash value
}
func (t *TorrentManager) getName(name, originalName string) string {
// drop the extension from the name
if t.config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
@@ -393,41 +431,40 @@ func (t *TorrentManager) getDirectories(torrent *realdebrid.TorrentInfo) []strin
return ret
}
func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) {
filePath := fmt.Sprintf("data/%s.bin", torrent.ID)
func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) error {
filePath := "data/" + torrent.ID + ".bin"
file, err := os.Create(filePath)
if err != nil {
t.log.Fatalf("Failed creating file: %s", err)
return
return fmt.Errorf("failed creating file: %w", err)
}
defer file.Close()
w := bufio.NewWriter(file)
defer w.Flush()
torrent.Version = t.requiredVersion
dataEncoder := gob.NewEncoder(file)
dataEncoder.Encode(torrent)
dataEncoder := gob.NewEncoder(w)
if err := dataEncoder.Encode(torrent); err != nil {
return fmt.Errorf("failed encoding torrent: %w", err)
}
return nil
}
func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo {
filePath := fmt.Sprintf("data/%s.bin", torrentID)
fileInfo, err := os.Stat(filePath)
if err != nil {
return nil
}
if time.Since(fileInfo.ModTime()) > time.Duration(t.config.GetCacheTimeHours())*time.Hour {
return nil
}
filePath := "data/" + torrentID + ".bin"
file, err := os.Open(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return nil
}
defer file.Close()
r := bufio.NewReader(file)
var torrent realdebrid.TorrentInfo
dataDecoder := gob.NewDecoder(file)
err = dataDecoder.Decode(&torrent)
if err != nil {
dataDecoder := gob.NewDecoder(r)
if err := dataDecoder.Decode(&torrent); err != nil {
return nil
}
if torrent.Version != t.requiredVersion {
@@ -482,8 +519,9 @@ func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap
Bytes: result.Response.Filesize,
Selected: 1,
},
Added: time.Now().Format(time.RFC3339),
Link: result.Response.Link,
Added: time.Now().Format(time.RFC3339),
Link: result.Response.Link,
ZurgFS: hashStringToFh(result.Response.Filename),
})
} else {
isChaotic = true
@@ -495,6 +533,7 @@ func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap
}
func (t *TorrentManager) repairAll() {
t.log.Info("Checking for torrents to repair")
// side note: iteration works!
for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
torrent := el.Value
@@ -515,13 +554,15 @@ func (t *TorrentManager) repairAll() {
if !forRepair {
// if it was marked for repair, unmark it
torrent.ForRepair = false
t.mu.Lock()
t.TorrentMap.Set(torrent.AccessKey, torrent)
t.mu.Unlock()
continue
}
// when getting info, we mark it for repair if it's missing some links
if torrent.ForRepair {
t.log.Infof("Found torrent for repair %s", torrent.AccessKey)
t.log.Infof("Found torrent for repair: %s", torrent.AccessKey)
t.Repair(torrent.AccessKey)
break // only repair the first one for repair and then move on
}
@@ -534,7 +575,9 @@ func (t *TorrentManager) Repair(accessKey string) {
return
}
}
t.mu.Lock()
t.repairMap.Set(accessKey, time.Now())
t.mu.Unlock()
if !t.config.EnableRepair() {
t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair")
@@ -569,7 +612,9 @@ func (t *TorrentManager) Repair(accessKey string) {
}
selectedFiles, _ := t.organizeChaos(links, torrent.SelectedFiles)
torrent.SelectedFiles = selectedFiles
t.mu.Lock()
t.TorrentMap.Set(torrent.AccessKey, torrent)
t.mu.Unlock()
// first solution: add the same selection, maybe it can be fixed by reinsertion?
if t.reinsertTorrent(torrent, "") {

5
internal/torrent/types.go
View File

@@ -18,6 +18,7 @@ type Torrent struct {
type File struct {
realdebrid.File
Added string
Link string
Added string
Link string
ZurgFS uint64
}