hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use a new thread safe map

Browse Source
This commit is contained in:
Ben Sarmiento
2023-11-18 12:53:39 +01:00
parent b669f8d673
commit 0e9302f3b5
15 changed files with 577 additions and 535 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
cmd/zurg/main.go
View File

@@ -6,7 +6,6 @@ import (
"net/http" "net/http"
"os" "os"
"os/signal" "os/signal"
"runtime"
"syscall" "syscall"
"time" "time"
@@ -14,13 +13,11 @@ import (
"github.com/debridmediamanager.com/zurg/internal/net" "github.com/debridmediamanager.com/zurg/internal/net"
"github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/internal/version" "github.com/debridmediamanager.com/zurg/internal/version"
"github.com/debridmediamanager.com/zurg/internal/zfs"
"github.com/debridmediamanager.com/zurg/pkg/chunk" // zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
"github.com/debridmediamanager.com/zurg/pkg/logutil" "github.com/debridmediamanager.com/zurg/pkg/logutil"
"github.com/debridmediamanager.com/zurg/pkg/realdebrid" "github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"github.com/hashicorp/golang-lru/v2/expirable" "github.com/hashicorp/golang-lru/v2/expirable"
"github.com/winfsp/cgofuse/fuse"
) )
func main() { func main() {
@@ -65,29 +62,29 @@ func main() {
} }
}() }()
log.Debugf("Initializing chunk manager, cores: %d", runtime.NumCPU()) // log.Debugf("Initializing chunk manager, cores: %d", runtime.NumCPU())
client := zurghttp.NewHTTPClient(config.GetToken(), 10, config) // client := zurghttp.NewHTTPClient(config.GetToken(), 10, config)
chunkMgr, err := chunk.NewManager( // chunkMgr, err := chunk.NewManager(
"", // in-memory chunks // "", // in-memory chunks
10485760, // 10MB chunk size // 10485760, // 10MB chunk size
max(runtime.NumCPU()/2, 1), // 8 cores/2 = 4 chunks to load ahead // max(runtime.NumCPU()/2, 1), // 8 cores/2 = 4 chunks to load ahead
max(runtime.NumCPU()/2, 1), // 4 check threads // max(runtime.NumCPU()/2, 1), // 4 check threads
max(runtime.NumCPU()-1, 1), // number of chunks that should be read ahead // max(runtime.NumCPU()-1, 1), // number of chunks that should be read ahead
runtime.NumCPU()*2, // total chunks kept in memory // runtime.NumCPU()*2, // total chunks kept in memory
torrentMgr, // torrentMgr,
client) // client)
if nil != err { // if nil != err {
log.Panicf("Failed to initialize chunk manager: %v", err) // log.Panicf("Failed to initialize chunk manager: %v", err)
} // }
fs := zfs.NewZurgFS(torrentMgr, config, chunkMgr, logutil.NewLogger().Named("zfs")) // fs := zfs.NewZurgFS(torrentMgr, config, chunkMgr, logutil.NewLogger().Named("zfs"))
host := fuse.NewFileSystemHost(fs) // host := fuse.NewFileSystemHost(fs)
go func() { // go func() {
log.Infof("Mounting on %s", config.GetMountPoint()) // log.Infof("Mounting on %s", config.GetMountPoint())
if err := zfs.Mount(host, config); err != nil { // if err := zfs.Mount(host, config); err != nil {
log.Panicf("Failed to mount: %v", err) // log.Panicf("Failed to mount: %v", err)
} // }
}() // }()
<-shutdown <-shutdown
@@ -97,9 +94,9 @@ func main() {
if err := server.Shutdown(ctx); err != nil { if err := server.Shutdown(ctx); err != nil {
log.Errorf("Server shutdown error: %v\n", err) log.Errorf("Server shutdown error: %v\n", err)
} }
if err := zfs.Unmount(host); err != nil { // if err := zfs.Unmount(host); err != nil {
log.Errorf("Unmount error: %v\n", err) // log.Errorf("Unmount error: %v\n", err)
} // }
log.Info("BYE") log.Info("BYE")
} }

3
go.mod
View File

@@ -3,13 +3,14 @@ module github.com/debridmediamanager.com/zurg
go 1.21.3 go 1.21.3
require ( require (
github.com/elliotchance/orderedmap/v2 v2.2.0
github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/golang-lru/v2 v2.0.7
go.uber.org/zap v1.26.0 go.uber.org/zap v1.26.0
golang.org/x/sys v0.14.0 golang.org/x/sys v0.14.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
) )
require github.com/orcaman/concurrent-map/v2 v2.0.1
require ( require (
github.com/winfsp/cgofuse v1.5.0 github.com/winfsp/cgofuse v1.5.0
go.uber.org/multierr v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect

4
go.sum
View File

@@ -1,9 +1,9 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=

2
internal/config/load.go
View File

@@ -18,7 +18,7 @@ type ConfigInterface interface {
GetHost() string GetHost() string
GetPort() string GetPort() string
GetDirectories() []string GetDirectories() []string
MeetsConditions(directory, torrentID, torrentName string, fileNames []string) bool MeetsConditions(directory, torrentName string, torrentIDs, fileNames []string) bool
GetOnLibraryUpdate() string GetOnLibraryUpdate() string
GetNetworkBufferSize() int GetNetworkBufferSize() int
GetMountPoint() string GetMountPoint() string

16
internal/config/v1.go
View File

@@ -63,22 +63,26 @@ func (z *ZurgConfigV1) GetGroupMap() map[string][]string {
return result return result
} }
func (z *ZurgConfigV1) MeetsConditions(directory, torrentID, torrentName string, fileNames []string) bool { func (z *ZurgConfigV1) MeetsConditions(directory, torrentName string, torrentIDs, fileNames []string) bool {
if _, ok := z.Directories[directory]; !ok { if _, ok := z.Directories[directory]; !ok {
return false return false
} }
for _, filter := range z.Directories[directory].Filters { for _, filter := range z.Directories[directory].Filters {
if z.matchFilter(torrentID, torrentName, fileNames, filter) { if z.matchFilter(torrentName, torrentIDs, fileNames, filter) {
return true return true
} }
} }
return false return false
} }
func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, fileNames []string, filter *FilterConditionsV1) bool { func (z *ZurgConfigV1) matchFilter(torrentName string, torrentIDs, fileNames []string, filter *FilterConditionsV1) bool {
if filter.ID != "" && fileID == filter.ID { if filter.ID != "" {
for _, torrentID := range torrentIDs {
if torrentID == filter.ID {
return true return true
} }
}
}
if filter.RegexStr != "" { if filter.RegexStr != "" {
regex := compilePattern(filter.RegexStr) regex := compilePattern(filter.RegexStr)
if regex.MatchString(torrentName) { if regex.MatchString(torrentName) {
@@ -100,7 +104,7 @@ func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, fileNames []strin
if len(filter.And) > 0 { if len(filter.And) > 0 {
andResult := true andResult := true
for _, andFilter := range filter.And { for _, andFilter := range filter.And {
andResult = andResult && z.matchFilter(fileID, torrentName, fileNames, andFilter) andResult = andResult && z.matchFilter(torrentName, torrentIDs, fileNames, andFilter)
if !andResult { if !andResult {
return false return false
} }
@@ -109,7 +113,7 @@ func (z *ZurgConfigV1) matchFilter(fileID, torrentName string, fileNames []strin
} }
if len(filter.Or) > 0 { if len(filter.Or) > 0 {
for _, orFilter := range filter.Or { for _, orFilter := range filter.Or {
if z.matchFilter(fileID, torrentName, fileNames, orFilter) { if z.matchFilter(torrentName, torrentIDs, fileNames, orFilter) {
return true return true
} }
} }

69
internal/dav/listing.go
View File

@@ -4,8 +4,10 @@ import (
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"net/http" "net/http"
"net/url"
"path" "path"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/config"
@@ -27,11 +29,11 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
switch { switch {
case len(filteredSegments) == 1 && filteredSegments[0] == "": case len(filteredSegments) == 1 && filteredSegments[0] == "":
output, err = handleRoot(w, r, c) output, err = handleRoot(c)
case len(filteredSegments) == 1: case len(filteredSegments) == 1:
output, err = handleListOfTorrents(requestPath, w, r, t, c) output, err = handleListOfTorrents(requestPath, t)
case len(filteredSegments) == 2: case len(filteredSegments) == 2:
output, err = handleSingleTorrent(requestPath, w, r, t) output, err = handleSingleTorrent(requestPath, t)
default: default:
log.Warnf("Request %s %s not found", r.Method, requestPath) log.Warnf("Request %s %s not found", r.Method, requestPath)
http.Error(w, "Not Found", http.StatusNotFound) http.Error(w, "Not Found", http.StatusNotFound)
@@ -56,7 +58,7 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
} }
} }
func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface) ([]byte, error) { func handleRoot(c config.ConfigInterface) ([]byte, error) {
var responses []dav.Response var responses []dav.Response
responses = append(responses, dav.Directory("")) responses = append(responses, dav.Directory(""))
for _, directory := range c.GetDirectories() { for _, directory := range c.GetDirectories() {
@@ -69,28 +71,23 @@ func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface
return xml.Marshal(rootResponse) return xml.Marshal(rootResponse)
} }
func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) ([]byte, error) { func handleListOfTorrents(requestPath string, t *torrent.TorrentManager) ([]byte, error) {
basePath := path.Base(requestPath) basePath := path.Base(requestPath)
torrents, ok := t.DirectoryMap.Get(basePath)
for _, directory := range c.GetDirectories() { if !ok {
if basePath == directory { return nil, fmt.Errorf("cannot find directory %s", basePath)
}
var responses []dav.Response var responses []dav.Response
// initial response is the directory itself
responses = append(responses, dav.Directory(basePath)) responses = append(responses, dav.Directory(basePath))
for el := t.TorrentMap.Front(); el != nil; el = el.Next() { var accessKeys []string
accessKey := el.Key torrents.IterCb(func(accessKey string, _ *torrent.Torrent) {
torrent := el.Value accessKeys = append(accessKeys, accessKey)
if torrent.InProgress { })
continue sort.Strings(accessKeys)
} for _, accessKey := range accessKeys {
for _, dir := range torrent.Directories { responses = append(responses, dav.Directory(filepath.Join(basePath, accessKey)))
if dir == basePath {
path := filepath.Join(basePath, accessKey)
responses = append(responses, dav.Directory(path))
break
}
}
} }
resp := &dav.MultiStatus{ resp := &dav.MultiStatus{
@@ -98,40 +95,38 @@ func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Req
Response: responses, Response: responses,
} }
return xml.Marshal(resp) return xml.Marshal(resp)
}
}
return nil, fmt.Errorf("cannot find directory when generating list: %s", requestPath)
} }
func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) ([]byte, error) { func handleSingleTorrent(requestPath string, t *torrent.TorrentManager) ([]byte, error) {
basePath := path.Base(path.Dir(requestPath))
torrents, ok := t.DirectoryMap.Get(basePath)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", basePath)
}
accessKey := path.Base(requestPath) accessKey := path.Base(requestPath)
torrent, exists := t.TorrentMap.Get(accessKey) tor, ok := torrents.Get(accessKey)
if !exists { if !ok {
return nil, fmt.Errorf("cannot find torrent %s", accessKey) return nil, fmt.Errorf("cannot find torrent %s", accessKey)
} }
var responses []dav.Response var responses []dav.Response
// initial response is the directory itself // initial response is the directory itself
responses = append(responses, dav.Directory(requestPath)) responses = append(responses, dav.Directory(requestPath))
for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() { tor.SelectedFiles.IterCb(func(filename string, file *torrent.File) {
file := el.Value
if file.Link == "" { if file.Link == "" {
// will be caught by torrent manager's repairAll // will be caught by torrent manager's repairAll
// just skip it for now // just skip it for now
continue return
} }
filename := filepath.Base(file.Path) filePath := filepath.Join(requestPath, url.PathEscape(filename))
filePath := filepath.Join(requestPath, filename)
responses = append(responses, dav.File( responses = append(responses, dav.File(
filePath, filePath,
file.Bytes, file.Bytes,
convertRFC3339toRFC1123(torrent.LatestAdded), convertRFC3339toRFC1123(tor.LatestAdded),
file.Link, file.Link,
)) ))
} })
resp := &dav.MultiStatus{ resp := &dav.MultiStatus{
XMLNS: "DAV:", XMLNS: "DAV:",

63
internal/http/listing.go
View File

@@ -6,6 +6,7 @@ import (
"net/url" "net/url"
"path" "path"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/config"
@@ -24,11 +25,11 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
filteredSegments := removeEmptySegments(strings.Split(requestPath, "/")) filteredSegments := removeEmptySegments(strings.Split(requestPath, "/"))
switch { switch {
case len(filteredSegments) == 1: case len(filteredSegments) == 1:
output, err = handleRoot(w, r, c) output, err = handleRoot(c)
case len(filteredSegments) == 2: case len(filteredSegments) == 2:
output, err = handleListOfTorrents(requestPath, w, r, t, c) output, err = handleListOfTorrents(requestPath, t)
case len(filteredSegments) == 3: case len(filteredSegments) == 3:
output, err = handleSingleTorrent(requestPath, w, r, t) output, err = handleSingleTorrent(requestPath, t)
default: default:
log.Warnf("Request %s %s not found", r.Method, requestPath) log.Warnf("Request %s %s not found", r.Method, requestPath)
http.Error(w, "Not Found", http.StatusNotFound) http.Error(w, "Not Found", http.StatusNotFound)
@@ -51,7 +52,7 @@ func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.T
} }
} }
func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface) (*string, error) { func handleRoot(c config.ConfigInterface) (*string, error) {
htmlDoc := "<ul>" htmlDoc := "<ul>"
for _, directory := range c.GetDirectories() { for _, directory := range c.GetDirectories() {
@@ -62,50 +63,50 @@ func handleRoot(w http.ResponseWriter, r *http.Request, c config.ConfigInterface
return &htmlDoc, nil return &htmlDoc, nil
} }
func handleListOfTorrents(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface) (*string, error) { func handleListOfTorrents(requestPath string, t *torrent.TorrentManager) (*string, error) {
basePath := path.Base(requestPath) basePath := path.Base(requestPath)
torrents, ok := t.DirectoryMap.Get(basePath)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", basePath)
}
for _, directory := range c.GetDirectories() {
if basePath == directory {
htmlDoc := "<ol>" htmlDoc := "<ol>"
for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
accessKey := el.Key var accessKeys []string
torrent := el.Value torrents.IterCb(func(accessKey string, _ *torrent.Torrent) {
if torrent.InProgress { accessKeys = append(accessKeys, accessKey)
continue })
} sort.Strings(accessKeys)
for _, dir := range torrent.Directories { for _, accessKey := range accessKeys {
if dir == basePath { htmlDoc = htmlDoc + fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(accessKey)), accessKey)
htmlDoc += fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(accessKey)), accessKey)
break
}
}
}
return &htmlDoc, nil
}
} }
return nil, fmt.Errorf("cannot find directory when generating list: %s", requestPath) return &htmlDoc, nil
} }
func handleSingleTorrent(requestPath string, w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager) (*string, error) { func handleSingleTorrent(requestPath string, t *torrent.TorrentManager) (*string, error) {
basePath := path.Base(path.Dir(requestPath))
torrents, ok := t.DirectoryMap.Get(basePath)
if !ok {
return nil, fmt.Errorf("cannot find directory %s", basePath)
}
accessKey := path.Base(requestPath) accessKey := path.Base(requestPath)
torrent, _ := t.TorrentMap.Get(accessKey) tor, ok := torrents.Get(accessKey)
if torrent == nil { if !ok {
return nil, fmt.Errorf("cannot find torrent %s", accessKey) return nil, fmt.Errorf("cannot find torrent %s", accessKey)
} }
htmlDoc := "<ol>" htmlDoc := "<ol>"
for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
file := el.Value tor.SelectedFiles.IterCb(func(filename string, file *torrent.File) {
if file.Link == "" { if file.Link == "" {
// will be caught by torrent manager's repairAll // will be caught by torrent manager's repairAll
// just skip it for now // just skip it for now
continue return
} }
filename := filepath.Base(file.Path)
filePath := filepath.Join(requestPath, url.PathEscape(filename)) filePath := filepath.Join(requestPath, url.PathEscape(filename))
htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename) htmlDoc += fmt.Sprintf("<li><a href=\"%s\">%s</a></li>", filePath, filename)
} })
return &htmlDoc, nil return &htmlDoc, nil
} }

703
internal/torrent/manager.go
View File

@@ -15,15 +15,13 @@ import (
"github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/pkg/logutil" "github.com/debridmediamanager.com/zurg/pkg/logutil"
"github.com/debridmediamanager.com/zurg/pkg/realdebrid" "github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"github.com/elliotchance/orderedmap/v2" cmap "github.com/orcaman/concurrent-map/v2"
"go.uber.org/zap" "go.uber.org/zap"
) )
type TorrentManager struct { type TorrentManager struct {
config config.ConfigInterface cfg config.ConfigInterface
DirectoryMap *orderedmap.OrderedMap[string, *orderedmap.OrderedMap[string, *Torrent]] DirectoryMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
TorrentMap *orderedmap.OrderedMap[string, *Torrent] // accessKey -> Torrent
repairMap *orderedmap.OrderedMap[string, time.Time] // accessKey -> time last repaired
requiredVersion string requiredVersion string
checksum string checksum string
api *realdebrid.RealDebrid api *realdebrid.RealDebrid
@@ -35,19 +33,24 @@ type TorrentManager struct {
// NewTorrentManager creates a new torrent manager // NewTorrentManager creates a new torrent manager
// it will fetch all torrents and their info in the background // it will fetch all torrents and their info in the background
// and store them in-memory and cached in files // and store them in-memory and cached in files
func NewTorrentManager(config config.ConfigInterface, api *realdebrid.RealDebrid) *TorrentManager { func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid) *TorrentManager {
t := &TorrentManager{ t := &TorrentManager{
config: config, cfg: cfg,
DirectoryMap: orderedmap.NewOrderedMap[string, *orderedmap.OrderedMap[string, *Torrent]](), DirectoryMap: cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
TorrentMap: orderedmap.NewOrderedMap[string, *Torrent](),
repairMap: orderedmap.NewOrderedMap[string, time.Time](),
requiredVersion: "10.11.2023", requiredVersion: "10.11.2023",
api: api, api: api,
workerPool: make(chan bool, config.GetNumOfWorkers()), workerPool: make(chan bool, cfg.GetNumOfWorkers()),
mu: &sync.Mutex{}, mu: &sync.Mutex{},
log: logutil.NewLogger().Named("manager"), log: logutil.NewLogger().Named("manager"),
} }
// create special directory
t.DirectoryMap.Set("__all__", cmap.New[*Torrent]()) // key is AccessKey
// create directory maps
for _, directory := range cfg.GetDirectories() {
t.DirectoryMap.Set(directory, cmap.New[*Torrent]())
}
newTorrents, _, err := t.api.GetTorrents(0) newTorrents, _, err := t.api.GetTorrents(0)
if err != nil { if err != nil {
t.log.Fatalf("Cannot get torrents: %v\n", err) t.log.Fatalf("Cannot get torrents: %v\n", err)
@@ -60,72 +63,109 @@ func NewTorrentManager(config config.ConfigInterface, api *realdebrid.RealDebrid
go func(idx int) { go func(idx int) {
defer wg.Done() defer wg.Done()
t.workerPool <- true t.workerPool <- true
// TODO wrap getMoreInfo and limit the execution time!
torrentsChan <- t.getMoreInfo(newTorrents[idx]) torrentsChan <- t.getMoreInfo(newTorrents[idx])
<-t.workerPool <-t.workerPool
}(i) }(i)
} }
t.log.Infof("Received %d torrents", len(newTorrents))
wg.Wait() wg.Wait()
t.log.Infof("Fetched info for %d torrents", len(newTorrents))
close(torrentsChan) close(torrentsChan)
count := 0 t.log.Infof("Fetched info for %d torrents", len(newTorrents))
for newTorrent := range torrentsChan {
if newTorrent == nil { noInfoCount := 0
count++ allCt := 0
allTorrents, _ := t.DirectoryMap.Get("__all__")
for info := range torrentsChan {
allCt++
if info == nil {
noInfoCount++
continue continue
} }
torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey) if torrent, exists := allTorrents.Get(info.AccessKey); exists {
if torrent != nil { mainTorrent := t.mergeToMain(torrent, info)
t.mu.Lock() allTorrents.Set(info.AccessKey, mainTorrent)
t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
t.mu.Unlock()
} else { } else {
t.mu.Lock() allTorrents.Set(info.AccessKey, info)
t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
t.mu.Unlock()
} }
} }
t.log.Infof("Compiled all torrents to %d unique movies and shows, %d were missing info", t.TorrentMap.Len(), count)
anotherCt := 0
allTorrents.IterCb(func(accessKey string, torrent *Torrent) {
anotherCt++
// get IDs
var torrentIDs []string
for _, instance := range torrent.Instances {
torrentIDs = append(torrentIDs, instance.ID)
}
// get filenames
var filenames []string
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
filenames = append(filenames, file.Path)
})
// Map torrents to directories
switch t.cfg.GetVersion() {
case "v1":
configV1 := t.cfg.(*config.ZurgConfigV1)
for _, directories := range configV1.GetGroupMap() {
for _, directory := range directories {
if t.cfg.MeetsConditions(directory, torrent.AccessKey, torrentIDs, filenames) {
torrents, _ := t.DirectoryMap.Get(directory)
torrents.Set(accessKey, torrent)
break
}
}
}
}
})
t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
t.checksum = t.getChecksum() t.checksum = t.getChecksum()
if t.config.EnableRepair() { // if t.config.EnableRepair() {
go t.repairAll() // go t.repairAll()
} // }
// go t.startRefreshJob() go t.startRefreshJob()
return t return t
} }
func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent { func (t *TorrentManager) mergeToMain(t1, t2 *Torrent) *Torrent {
merged := t1 mainTorrent := t1
// Merge SelectedFiles // Merge SelectedFiles - itercb accesses a different copy of the selectedfiles map
// side note: iteration works! t2.SelectedFiles.IterCb(func(key string, file *File) {
for el := t2.SelectedFiles.Front(); el != nil; el = el.Next() { // see if it already exists in the main torrent
if _, ok := merged.SelectedFiles.Get(el.Key); !ok { if mainFile, ok := mainTorrent.SelectedFiles.Get(key); !ok {
merged.SelectedFiles.Set(el.Key, el.Value) mainTorrent.SelectedFiles.Set(key, file)
} } else if file.Link != "" && mainFile.Link == "" {
// if it exists, but the link is empty, then we can update it
mainTorrent.SelectedFiles.Set(key, file)
} }
})
// Merge Instances // Merge Instances
merged.Instances = append(t1.Instances, t2.Instances...) mainTorrent.Instances = append(t1.Instances, t2.Instances...)
// LatestAdded // LatestAdded
if t1.LatestAdded < t2.LatestAdded { if t1.LatestAdded < t2.LatestAdded {
merged.LatestAdded = t2.LatestAdded mainTorrent.LatestAdded = t2.LatestAdded
} }
// InProgress - if one of the instances is in progress, then the whole torrent is in progress // InProgress - if one of the instances is in progress, then the whole torrent is in progress
for _, instance := range merged.Instances { mainTorrent.InProgress = false
for _, instance := range mainTorrent.Instances {
if instance.Progress != 100 { if instance.Progress != 100 {
merged.InProgress = true mainTorrent.InProgress = true
} }
if instance.ForRepair { if instance.ForRepair {
merged.ForRepair = true mainTorrent.ForRepair = true
} }
} }
return merged return mainTorrent
} }
// proxy // proxy
@@ -195,7 +235,7 @@ func (t *TorrentManager) getChecksum() string {
func (t *TorrentManager) startRefreshJob() { func (t *TorrentManager) startRefreshJob() {
t.log.Info("Starting periodic refresh") t.log.Info("Starting periodic refresh")
for { for {
<-time.After(time.Duration(t.config.GetRefreshEverySeconds()) * time.Second) <-time.After(time.Duration(t.cfg.GetRefreshEverySeconds()) * time.Second)
checksum := t.getChecksum() checksum := t.getChecksum()
if checksum == t.checksum { if checksum == t.checksum {
@@ -220,57 +260,85 @@ func (t *TorrentManager) startRefreshJob() {
<-t.workerPool <-t.workerPool
}(i) }(i)
} }
wg.Wait()
close(torrentsChan)
t.log.Infof("Fetched info for %d torrents", len(newTorrents))
// side note: iteration works! noInfoCount := 0
allTorrents, _ := t.DirectoryMap.Get("__all__")
var retain []string
for info := range torrentsChan {
if info == nil {
noInfoCount++
continue
}
retain = append(retain, info.AccessKey)
if torrent, exists := allTorrents.Get(info.AccessKey); exists {
mainTorrent := t.mergeToMain(torrent, info)
allTorrents.Set(info.AccessKey, mainTorrent)
} else {
allTorrents.Set(info.AccessKey, info)
}
}
allTorrents.IterCb(func(accessKey string, torrent *Torrent) {
// get IDs
var torrentIDs []string
for _, instance := range torrent.Instances {
torrentIDs = append(torrentIDs, instance.ID)
}
// get filenames
var filenames []string
torrent.SelectedFiles.IterCb(func(_ string, file *File) {
filenames = append(filenames, file.Path)
})
// Map torrents to directories
switch t.cfg.GetVersion() {
case "v1":
configV1 := t.cfg.(*config.ZurgConfigV1)
for _, directories := range configV1.GetGroupMap() {
for _, directory := range directories {
if t.cfg.MeetsConditions(directory, torrent.AccessKey, torrentIDs, filenames) {
torrents, _ := t.DirectoryMap.Get(directory)
torrents.Set(accessKey, torrent)
break
}
}
}
}
})
// delete torrents that no longer exist
var toDelete []string var toDelete []string
for el := t.TorrentMap.Front(); el != nil; el = el.Next() { allTorrents.IterCb(func(_ string, torrent *Torrent) {
found := false found := false
for _, newTorrent := range newTorrents { for _, accessKey := range retain {
if newTorrent.ID == el.Value.AccessKey { if torrent.AccessKey == accessKey {
found = true found = true
break break
} }
} }
if !found { if !found {
toDelete = append(toDelete, el.Key) toDelete = append(toDelete, torrent.AccessKey)
}
} }
})
for _, accessKey := range toDelete { for _, accessKey := range toDelete {
t.TorrentMap.Delete(accessKey) t.DirectoryMap.IterCb(func(_ string, torrents cmap.ConcurrentMap[string, *Torrent]) {
for el := t.DirectoryMap.Front(); el != nil; el = el.Next() { torrents.Remove(accessKey)
torrents := el.Value })
for el2 := torrents.Front(); el2 != nil; el2 = el2.Next() {
if el2.Key == accessKey {
torrents.Delete(accessKey)
break
}
}
}
} }
// end delete torrents that no longer exist
t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
wg.Wait()
close(torrentsChan)
for newTorrent := range torrentsChan {
if newTorrent == nil {
continue
}
torrent, _ := t.TorrentMap.Get(newTorrent.AccessKey)
if torrent != nil {
t.mu.Lock()
t.TorrentMap.Set(newTorrent.AccessKey, t.mergeToMain(torrent, newTorrent))
t.mu.Unlock()
} else {
t.mu.Lock()
t.TorrentMap.Set(newTorrent.AccessKey, newTorrent)
t.mu.Unlock()
}
}
t.checksum = t.getChecksum() t.checksum = t.getChecksum()
if t.config.EnableRepair() { // if t.config.EnableRepair() {
go t.repairAll() // go t.repairAll()
} // }
go OnLibraryUpdateHook(t.config) go OnLibraryUpdateHook(t.cfg)
} }
} }
@@ -299,7 +367,7 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
// it also has a Link field, which can be empty // it also has a Link field, which can be empty
// if it is empty, it means the file is no longer available // if it is empty, it means the file is no longer available
// Files+Links together are the same as SelectedFiles // Files+Links together are the same as SelectedFiles
selectedFiles := orderedmap.NewOrderedMap[string, *File]() selectedFiles := cmap.New[*File]()
streamableCount := 0 streamableCount := 0
// if some Links are empty, we need to repair it // if some Links are empty, we need to repair it
forRepair := false forRepair := false
@@ -317,66 +385,50 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
ZurgFS: hashStringToFh(file.Path + info.Hash), ZurgFS: hashStringToFh(file.Path + info.Hash),
}) })
} }
if selectedFiles.Len() > len(info.Links) && info.Progress == 100 { if selectedFiles.Count() > len(info.Links) && info.Progress == 100 {
// chaotic file means RD will not output the desired file selection // chaotic file means RD will not output the desired file selection
// e.g. even if we select just a single mkv, it will output a rar // e.g. even if we select just a single mkv, it will output a rar
var isChaotic bool var isChaotic bool
selectedFiles, isChaotic = t.organizeChaos(info.Links, selectedFiles) selectedFiles, isChaotic = t.organizeChaos(info.Links, selectedFiles)
if isChaotic { if isChaotic {
t.log.Warnf("Torrent id=%s %s is unrepairable, it is always returning a rar file (it will no longer show up in your directories)", info.ID, info.Name) t.log.Warnf("Torrent id=%s %s is unplayable; it is always returning a rar file (it will no longer show up in your directories)", info.ID, info.Name)
// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash) // t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
return nil return nil
} else { } else {
if streamableCount > 1 { if streamableCount > 1 && t.cfg.EnableRepair() {
// case for repair 1: it's missing some links (or all links) // case for repair 1: it's missing some links (or all links)
// if we download it as is, we might get the same file over and over again // if we download it as is, we might get the same file over and over again
// so we need to redownload it with other files selected // so we need to redownload it with other files selected
// that is why we check if there are other streamable files // that is why we check if there are other streamable files
t.log.Infof("Torrent id=%s %s marked for repair", info.ID, info.Name) t.log.Infof("Torrent id=%s %s marked for repair", info.ID, info.Name)
forRepair = true forRepair = true
} else { } else if streamableCount == 1 {
t.log.Warnf("Torrent id=%s %s is unrepairable, the lone streamable link has expired (it will no longer show up in your directories)", info.ID, info.Name) t.log.Warnf("Torrent id=%s %s is unplayable; the lone streamable link has expired (it will no longer show up in your directories)", info.ID, info.Name)
// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash) // t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
return nil return nil
} }
} }
} else if selectedFiles.Len() == len(info.Links) { } else if selectedFiles.Count() == len(info.Links) {
// all links are still intact! good! // all links are still intact! good!
// side note: iteration works! // side note: iteration works!
i := 0 i := 0
for el := selectedFiles.Front(); el != nil; el = el.Next() { selectedFiles.IterCb(func(_ string, file *File) {
if i < len(info.Links) { if i < len(info.Links) {
file := el.Value
file.Link = info.Links[i] // verified working! file.Link = info.Links[i] // verified working!
selectedFiles.Set(el.Key, file)
i++ i++
} }
} })
} }
info.ForRepair = forRepair info.ForRepair = forRepair
torrent := Torrent{ torrent := Torrent{
AccessKey: t.getName(info.Name, info.OriginalName), AccessKey: t.getName(info.Name, info.OriginalName),
SelectedFiles: selectedFiles, SelectedFiles: selectedFiles,
Directories: t.getDirectories(info),
LatestAdded: info.Added, LatestAdded: info.Added,
InProgress: info.Progress != 100, InProgress: info.Progress != 100,
Instances: []realdebrid.TorrentInfo{*info}, Instances: []realdebrid.TorrentInfo{*info},
} }
for _, directory := range torrent.Directories { if selectedFiles.Count() > 0 && torrentFromFile == nil {
if _, ok := t.DirectoryMap.Get(directory); !ok {
newMap := orderedmap.NewOrderedMap[string, *Torrent]()
t.mu.Lock()
t.DirectoryMap.Set(directory, newMap)
t.mu.Unlock()
} else {
torrents, _ := t.DirectoryMap.Get(directory)
t.mu.Lock()
torrents.Set(torrent.AccessKey, &torrent)
t.mu.Unlock()
}
}
if selectedFiles.Len() > 0 && torrentFromFile == nil {
t.writeToFile(info) // only when there are selected files, else it's useless t.writeToFile(info) // only when there are selected files, else it's useless
} }
return &torrent return &torrent
@@ -390,7 +442,7 @@ func hashStringToFh(s string) (fh uint64) {
func (t *TorrentManager) getName(name, originalName string) string { func (t *TorrentManager) getName(name, originalName string) string {
// drop the extension from the name // drop the extension from the name
if t.config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) { if t.cfg.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
return name return name
} else { } else {
ret := strings.TrimSuffix(originalName, ".mp4") ret := strings.TrimSuffix(originalName, ".mp4")
@@ -399,38 +451,6 @@ func (t *TorrentManager) getName(name, originalName string) string {
} }
} }
func (t *TorrentManager) getDirectories(torrent *realdebrid.TorrentInfo) []string {
var ret []string
// Map torrents to directories
switch t.config.GetVersion() {
case "v1":
configV1 := t.config.(*config.ZurgConfigV1)
groupMap := configV1.GetGroupMap()
// for every group, iterate over every torrent
// and then sprinkle/distribute the torrents to the directories of the group
for _, directories := range groupMap {
for _, directory := range directories {
var filenames []string
for _, file := range torrent.Files {
if file.Selected == 0 {
continue
}
filenames = append(filenames, file.Path)
}
accessKey := t.getName(torrent.Name, torrent.OriginalName)
if configV1.MeetsConditions(directory, torrent.ID, accessKey, filenames) {
ret = append(ret, directory)
break // we found a directory for this torrent for this group, so we can stop looking for more
}
}
}
default:
t.log.Error("Unknown config version")
}
// t.log.Debugf("Torrent %s is in directories %v", t.getName(torrent.Name, torrent.OriginalName), ret)
return ret
}
func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) error { func (t *TorrentManager) writeToFile(torrent *realdebrid.TorrentInfo) error {
filePath := "data/" + torrent.ID + ".bin" filePath := "data/" + torrent.ID + ".bin"
file, err := os.Create(filePath) file, err := os.Create(filePath)
@@ -473,7 +493,7 @@ func (t *TorrentManager) readFromFile(torrentID string) *realdebrid.TorrentInfo
return &torrent return &torrent
} }
func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap.OrderedMap[string, *File]) (*orderedmap.OrderedMap[string, *File], bool) { func (t *TorrentManager) organizeChaos(links []string, selectedFiles cmap.ConcurrentMap[string, *File]) (cmap.ConcurrentMap[string, *File], bool) {
type Result struct { type Result struct {
Response *realdebrid.UnrestrictResponse Response *realdebrid.UnrestrictResponse
} }
@@ -503,13 +523,12 @@ func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap
continue continue
} }
found := false found := false
// side note: iteration works! selectedFiles.IterCb(func(_ string, file *File) {
for el := selectedFiles.Front(); el != nil; el = el.Next() { if strings.Contains(file.Path, result.Response.Filename) {
if file, _ := selectedFiles.Get(el.Key); strings.Contains(file.Path, result.Response.Filename) {
file.Link = result.Response.Link file.Link = result.Response.Link
found = true found = true
} }
} })
if !found { if !found {
if result.Response.Streamable == 1 { if result.Response.Streamable == 1 {
selectedFiles.Set(filepath.Base(result.Response.Filename), &File{ selectedFiles.Set(filepath.Base(result.Response.Filename), &File{
@@ -532,219 +551,219 @@ func (t *TorrentManager) organizeChaos(links []string, selectedFiles *orderedmap
return selectedFiles, isChaotic return selectedFiles, isChaotic
} }
func (t *TorrentManager) repairAll() { // func (t *TorrentManager) repairAll() {
t.log.Info("Checking for torrents to repair") // t.log.Info("Checking for torrents to repair")
// side note: iteration works! // // side note: iteration works!
for el := t.TorrentMap.Front(); el != nil; el = el.Next() { // for el := t.TorrentMap.Front(); el != nil; el = el.Next() {
torrent := el.Value // torrent := el.Value
// do not repair if in progress // // do not repair if in progress
if torrent.InProgress { // if torrent.InProgress {
continue // continue
} // }
// do not repair if all files have links // // do not repair if all files have links
forRepair := false // forRepair := false
for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() { // for el2 := torrent.SelectedFiles.Front(); el2 != nil; el2 = el2.Next() {
file := el2.Value // file := el2.Value
if file.Link == "" { // if file.Link == "" {
forRepair = true // forRepair = true
break // break
} // }
} // }
if !forRepair { // if !forRepair {
// if it was marked for repair, unmark it // // if it was marked for repair, unmark it
torrent.ForRepair = false // torrent.ForRepair = false
t.mu.Lock() // t.mu.Lock()
t.TorrentMap.Set(torrent.AccessKey, torrent) // t.TorrentMap.Set(torrent.AccessKey, torrent)
t.mu.Unlock() // t.mu.Unlock()
continue // continue
} // }
// when getting info, we mark it for repair if it's missing some links // // when getting info, we mark it for repair if it's missing some links
if torrent.ForRepair { // if torrent.ForRepair {
t.log.Infof("Found torrent for repair: %s", torrent.AccessKey) // t.log.Infof("Found torrent for repair: %s", torrent.AccessKey)
t.Repair(torrent.AccessKey) // t.Repair(torrent.AccessKey)
break // only repair the first one for repair and then move on // break // only repair the first one for repair and then move on
} // }
} // }
} // }
func (t *TorrentManager) Repair(accessKey string) { // func (t *TorrentManager) Repair(accessKey string) {
if lastRepair, ok := t.repairMap.Get(accessKey); ok { // if lastRepair, ok := t.repairMap.Get(accessKey); ok {
if time.Since(lastRepair) < time.Duration(24*time.Hour) { // magic number: 24 hrs // if time.Since(lastRepair) < time.Duration(24*time.Hour) { // magic number: 24 hrs
return // return
} // }
} // }
t.mu.Lock() // t.mu.Lock()
t.repairMap.Set(accessKey, time.Now()) // t.repairMap.Set(accessKey, time.Now())
t.mu.Unlock() // t.mu.Unlock()
if !t.config.EnableRepair() { // if !t.config.EnableRepair() {
t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair") // t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair")
return // return
} // }
torrent, _ := t.TorrentMap.Get(accessKey) // torrent, _ := t.TorrentMap.Get(accessKey)
if torrent == nil { // if torrent == nil {
t.log.Warnf("Cannot find torrent %s anymore to repair it", accessKey) // t.log.Warnf("Cannot find torrent %s anymore to repair it", accessKey)
return // return
} // }
if torrent.InProgress { // if torrent.InProgress {
t.log.Infof("Torrent %s is in progress, cannot repair", torrent.AccessKey) // t.log.Infof("Torrent %s is in progress, cannot repair", torrent.AccessKey)
return // return
} // }
// check if we can still add more downloads // // check if we can still add more downloads
proceed := t.canCapacityHandle() // proceed := t.canCapacityHandle()
if !proceed { // if !proceed {
t.log.Error("Cannot add more torrents, ignoring repair request") // t.log.Error("Cannot add more torrents, ignoring repair request")
return // return
} // }
// make the file messy // // make the file messy
var links []string // var links []string
for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() { // for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
file := el.Value // file := el.Value
if file.Link != "" { // if file.Link != "" {
links = append(links, file.Link) // links = append(links, file.Link)
} // }
file.Link = "" // file.Link = ""
} // }
selectedFiles, _ := t.organizeChaos(links, torrent.SelectedFiles) // selectedFiles, _ := t.organizeChaos(links, torrent.SelectedFiles)
torrent.SelectedFiles = selectedFiles // torrent.SelectedFiles = selectedFiles
t.mu.Lock() // t.mu.Lock()
t.TorrentMap.Set(torrent.AccessKey, torrent) // t.TorrentMap.Set(torrent.AccessKey, torrent)
t.mu.Unlock() // t.mu.Unlock()
// first solution: add the same selection, maybe it can be fixed by reinsertion? // // first solution: add the same selection, maybe it can be fixed by reinsertion?
if t.reinsertTorrent(torrent, "") { // if t.reinsertTorrent(torrent, "") {
t.log.Infof("Successfully downloaded torrent %s to repair it", torrent.AccessKey) // t.log.Infof("Successfully downloaded torrent %s to repair it", torrent.AccessKey)
return // return
} // }
// if all the selected files are missing but there are other streamable files // // if all the selected files are missing but there are other streamable files
var missingFiles []File // var missingFiles []File
for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() { // for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
file := el.Value // file := el.Value
if file.Link == "" { // if file.Link == "" {
missingFiles = append(missingFiles, *file) // missingFiles = append(missingFiles, *file)
} // }
} // }
if len(missingFiles) > 0 { // if len(missingFiles) > 0 {
t.log.Infof("Redownloading %d missing files for torrent %s", len(missingFiles), torrent.AccessKey) // t.log.Infof("Redownloading %d missing files for torrent %s", len(missingFiles), torrent.AccessKey)
// if not, last resort: add only the missing files but do it in 2 batches // // if not, last resort: add only the missing files but do it in 2 batches
half := len(missingFiles) / 2 // half := len(missingFiles) / 2
missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",") // missingFiles1 := strings.Join(getFileIDs(missingFiles[:half]), ",")
missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",") // missingFiles2 := strings.Join(getFileIDs(missingFiles[half:]), ",")
if missingFiles1 != "" { // if missingFiles1 != "" {
t.reinsertTorrent(torrent, missingFiles1) // t.reinsertTorrent(torrent, missingFiles1)
} // }
if missingFiles2 != "" { // if missingFiles2 != "" {
t.reinsertTorrent(torrent, missingFiles2) // t.reinsertTorrent(torrent, missingFiles2)
} // }
} // }
} // }
func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string) bool { // func (t *TorrentManager) reinsertTorrent(torrent *Torrent, missingFiles string) bool {
// if missingFiles is not provided, look for missing files // // if missingFiles is not provided, look for missing files
if missingFiles == "" { // if missingFiles == "" {
var tmpSelection string // var tmpSelection string
for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() { // for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() {
file := el.Value // file := el.Value
tmpSelection += fmt.Sprintf("%d,", file.ID) // tmpSelection += fmt.Sprintf("%d,", file.ID)
} // }
if tmpSelection == "" { // if tmpSelection == "" {
return false // return false
} // }
if len(tmpSelection) > 0 { // if len(tmpSelection) > 0 {
missingFiles = tmpSelection[:len(tmpSelection)-1] // missingFiles = tmpSelection[:len(tmpSelection)-1]
} // }
} // }
// redownload torrent // // redownload torrent
resp, err := t.api.AddMagnetHash(torrent.Instances[0].Hash) // resp, err := t.api.AddMagnetHash(torrent.Instances[0].Hash)
if err != nil { // if err != nil {
t.log.Warnf("Cannot redownload torrent: %v", err) // t.log.Warnf("Cannot redownload torrent: %v", err)
return false // return false
} // }
time.Sleep(1 * time.Second) // time.Sleep(1 * time.Second)
// select files // // select files
newTorrentID := resp.ID // newTorrentID := resp.ID
err = t.api.SelectTorrentFiles(newTorrentID, missingFiles) // err = t.api.SelectTorrentFiles(newTorrentID, missingFiles)
if err != nil { // if err != nil {
t.log.Warnf("Cannot start redownloading: %v", err) // t.log.Warnf("Cannot start redownloading: %v", err)
t.api.DeleteTorrent(newTorrentID) // t.api.DeleteTorrent(newTorrentID)
return false // return false
} // }
time.Sleep(10 * time.Second) // time.Sleep(10 * time.Second)
// see if the torrent is ready // // see if the torrent is ready
info, err := t.api.GetTorrentInfo(newTorrentID) // info, err := t.api.GetTorrentInfo(newTorrentID)
if err != nil { // if err != nil {
t.log.Warnf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err) // t.log.Warnf("Cannot get info on redownloaded torrent id=%s : %v", newTorrentID, err)
t.api.DeleteTorrent(newTorrentID) // t.api.DeleteTorrent(newTorrentID)
return false // return false
} // }
if info.Status == "magnet_error" || info.Status == "error" || info.Status == "virus" || info.Status == "dead" { // if info.Status == "magnet_error" || info.Status == "error" || info.Status == "virus" || info.Status == "dead" {
t.log.Warnf("The redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status) // t.log.Warnf("The redownloaded torrent id=%s is in error state: %s", newTorrentID, info.Status)
t.api.DeleteTorrent(newTorrentID) // t.api.DeleteTorrent(newTorrentID)
return false // return false
} // }
if info.Progress != 100 { // if info.Progress != 100 {
t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion (this should fix the issue already)", info.ID) // t.log.Infof("Torrent id=%s is not cached anymore so we have to wait until completion (this should fix the issue already)", info.ID)
return true // return true
} // }
missingCount := len(strings.Split(missingFiles, ",")) // missingCount := len(strings.Split(missingFiles, ","))
if len(info.Links) != missingCount { // if len(info.Links) != missingCount {
t.log.Infof("It did not fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount) // t.log.Infof("It did not fix the issue for id=%s, only got %d files but we need %d, undoing", info.ID, len(info.Links), missingCount)
t.api.DeleteTorrent(newTorrentID) // t.api.DeleteTorrent(newTorrentID)
return false // return false
} // }
t.log.Infof("Repair successful id=%s", newTorrentID) // t.log.Infof("Repair successful id=%s", newTorrentID)
return true // return true
} // }
func (t *TorrentManager) canCapacityHandle() bool { // func (t *TorrentManager) canCapacityHandle() bool {
// max waiting time is 45 minutes // // max waiting time is 45 minutes
const maxRetries = 50 // const maxRetries = 50
const baseDelay = 1 * time.Second // const baseDelay = 1 * time.Second
const maxDelay = 60 * time.Second // const maxDelay = 60 * time.Second
retryCount := 0 // retryCount := 0
for { // for {
count, err := t.api.GetActiveTorrentCount() // count, err := t.api.GetActiveTorrentCount()
if err != nil { // if err != nil {
t.log.Warnf("Cannot get active downloads count: %v", err) // t.log.Warnf("Cannot get active downloads count: %v", err)
if retryCount >= maxRetries { // if retryCount >= maxRetries {
t.log.Error("Max retries reached. Exiting.") // t.log.Error("Max retries reached. Exiting.")
return false // return false
} // }
delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay // delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
if delay > maxDelay { // if delay > maxDelay {
delay = maxDelay // delay = maxDelay
} // }
time.Sleep(delay) // time.Sleep(delay)
retryCount++ // retryCount++
continue // continue
} // }
if count.DownloadingCount < count.MaxNumberOfTorrents { // if count.DownloadingCount < count.MaxNumberOfTorrents {
t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount) // t.log.Infof("We can still add a new torrent, we have capacity for %d more", count.MaxNumberOfTorrents-count.DownloadingCount)
return true // return true
} // }
if retryCount >= maxRetries { // if retryCount >= maxRetries {
t.log.Error("Max retries reached, exiting") // t.log.Error("Max retries reached, exiting")
return false // return false
} // }
delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay // delay := time.Duration(math.Pow(2, float64(retryCount))) * baseDelay
if delay > maxDelay { // if delay > maxDelay {
delay = maxDelay // delay = maxDelay
} // }
time.Sleep(delay) // time.Sleep(delay)
retryCount++ // retryCount++
} // }
} // }

5
internal/torrent/types.go
View File

@@ -2,13 +2,12 @@ package torrent
import ( import (
"github.com/debridmediamanager.com/zurg/pkg/realdebrid" "github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"github.com/elliotchance/orderedmap/v2" cmap "github.com/orcaman/concurrent-map/v2"
) )
type Torrent struct { type Torrent struct {
AccessKey string AccessKey string
SelectedFiles *orderedmap.OrderedMap[string, *File] SelectedFiles cmap.ConcurrentMap[string, *File]
Directories []string
LatestAdded string LatestAdded string
InProgress bool InProgress bool
ForRepair bool ForRepair bool

76
internal/universal/get.go
View File

@@ -46,22 +46,28 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *intTor.TorrentM
accessKey := segments[len(segments)-2] accessKey := segments[len(segments)-2]
filename := segments[len(segments)-1] filename := segments[len(segments)-1]
torrent, _ := t.TorrentMap.Get(accessKey) torrents, ok := t.DirectoryMap.Get(baseDirectory)
if torrent == nil { if !ok {
log.Warnf("Cannot find directory %s", baseDirectory)
http.Error(w, "File not found", http.StatusNotFound)
return
}
torrent, ok := torrents.Get(accessKey)
if !ok {
log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory) log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
http.Error(w, "File not found", http.StatusNotFound) http.Error(w, "File not found", http.StatusNotFound)
return return
} }
file, _ := torrent.SelectedFiles.Get(filename) file, ok := torrent.SelectedFiles.Get(filename)
if file == nil { if !ok {
log.Warnf("Cannot find file from path %s", requestPath) log.Warnf("Cannot find file from path %s", requestPath)
http.Error(w, "File not found", http.StatusNotFound) http.Error(w, "File not found", http.StatusNotFound)
return return
} }
if data, exists := cache.Get(requestPath); exists { if data, exists := cache.Get(requestPath); exists {
streamFileToResponse(torrent, data, w, r, t, c, log) streamFileToResponse(data, w, r, t, c, log)
return return
} }
@@ -75,7 +81,7 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *intTor.TorrentM
resp := t.UnrestrictUntilOk(link) resp := t.UnrestrictUntilOk(link)
if resp == nil { if resp == nil {
go t.Repair(torrent.AccessKey) // go t.Repair(torrent.AccessKey)
log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path) log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path)
streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log) streamErrorVideo("https://www.youtube.com/watch?v=gea_FJrtFVA", w, r, t, c, log)
return return
@@ -93,15 +99,15 @@ func HandleGetRequest(w http.ResponseWriter, r *http.Request, t *intTor.TorrentM
} }
} }
cache.Add(requestPath, resp.Download) cache.Add(requestPath, resp.Download)
streamFileToResponse(torrent, resp.Download, w, r, t, c, log) streamFileToResponse(resp.Download, w, r, t, c, log)
} }
func streamFileToResponse(torrent *intTor.Torrent, url string, w http.ResponseWriter, r *http.Request, t *intTor.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) { func streamFileToResponse(url string, w http.ResponseWriter, r *http.Request, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) {
// Create a new request for the file download. // Create a new request for the file download.
req, err := http.NewRequest(http.MethodGet, url, nil) req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil { if err != nil {
log.Errorf("Error creating new request: %v", err) log.Errorf("Error creating new request: %v", err)
streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, t, c, log) streamErrorVideo("https://www.youtube.com/watch?v=H3NSrObyAxM", w, r, torMgr, cfg, log)
return return
} }
@@ -111,25 +117,25 @@ func streamFileToResponse(torrent *intTor.Torrent, url string, w http.ResponseWr
} }
// Create a custom HTTP client // Create a custom HTTP client
client := zurghttp.NewHTTPClient(c.GetToken(), 10, c) client := zurghttp.NewHTTPClient(cfg.GetToken(), 10, cfg)
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
log.Warnf("Cannot download file %v ; torrent is marked for repair", err) log.Warnf("Cannot download file %v ; torrent is marked for repair", err)
if torrent != nil { // if torrent != nil {
go t.Repair(torrent.AccessKey) // go t.Repair(torrent.AccessKey)
} // }
streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, t, c, log) streamErrorVideo("https://www.youtube.com/watch?v=FSSd8cponAA", w, r, torMgr, cfg, log)
return return
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status) log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status)
if torrent != nil { // if torrent != nil {
go t.Repair(torrent.AccessKey) // go t.Repair(torrent.AccessKey)
} // }
streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, t, c, log) streamErrorVideo("https://www.youtube.com/watch?v=BcseUxviVqE", w, r, torMgr, cfg, log)
return return
} }
@@ -139,7 +145,7 @@ func streamFileToResponse(torrent *intTor.Torrent, url string, w http.ResponseWr
} }
} }
buf := make([]byte, c.GetNetworkBufferSize()) buf := make([]byte, cfg.GetNetworkBufferSize())
io.CopyBuffer(w, resp.Body, buf) io.CopyBuffer(w, resp.Body, buf)
} }
@@ -149,7 +155,7 @@ func streamErrorVideo(link string, w http.ResponseWriter, r *http.Request, t *in
http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError) http.Error(w, "REAL-DEBRID IS DOWN", http.StatusInternalServerError)
return return
} }
streamFileToResponse(nil, resp.Download, w, r, t, c, log) streamFileToResponse(resp.Download, w, r, t, c, log)
} }
func createErrorFile(path, link string) *intTor.File { func createErrorFile(path, link string) *intTor.File {
@@ -160,7 +166,7 @@ func createErrorFile(path, link string) *intTor.File {
return &ret return &ret
} }
func GetFileReader(torrent *intTor.Torrent, file *intTor.File, offset int64, size int, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) []byte { func GetFileReader(file *intTor.File, offset int64, size int, torMgr *intTor.TorrentManager, cfg config.ConfigInterface, log *zap.SugaredLogger) []byte {
unres := torMgr.UnrestrictUntilOk(file.Link) unres := torMgr.UnrestrictUntilOk(file.Link)
if unres == nil { if unres == nil {
if strings.Contains(file.Link, "www.youtube.com") { if strings.Contains(file.Link, "www.youtube.com") {
@@ -168,11 +174,11 @@ func GetFileReader(torrent *intTor.Torrent, file *intTor.File, offset int64, siz
return nil return nil
} }
log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path) log.Warnf("File %s is no longer available, torrent is marked for repair", file.Path)
if torrent != nil { // if torrent != nil {
go torMgr.Repair(torrent.AccessKey) // go torMgr.Repair(torrent.AccessKey)
} // }
errFile := createErrorFile("unavailable.mp4", "https://www.youtube.com/watch?v=gea_FJrtFVA") errFile := createErrorFile("unavailable.mp4", "https://www.youtube.com/watch?v=gea_FJrtFVA")
return GetFileReader(nil, errFile, 0, 0, torMgr, cfg, log) return GetFileReader(errFile, 0, 0, torMgr, cfg, log)
} }
req, err := http.NewRequest(http.MethodGet, unres.Download, nil) req, err := http.NewRequest(http.MethodGet, unres.Download, nil)
@@ -183,7 +189,7 @@ func GetFileReader(torrent *intTor.Torrent, file *intTor.File, offset int64, siz
} }
log.Errorf("Error creating new request: %v", err) log.Errorf("Error creating new request: %v", err)
errFile := createErrorFile("new_request.mp4", "https://www.youtube.com/watch?v=H3NSrObyAxM") errFile := createErrorFile("new_request.mp4", "https://www.youtube.com/watch?v=H3NSrObyAxM")
return GetFileReader(nil, errFile, 0, 0, torMgr, cfg, log) return GetFileReader(errFile, 0, 0, torMgr, cfg, log)
} }
if size == 0 { if size == 0 {
@@ -199,11 +205,11 @@ func GetFileReader(torrent *intTor.Torrent, file *intTor.File, offset int64, siz
return nil return nil
} }
log.Warnf("Cannot download file %v ; torrent is marked for repair", err) log.Warnf("Cannot download file %v ; torrent is marked for repair", err)
if torrent != nil { // if torrent != nil {
go torMgr.Repair(torrent.AccessKey) // go torMgr.Repair(torrent.AccessKey)
} // }
errFile := createErrorFile("cannot_download.mp4", "https://www.youtube.com/watch?v=FSSd8cponAA") errFile := createErrorFile("cannot_download.mp4", "https://www.youtube.com/watch?v=FSSd8cponAA")
return GetFileReader(nil, errFile, 0, 0, torMgr, cfg, log) return GetFileReader(errFile, 0, 0, torMgr, cfg, log)
} }
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
@@ -212,11 +218,11 @@ func GetFileReader(torrent *intTor.Torrent, file *intTor.File, offset int64, siz
return nil return nil
} }
log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status) log.Warnf("Received a %s status code ; torrent is marked for repair", resp.Status)
if torrent != nil { // if torrent != nil {
go torMgr.Repair(torrent.AccessKey) // go torMgr.Repair(torrent.AccessKey)
} // }
errFile := createErrorFile("not_ok_status.mp4", "https://www.youtube.com/watch?v=BcseUxviVqE") errFile := createErrorFile("not_ok_status.mp4", "https://www.youtube.com/watch?v=BcseUxviVqE")
return GetFileReader(nil, errFile, 0, 0, torMgr, cfg, log) return GetFileReader(errFile, 0, 0, torMgr, cfg, log)
} }
defer resp.Body.Close() defer resp.Body.Close()
requestedBytes, err := io.ReadAll(resp.Body) requestedBytes, err := io.ReadAll(resp.Body)
@@ -224,7 +230,7 @@ func GetFileReader(torrent *intTor.Torrent, file *intTor.File, offset int64, siz
if err != io.EOF { if err != io.EOF {
log.Errorf("Error reading bytes: %v", err) log.Errorf("Error reading bytes: %v", err)
errFile := createErrorFile("read_error.mp4", "https://www.youtube.com/watch?v=t9VgOriBHwE") errFile := createErrorFile("read_error.mp4", "https://www.youtube.com/watch?v=t9VgOriBHwE")
return GetFileReader(nil, errFile, 0, 0, torMgr, cfg, log) return GetFileReader(errFile, 0, 0, torMgr, cfg, log)
} }
} }
return requestedBytes return requestedBytes

10
internal/universal/head.go
View File

@@ -43,8 +43,14 @@ func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.Torren
accessKey := segments[len(segments)-2] accessKey := segments[len(segments)-2]
filename := segments[len(segments)-1] filename := segments[len(segments)-1]
torrent, _ := t.TorrentMap.Get(accessKey) torrents, ok := t.DirectoryMap.Get(baseDirectory)
if torrent == nil { if !ok {
log.Warnf("Cannot find directory %s", baseDirectory)
http.Error(w, "File not found", http.StatusNotFound)
return
}
torrent, ok := torrents.Get(accessKey)
if !ok {
log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory) log.Warnf("Cannot find torrent %s in the directory %s", accessKey, baseDirectory)
http.Error(w, "File not found", http.StatusNotFound) http.Error(w, "File not found", http.StatusNotFound)
return return

27
internal/zfs/zfs.go
View File

@@ -1,11 +1,13 @@
package zfs package zfs
import ( import (
"fmt"
"strings" "strings"
"github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/pkg/chunk" "github.com/debridmediamanager.com/zurg/pkg/chunk"
cmap "github.com/orcaman/concurrent-map/v2"
"github.com/winfsp/cgofuse/fuse" "github.com/winfsp/cgofuse/fuse"
"go.uber.org/zap" "go.uber.org/zap"
) )
@@ -98,6 +100,7 @@ func (fs *ZurgFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int)
func (fs *ZurgFS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) { func (fs *ZurgFS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) {
segments := splitIntoSegments(path) segments := splitIntoSegments(path)
fmt.Println("seg", segments)
if len(segments) != 3 { if len(segments) != 3 {
return -fuse.ENOENT return -fuse.ENOENT
} else if directory, dirFound := fs.TorrentManager.DirectoryMap.Get(segments[0]); !dirFound { } else if directory, dirFound := fs.TorrentManager.DirectoryMap.Get(segments[0]); !dirFound {
@@ -138,30 +141,30 @@ func (fs *ZurgFS) Readdir(path string,
case 0: case 0:
fill(".", nil, 0) fill(".", nil, 0)
fill("..", nil, 0) fill("..", nil, 0)
for el := fs.TorrentManager.DirectoryMap.Front(); el != nil; el = el.Next() { fs.TorrentManager.DirectoryMap.IterCb(func(directoryName string, _ cmap.ConcurrentMap[string, *torrent.Torrent]) {
fill(el.Key, nil, 0) fill(directoryName, nil, 0)
} })
case 1: case 1:
fill(".", nil, 0) fill(".", nil, 0)
fill("..", nil, 0) fill("..", nil, 0)
if directory, dirFound := fs.TorrentManager.DirectoryMap.Get(segments[0]); !dirFound { if torrents, dirFound := fs.TorrentManager.DirectoryMap.Get(segments[0]); !dirFound {
return -fuse.ENOENT return -fuse.ENOENT
} else { } else {
for el := directory.Front(); el != nil; el = el.Next() { torrents.IterCb(func(accessKey string, _ *torrent.Torrent) {
fill(el.Key, nil, 0) fill(accessKey, nil, 0)
} })
} }
case 2: case 2:
fill(".", nil, 0) fill(".", nil, 0)
fill("..", nil, 0) fill("..", nil, 0)
if directory, dirFound := fs.TorrentManager.DirectoryMap.Get(segments[0]); !dirFound { if torrents, dirFound := fs.TorrentManager.DirectoryMap.Get(segments[0]); !dirFound {
return -fuse.ENOENT return -fuse.ENOENT
} else if torrent, torFound := directory.Get(segments[1]); !torFound { } else if tor, torFound := torrents.Get(segments[1]); !torFound {
return -fuse.ENOENT return -fuse.ENOENT
} else { } else {
for el := torrent.SelectedFiles.Front(); el != nil; el = el.Next() { tor.SelectedFiles.IterCb(func(filename string, _ *torrent.File) {
fill(el.Key, nil, 0) fill(filename, nil, 0)
} })
} }
default: default:
return -fuse.ENOENT return -fuse.ENOENT

1
pkg/chunk/download.go
View File

@@ -112,7 +112,6 @@ func (d *Downloader) downloadFromAPI(request *Request, buffer []byte, delay int6
downloadURL := resp.Download downloadURL := resp.Download
req, err := http.NewRequest("GET", downloadURL, nil) req, err := http.NewRequest("GET", downloadURL, nil)
if nil != err { if nil != err {
d.log.Debugf("request init error: %v", err)
return fmt.Errorf("could not create request object %s %s from API", request.file.Path, request.file.Link) return fmt.Errorf("could not create request object %s %s from API", request.file.Path, request.file.Link)
} }
req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", request.offsetStart, request.offsetEnd-1)) req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", request.offsetStart, request.offsetEnd-1))

5
pkg/chunk/storage.go
View File

@@ -381,23 +381,18 @@ func (s *Storage) Store(id RequestID, bytes []byte) (err error) {
if nil != chunk { if nil != chunk {
if chunk.valid(id) { if chunk.valid(id) {
s.log.Debugf("Create chunk %v (exists: valid)", id)
return nil return nil
} }
s.log.Warnf("Create chunk %v(exists: overwrite)", id) s.log.Warnf("Create chunk %v(exists: overwrite)", id)
} else { } else {
index := s.stack.Pop() index := s.stack.Pop()
if index == -1 { if index == -1 {
s.log.Debugf("Create chunk %v (failed)", id)
return fmt.Errorf("no buffers available") return fmt.Errorf("no buffers available")
} }
chunk = s.buffers[index] chunk = s.buffers[index]
deleteID := chunk.id deleteID := chunk.id
if blankRequestID != deleteID { if blankRequestID != deleteID {
delete(s.chunks, deleteID) delete(s.chunks, deleteID)
s.log.Debugf("Create chunk %v (reused)", id)
} else {
s.log.Debugf("Create chunk %v (stored)", id)
} }
s.chunks[id] = index s.chunks[id] = index
chunk.item = s.stack.Push(index) chunk.item = s.stack.Push(index)

17
pkg/realdebrid/types.go
View File

@@ -29,6 +29,23 @@ type Torrent struct {
Links []string `json:"links"` Links []string `json:"links"`
} }
func (i *Torrent) UnmarshalJSON(data []byte) error {
type Alias Torrent
aux := &struct {
Progress float64 `json:"progress"`
*Alias
}{
Alias: (*Alias)(i),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
i.Progress = int(math.Round(aux.Progress))
return nil
}
type TorrentInfo struct { type TorrentInfo struct {
ID string `json:"id"` ID string `json:"id"`
Name string `json:"filename"` Name string `json:"filename"`