Remove ristretto cache
This commit is contained in:
22
go.mod
22
go.mod
@@ -9,25 +9,21 @@ require (
|
|||||||
|
|
||||||
require github.com/orcaman/concurrent-map/v2 v2.0.1
|
require github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||||
|
|
||||||
require github.com/panjf2000/ants/v2 v2.8.2
|
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/json-iterator/go v1.1.12 // indirect
|
github.com/json-iterator/go v1.1.12
|
||||||
github.com/julienschmidt/httprouter v1.3.0 // indirect
|
github.com/julienschmidt/httprouter v1.3.0
|
||||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
github.com/panjf2000/ants/v2 v2.8.2
|
||||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
github.com/scylladb/go-set v1.0.2
|
||||||
github.com/scylladb/go-set v1.0.2 // indirect
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||||
|
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/cespare/xxhash/v2 v2.1.1 // indirect
|
|
||||||
github.com/dgraph-io/ristretto v0.1.1
|
|
||||||
github.com/dustin/go-humanize v1.0.0 // indirect
|
|
||||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
|
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
|
||||||
github.com/spf13/cobra v1.8.0
|
github.com/spf13/cobra v1.8.0
|
||||||
github.com/spf13/pflag v1.0.5 // indirect
|
github.com/spf13/pflag v1.0.5 // indirect
|
||||||
go.uber.org/multierr v1.10.0 // indirect
|
go.uber.org/multierr v1.10.0 // indirect
|
||||||
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
17
go.sum
17
go.sum
@@ -1,18 +1,9 @@
|
|||||||
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
|
|
||||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
|
github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
|
||||||
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
|
|
||||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
|
||||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
|
||||||
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
|
|
||||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
|
||||||
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
|
github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
|
||||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
|
||||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
|
||||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
@@ -28,8 +19,6 @@ github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/
|
|||||||
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
|
||||||
github.com/panjf2000/ants/v2 v2.8.2 h1:D1wfANttg8uXhC9149gRt1PDQ+dLVFjNXkCEycMcvQQ=
|
github.com/panjf2000/ants/v2 v2.8.2 h1:D1wfANttg8uXhC9149gRt1PDQ+dLVFjNXkCEycMcvQQ=
|
||||||
github.com/panjf2000/ants/v2 v2.8.2/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
|
github.com/panjf2000/ants/v2 v2.8.2/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
@@ -43,7 +32,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
|
||||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||||
@@ -56,11 +44,8 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
|||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||||
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
|
|
||||||
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import (
|
|||||||
"github.com/debridmediamanager/zurg/pkg/logutil"
|
"github.com/debridmediamanager/zurg/pkg/logutil"
|
||||||
"github.com/debridmediamanager/zurg/pkg/realdebrid"
|
"github.com/debridmediamanager/zurg/pkg/realdebrid"
|
||||||
"github.com/debridmediamanager/zurg/pkg/utils"
|
"github.com/debridmediamanager/zurg/pkg/utils"
|
||||||
"github.com/dgraph-io/ristretto"
|
|
||||||
"github.com/julienschmidt/httprouter"
|
"github.com/julienschmidt/httprouter"
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
)
|
)
|
||||||
@@ -45,17 +44,7 @@ func MainApp(configPath string) {
|
|||||||
|
|
||||||
utils.EnsureDirExists("data")
|
utils.EnsureDirExists("data")
|
||||||
|
|
||||||
cache, err := ristretto.NewCache(&ristretto.Config{
|
torrentMgr := torrent.NewTorrentManager(config, rd, p, log.Named("manager"))
|
||||||
NumCounters: 1e5, // 200,000 to track frequency for 100,000 items.
|
|
||||||
MaxCost: 100 << 20, // maximum cost of cache (100MB).
|
|
||||||
BufferItems: 1 << 10, // number of keys per Get buffer, can be adjusted.
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
zurglog.Errorf("Failed to create cache: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
|
|
||||||
torrentMgr := torrent.NewTorrentManager(config, rd, p, cache, log.Named("manager"))
|
|
||||||
|
|
||||||
downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient"))
|
downloadClient := http.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient"))
|
||||||
getfile := universal.NewGetFile(downloadClient)
|
getfile := universal.NewGetFile(downloadClient)
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ func HandleDeleteTorrent(directory, torrentName string, t *torrent.TorrentManage
|
|||||||
if !torrents.Has(torrentName) {
|
if !torrents.Has(torrentName) {
|
||||||
return fmt.Errorf("cannot find torrent %s", torrentName)
|
return fmt.Errorf("cannot find torrent %s", torrentName)
|
||||||
}
|
}
|
||||||
t.Delete(torrentName, true, true)
|
t.Delete(torrentName, true)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,10 +33,9 @@ func HandleDeleteFile(directory, torrentName, fileName string, t *torrent.Torren
|
|||||||
}
|
}
|
||||||
file.Link = "unselect"
|
file.Link = "unselect"
|
||||||
if t.CheckDeletedState(torrent) {
|
if t.CheckDeletedState(torrent) {
|
||||||
t.Delete(torrentName, true, true)
|
t.Delete(torrentName, true)
|
||||||
} else {
|
|
||||||
updatedPaths := t.UpdateTorrentResponseCache(torrent)
|
|
||||||
t.TriggerHookOnLibraryUpdate(updatedPaths)
|
|
||||||
}
|
}
|
||||||
|
// todo: triggeer an update ???
|
||||||
|
// t.TriggerHookOnLibraryUpdate(updatedPaths)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,9 +13,7 @@ import (
|
|||||||
|
|
||||||
func HandleListDirectories(t *torrent.TorrentManager) (*string, error) {
|
func HandleListDirectories(t *torrent.TorrentManager) (*string, error) {
|
||||||
davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">"
|
davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">"
|
||||||
// initial response is the directory itself
|
|
||||||
davDoc += dav.BaseDirectory("", "")
|
davDoc += dav.BaseDirectory("", "")
|
||||||
|
|
||||||
directories := t.DirectoryMap.Keys()
|
directories := t.DirectoryMap.Keys()
|
||||||
sort.Strings(directories)
|
sort.Strings(directories)
|
||||||
for _, directory := range directories {
|
for _, directory := range directories {
|
||||||
@@ -24,7 +22,6 @@ func HandleListDirectories(t *torrent.TorrentManager) (*string, error) {
|
|||||||
}
|
}
|
||||||
davDoc += dav.Directory(directory, "")
|
davDoc += dav.Directory(directory, "")
|
||||||
}
|
}
|
||||||
|
|
||||||
davDoc += "</d:multistatus>"
|
davDoc += "</d:multistatus>"
|
||||||
return &davDoc, nil
|
return &davDoc, nil
|
||||||
}
|
}
|
||||||
@@ -35,8 +32,6 @@ func HandleListTorrents(directory string, t *torrent.TorrentManager, log *zap.Su
|
|||||||
return nil, fmt.Errorf("cannot find directory %s", directory)
|
return nil, fmt.Errorf("cannot find directory %s", directory)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp, ok := t.ResponseCache.Get(directory + ".dav"); !ok {
|
|
||||||
log.Debugf("Generating xml for directory %s", directory)
|
|
||||||
davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">"
|
davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">"
|
||||||
davDoc += dav.Directory("", "")
|
davDoc += dav.Directory("", "")
|
||||||
directories := t.DirectoryMap.Keys()
|
directories := t.DirectoryMap.Keys()
|
||||||
@@ -46,10 +41,6 @@ func HandleListTorrents(directory string, t *torrent.TorrentManager, log *zap.Su
|
|||||||
}
|
}
|
||||||
davDoc += "</d:multistatus>"
|
davDoc += "</d:multistatus>"
|
||||||
return &davDoc, nil
|
return &davDoc, nil
|
||||||
} else {
|
|
||||||
davDoc := resp.(string)
|
|
||||||
return &davDoc, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func HandleListFiles(directory, torrentName string, t *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
|
func HandleListFiles(directory, torrentName string, t *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
|
||||||
@@ -62,22 +53,16 @@ func HandleListFiles(directory, torrentName string, t *torrent.TorrentManager, l
|
|||||||
return nil, fmt.Errorf("cannot find torrent %s", torrentName)
|
return nil, fmt.Errorf("cannot find torrent %s", torrentName)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp, ok := t.ResponseCache.Get(directory + "/" + torrentName + ".dav"); !ok {
|
|
||||||
log.Debugf("Generating xml for torrent %s", torrentName)
|
|
||||||
davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">" + dav.BaseDirectory(filepath.Join(directory, tor.AccessKey), tor.LatestAdded)
|
davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">" + dav.BaseDirectory(filepath.Join(directory, tor.AccessKey), tor.LatestAdded)
|
||||||
filenames := tor.SelectedFiles.Keys()
|
filenames := tor.SelectedFiles.Keys()
|
||||||
sort.Strings(filenames)
|
sort.Strings(filenames)
|
||||||
for _, filename := range filenames {
|
for _, filename := range filenames {
|
||||||
file, _ := tor.SelectedFiles.Get(filename)
|
file, ok := tor.SelectedFiles.Get(filename)
|
||||||
if file == nil || !strings.HasPrefix(file.Link, "http") {
|
if !ok || !strings.HasPrefix(file.Link, "http") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
davDoc += dav.File(filename, file.Bytes, file.Ended)
|
davDoc += dav.File(filename, file.Bytes, file.Ended)
|
||||||
}
|
}
|
||||||
davDoc += "</d:multistatus>"
|
davDoc += "</d:multistatus>"
|
||||||
return &davDoc, nil
|
return &davDoc, nil
|
||||||
} else {
|
|
||||||
davDoc := resp.(string)
|
|
||||||
return &davDoc, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,8 +32,6 @@ func HandleListTorrents(directory string, t *torrent.TorrentManager, log *zap.Su
|
|||||||
return nil, fmt.Errorf("cannot find directory %s", directory)
|
return nil, fmt.Errorf("cannot find directory %s", directory)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp, ok := t.ResponseCache.Get(directory + ".html"); !ok {
|
|
||||||
log.Debugf("Generating html for directory %s", directory)
|
|
||||||
htmlDoc := "<ol>"
|
htmlDoc := "<ol>"
|
||||||
var allTorrents []*torrent.Torrent
|
var allTorrents []*torrent.Torrent
|
||||||
torrents.IterCb(func(_ string, tor *torrent.Torrent) {
|
torrents.IterCb(func(_ string, tor *torrent.Torrent) {
|
||||||
@@ -49,10 +47,6 @@ func HandleListTorrents(directory string, t *torrent.TorrentManager, log *zap.Su
|
|||||||
htmlDoc = htmlDoc + fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", filepath.Join(directory, url.PathEscape(tor.AccessKey)), tor.AccessKey)
|
htmlDoc = htmlDoc + fmt.Sprintf("<li><a href=\"/http/%s/\">%s</a></li>", filepath.Join(directory, url.PathEscape(tor.AccessKey)), tor.AccessKey)
|
||||||
}
|
}
|
||||||
return &htmlDoc, nil
|
return &htmlDoc, nil
|
||||||
} else {
|
|
||||||
htmlDoc := resp.(string)
|
|
||||||
return &htmlDoc, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func HandleListFiles(directory, torrentName string, t *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
|
func HandleListFiles(directory, torrentName string, t *torrent.TorrentManager, log *zap.SugaredLogger) (*string, error) {
|
||||||
@@ -65,22 +59,16 @@ func HandleListFiles(directory, torrentName string, t *torrent.TorrentManager, l
|
|||||||
return nil, fmt.Errorf("cannot find torrent %s", torrentName)
|
return nil, fmt.Errorf("cannot find torrent %s", torrentName)
|
||||||
}
|
}
|
||||||
|
|
||||||
if resp, ok := t.ResponseCache.Get(directory + "/" + torrentName + ".html"); !ok {
|
|
||||||
log.Debugf("Generating html for torrent %s", torrentName)
|
|
||||||
htmlDoc := "<ol>"
|
htmlDoc := "<ol>"
|
||||||
filenames := tor.SelectedFiles.Keys()
|
filenames := tor.SelectedFiles.Keys()
|
||||||
sort.Strings(filenames)
|
sort.Strings(filenames)
|
||||||
for _, filename := range filenames {
|
for _, filename := range filenames {
|
||||||
file, _ := tor.SelectedFiles.Get(filename)
|
file, ok := tor.SelectedFiles.Get(filename)
|
||||||
if file == nil || !strings.HasPrefix(file.Link, "http") {
|
if !ok || !strings.HasPrefix(file.Link, "http") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
filePath := filepath.Join(directory, torrentName, url.PathEscape(filename))
|
filePath := filepath.Join(directory, torrentName, url.PathEscape(filename))
|
||||||
htmlDoc += fmt.Sprintf("<li><a href=\"/http/%s\">%s</a></li>", filePath, filename)
|
htmlDoc += fmt.Sprintf("<li><a href=\"/http/%s\">%s</a></li>", filePath, filename)
|
||||||
}
|
}
|
||||||
return &htmlDoc, nil
|
return &htmlDoc, nil
|
||||||
} else {
|
|
||||||
htmlDoc := resp.(string)
|
|
||||||
return &htmlDoc, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,18 +4,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
"net/url"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/debridmediamanager/zurg/internal/config"
|
"github.com/debridmediamanager/zurg/internal/config"
|
||||||
"github.com/debridmediamanager/zurg/pkg/dav"
|
|
||||||
"github.com/debridmediamanager/zurg/pkg/realdebrid"
|
"github.com/debridmediamanager/zurg/pkg/realdebrid"
|
||||||
"github.com/dgraph-io/ristretto"
|
|
||||||
cmap "github.com/orcaman/concurrent-map/v2"
|
cmap "github.com/orcaman/concurrent-map/v2"
|
||||||
"github.com/panjf2000/ants/v2"
|
"github.com/panjf2000/ants/v2"
|
||||||
"github.com/scylladb/go-set"
|
"github.com/scylladb/go-set"
|
||||||
@@ -33,7 +29,6 @@ type TorrentManager struct {
|
|||||||
Api *realdebrid.RealDebrid
|
Api *realdebrid.RealDebrid
|
||||||
DirectoryMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
|
DirectoryMap cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
|
||||||
DownloadCache cmap.ConcurrentMap[string, *realdebrid.Download]
|
DownloadCache cmap.ConcurrentMap[string, *realdebrid.Download]
|
||||||
ResponseCache *ristretto.Cache
|
|
||||||
accessKeySet *strset.Set
|
accessKeySet *strset.Set
|
||||||
latestState *LibraryState
|
latestState *LibraryState
|
||||||
requiredVersion string
|
requiredVersion string
|
||||||
@@ -45,13 +40,12 @@ type TorrentManager struct {
|
|||||||
// NewTorrentManager creates a new torrent manager
|
// NewTorrentManager creates a new torrent manager
|
||||||
// it will fetch all torrents and their info in the background
|
// it will fetch all torrents and their info in the background
|
||||||
// and store them in-memory and cached in files
|
// and store them in-memory and cached in files
|
||||||
func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, cache *ristretto.Cache, log *zap.SugaredLogger) *TorrentManager {
|
func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *zap.SugaredLogger) *TorrentManager {
|
||||||
initialSate := EmptyState()
|
initialSate := EmptyState()
|
||||||
|
|
||||||
t := &TorrentManager{
|
t := &TorrentManager{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Api: api,
|
Api: api,
|
||||||
ResponseCache: cache,
|
|
||||||
accessKeySet: set.NewStringSet(),
|
accessKeySet: set.NewStringSet(),
|
||||||
latestState: &initialSate,
|
latestState: &initialSate,
|
||||||
requiredVersion: "03.12.2023",
|
requiredVersion: "03.12.2023",
|
||||||
@@ -116,11 +110,11 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p
|
|||||||
return t
|
return t
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TorrentManager) RefreshTorrents() []string {
|
func (t *TorrentManager) RefreshTorrents() {
|
||||||
instances, _, err := t.Api.GetTorrents(0)
|
instances, _, err := t.Api.GetTorrents(0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.log.Warnf("Cannot get torrents: %v\n", err)
|
t.log.Warnf("Cannot get torrents: %v\n", err)
|
||||||
return nil
|
return
|
||||||
}
|
}
|
||||||
infoChan := make(chan *Torrent, len(instances))
|
infoChan := make(chan *Torrent, len(instances))
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
@@ -154,32 +148,18 @@ func (t *TorrentManager) RefreshTorrents() []string {
|
|||||||
}
|
}
|
||||||
t.log.Infof("Compiled into %d torrents, %d were missing info", oldTorrents.Count(), noInfoCount)
|
t.log.Infof("Compiled into %d torrents, %d were missing info", oldTorrents.Count(), noInfoCount)
|
||||||
|
|
||||||
var updatedPaths []string
|
|
||||||
somethingChanged := false
|
|
||||||
// removed
|
// removed
|
||||||
strset.Difference(t.accessKeySet, freshKeys).Each(func(accessKey string) bool {
|
strset.Difference(t.accessKeySet, freshKeys).Each(func(accessKey string) bool {
|
||||||
somethingChanged = true
|
t.Delete(accessKey, false)
|
||||||
torrentPaths := t.Delete(accessKey, false, false)
|
|
||||||
updatedPaths = append(updatedPaths, torrentPaths...)
|
|
||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
// new
|
// new
|
||||||
strset.Difference(freshKeys, t.accessKeySet).Each(func(accessKey string) bool {
|
strset.Difference(freshKeys, t.accessKeySet).Each(func(accessKey string) bool {
|
||||||
somethingChanged = true
|
|
||||||
torrent, _ := oldTorrents.Get(accessKey)
|
|
||||||
torrentPaths := t.UpdateTorrentResponseCache(torrent)
|
|
||||||
updatedPaths = append(updatedPaths, torrentPaths...)
|
|
||||||
t.accessKeySet.Add(accessKey)
|
t.accessKeySet.Add(accessKey)
|
||||||
return true
|
return true
|
||||||
})
|
})
|
||||||
// now we can build the directory responses
|
|
||||||
if somethingChanged {
|
|
||||||
t.UpdateDirectoryResponsesCache()
|
|
||||||
}
|
|
||||||
|
|
||||||
t.SetNewLatestState(t.getCurrentState())
|
t.SetNewLatestState(t.getCurrentState())
|
||||||
|
|
||||||
return updatedPaths
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// getMoreInfo gets original name, size and files for a torrent
|
// getMoreInfo gets original name, size and files for a torrent
|
||||||
@@ -400,13 +380,9 @@ func (t *TorrentManager) startRefreshJob() {
|
|||||||
}
|
}
|
||||||
t.log.Infof("Detected changes! Refreshing %d torrents", checksum.TotalCount)
|
t.log.Infof("Detected changes! Refreshing %d torrents", checksum.TotalCount)
|
||||||
|
|
||||||
updatedPaths := t.RefreshTorrents()
|
t.RefreshTorrents()
|
||||||
t.log.Info("Finished refreshing torrents")
|
t.log.Info("Finished refreshing torrents")
|
||||||
|
|
||||||
if updatedPaths != nil {
|
|
||||||
t.TriggerHookOnLibraryUpdate(updatedPaths)
|
|
||||||
}
|
|
||||||
|
|
||||||
if t.Config.EnableRepair() {
|
if t.Config.EnableRepair() {
|
||||||
t.RepairAll()
|
t.RepairAll()
|
||||||
} else {
|
} else {
|
||||||
@@ -539,7 +515,7 @@ func (t *TorrentManager) CheckDeletedState(torrent *Torrent) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TorrentManager) Delete(accessKey string, deleteInRD bool, updateDirectoryResponses bool) []string {
|
func (t *TorrentManager) Delete(accessKey string, deleteInRD bool) {
|
||||||
if deleteInRD {
|
if deleteInRD {
|
||||||
allTorrents, _ := t.DirectoryMap.Get(INT_ALL)
|
allTorrents, _ := t.DirectoryMap.Get(INT_ALL)
|
||||||
infoCache, _ := t.DirectoryMap.Get(INT_INFO_CACHE)
|
infoCache, _ := t.DirectoryMap.Get(INT_INFO_CACHE)
|
||||||
@@ -554,19 +530,9 @@ func (t *TorrentManager) Delete(accessKey string, deleteInRD bool, updateDirecto
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
t.log.Infof("Removing torrent %s from zurg database", accessKey)
|
t.log.Infof("Removing torrent %s from zurg database", accessKey)
|
||||||
var updatedPaths []string
|
|
||||||
t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
|
t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
|
||||||
if ok := torrents.Has(accessKey); ok {
|
|
||||||
torrents.Remove(accessKey)
|
torrents.Remove(accessKey)
|
||||||
pathKey := fmt.Sprintf("%s/%s", directory, accessKey)
|
|
||||||
updatedPaths = append(updatedPaths, pathKey)
|
|
||||||
t.ResponseCache.Del(pathKey)
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
if updateDirectoryResponses {
|
|
||||||
t.UpdateDirectoryResponsesCache()
|
|
||||||
}
|
|
||||||
return updatedPaths
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TorrentManager) repair(torrent *Torrent) {
|
func (t *TorrentManager) repair(torrent *Torrent) {
|
||||||
@@ -744,77 +710,6 @@ func (t *TorrentManager) Repair(torrent *Torrent) {
|
|||||||
t.repair(torrent)
|
t.repair(torrent)
|
||||||
t.log.Info("Finished repairing torrent %s", torrent.AccessKey)
|
t.log.Info("Finished repairing torrent %s", torrent.AccessKey)
|
||||||
})
|
})
|
||||||
updatedPaths := t.UpdateTorrentResponseCache(torrent)
|
|
||||||
t.TriggerHookOnLibraryUpdate(updatedPaths)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TorrentManager) UpdateTorrentResponseCache(torrent *Torrent) []string {
|
|
||||||
updatedPaths := []string{}
|
|
||||||
dav, html := t.buildTorrentResponses(torrent)
|
|
||||||
t.AssignedDirectoryCb(torrent, func(directory string) {
|
|
||||||
if strings.HasPrefix(directory, "int__") {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
torrents, _ := t.DirectoryMap.Get(directory)
|
|
||||||
torrents.Set(torrent.AccessKey, torrent)
|
|
||||||
|
|
||||||
// torrent responses
|
|
||||||
pathKey := fmt.Sprintf("%s/%s", directory, torrent.AccessKey)
|
|
||||||
updatedPaths = append(updatedPaths, pathKey)
|
|
||||||
newHtml := strings.ReplaceAll(html, "$dir", directory)
|
|
||||||
t.ResponseCache.Set(pathKey+".html", newHtml, 1)
|
|
||||||
newDav := strings.ReplaceAll(dav, "$dir", directory)
|
|
||||||
t.ResponseCache.Set(pathKey+".dav", newDav, 1)
|
|
||||||
})
|
|
||||||
return updatedPaths
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TorrentManager) UpdateDirectoryResponsesCache() {
|
|
||||||
t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
|
|
||||||
allKeys := torrents.Keys()
|
|
||||||
sort.Strings(allKeys)
|
|
||||||
davRet := ""
|
|
||||||
htmlRet := ""
|
|
||||||
for _, accessKey := range allKeys {
|
|
||||||
if tor, ok := torrents.Get(accessKey); ok {
|
|
||||||
if tor.AllInProgress() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
davRet += dav.Directory(tor.AccessKey, tor.LatestAdded)
|
|
||||||
htmlRet += fmt.Sprintf("<li><a href=\"/http/%s/%s/\">%s</a></li>", directory, tor.AccessKey, tor.AccessKey)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
cacheKey := directory
|
|
||||||
davRet = "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">" + dav.BaseDirectory(directory, "") + dav.BaseDirectory(directory, "") + davRet + "</d:multistatus>"
|
|
||||||
t.ResponseCache.Set(cacheKey+".dav", davRet, 1)
|
|
||||||
htmlRet = "<ol>" + htmlRet
|
|
||||||
t.ResponseCache.Set(cacheKey+".html", htmlRet, 1)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *TorrentManager) buildTorrentResponses(tor *Torrent) (string, string) {
|
|
||||||
davRet := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">" + dav.BaseDirectory(filepath.Join("$dir", tor.AccessKey), tor.LatestAdded)
|
|
||||||
htmlRet := "<ol>"
|
|
||||||
filenames := tor.SelectedFiles.Keys()
|
|
||||||
sort.Strings(filenames)
|
|
||||||
for _, filename := range filenames {
|
|
||||||
file, _ := tor.SelectedFiles.Get(filename)
|
|
||||||
if file == nil || !strings.HasPrefix(file.Link, "http") {
|
|
||||||
// will be caught by torrent manager's repairAll
|
|
||||||
// just skip it for now
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
davRet += dav.File(filename, file.Bytes, file.Ended)
|
|
||||||
|
|
||||||
filePath := filepath.Join("$dir", tor.AccessKey, url.PathEscape(filename))
|
|
||||||
htmlRet += fmt.Sprintf("<li><a href=\"/http/%s\">%s</a></li>", filePath, filename)
|
|
||||||
}
|
|
||||||
davRet += "</d:multistatus>"
|
|
||||||
|
|
||||||
return davRet, htmlRet
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *TorrentManager) AssignedDirectoryCb(tor *Torrent, cb func(string)) {
|
func (t *TorrentManager) AssignedDirectoryCb(tor *Torrent, cb func(string)) {
|
||||||
|
|||||||
Reference in New Issue
Block a user