diff --git a/.gitignore b/.gitignore index 2809d78..668d170 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ zurg config.yml error_videos/*.mp4 + +mnt/ diff --git a/cmd/zurg/main.go b/cmd/zurg/main.go index 04f881d..228c578 100644 --- a/cmd/zurg/main.go +++ b/cmd/zurg/main.go @@ -16,6 +16,7 @@ import ( "github.com/debridmediamanager.com/zurg/internal/version" "github.com/debridmediamanager.com/zurg/internal/zfs" "github.com/debridmediamanager.com/zurg/pkg/chunk" + zurghttp "github.com/debridmediamanager.com/zurg/pkg/http" "github.com/debridmediamanager.com/zurg/pkg/logutil" "github.com/debridmediamanager.com/zurg/pkg/realdebrid" "github.com/hashicorp/golang-lru/v2/expirable" @@ -44,7 +45,7 @@ func main() { cache := expirable.NewLRU[string, string](1e4, nil, time.Hour) - rd := realdebrid.NewRealDebrid(config.GetToken(), config, logutil.NewLogger().Named("realdebrid")) + rd := realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid")) torrentMgr := torrent.NewTorrentManager(config, rd) @@ -65,16 +66,16 @@ func main() { }() log.Debugf("Initializing chunk manager, cores: %d", runtime.NumCPU()) - // 64kb request size + client := zurghttp.NewHTTPClient(config.GetToken(), 10, config) chunkMgr, err := chunk.NewManager( - "", // in-memory chunk file - 5242880, // 10MB - max((runtime.NumCPU()/2)-1, 1), // 1 chunk - load ahead (1MB total) - max(runtime.NumCPU()/2, 1), // check threads - max(runtime.NumCPU()/2, 1), // load threads - runtime.NumCPU()*4, - torrentMgr, // max chunks - config) + "", // in-memory chunks + 10485760, // 10MB chunk size + max(runtime.NumCPU()/2, 1), // 8 cores/2 = 4 chunks to load ahead + max(runtime.NumCPU()/2, 1), // 4 check threads + max(runtime.NumCPU()-1, 1), // number of chunks that should be read ahead + runtime.NumCPU()*2, // total chunks kept in memory + torrentMgr, + client) if nil != err { log.Panicf("Failed to initialize chunk manager: %v", err) } diff --git a/go.mod b/go.mod index 8e1158d..89a76d9 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,6 @@ require ( ) require ( - github.com/winfsp/cgofuse v1.5.0 // indirect + github.com/winfsp/cgofuse v1.5.0 go.uber.org/multierr v1.10.0 // indirect ) diff --git a/go.sum b/go.sum index 0f6d766..963bc15 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 h1:A0NsYy4lDBZAC6QiYeJ4N+XuHIKBpyhAVRMHRQZKTeQ= -bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5/go.mod h1:gG3RZAMXCa/OTes6rr9EwusmR1OH1tDDy+cg9c5YliY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= @@ -10,8 +8,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/winfsp/cgofuse v1.5.0 h1:MsBP7Mi/LiJf/7/F3O/7HjjR009ds6KCdqXzKpZSWxI= github.com/winfsp/cgofuse v1.5.0/go.mod h1:h3awhoUOcn2VYVKCwDaYxSLlZwnyK+A8KaDoLUp2lbU= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= diff --git a/internal/net/router.go b/internal/net/router.go index c5d8caa..0d02245 100644 --- a/internal/net/router.go +++ b/internal/net/router.go @@ -7,7 +7,7 @@ import ( "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/dav" - intHttp "github.com/debridmediamanager.com/zurg/internal/http" + zurghttp "github.com/debridmediamanager.com/zurg/internal/http" "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/internal/universal" "github.com/debridmediamanager.com/zurg/pkg/logutil" @@ -25,11 +25,11 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 { universal.HandleGetRequest(w, r, t, c, cache) } else { - intHttp.HandleDirectoryListing(w, r, t, c) + zurghttp.HandleDirectoryListing(w, r, t, c) } case http.MethodHead: - universal.HandleHeadRequest(w, r, t, c, cache) + universal.HandleHeadRequest(w, r, t, cache) default: log.Errorf("Request %s %s not supported yet", r.Method, r.URL.Path) diff --git a/internal/universal/head.go b/internal/universal/head.go index 346839f..160dff0 100644 --- a/internal/universal/head.go +++ b/internal/universal/head.go @@ -7,13 +7,12 @@ import ( "path/filepath" "strings" - "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/torrent" "github.com/debridmediamanager.com/zurg/pkg/logutil" "github.com/hashicorp/golang-lru/v2/expirable" ) -func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) { +func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, cache *expirable.LRU[string, string]) { log := logutil.NewLogger().Named("unihead") requestPath := path.Clean(r.URL.Path) diff --git a/pkg/chunk/download.go b/pkg/chunk/download.go index 6f096a3..532373a 100644 --- a/pkg/chunk/download.go +++ b/pkg/chunk/download.go @@ -9,6 +9,7 @@ import ( "time" "github.com/debridmediamanager.com/zurg/internal/torrent" + zurghttp "github.com/debridmediamanager.com/zurg/pkg/http" "github.com/debridmediamanager.com/zurg/pkg/logutil" "go.uber.org/zap" "golang.org/x/sys/unix" @@ -23,12 +24,13 @@ type Downloader struct { storage *Storage torMgr *torrent.TorrentManager log *zap.SugaredLogger + client *zurghttp.HTTPClient } type DownloadCallback func(error, []byte) // NewDownloader creates a new download manager -func NewDownloader(threads int, storage *Storage, bufferSize int64, torMgr *torrent.TorrentManager) (*Downloader, error) { +func NewDownloader(threads int, storage *Storage, bufferSize int64, torMgr *torrent.TorrentManager, client *zurghttp.HTTPClient) (*Downloader, error) { rlog := logutil.NewLogger() log := rlog.Named("downloader") @@ -38,6 +40,7 @@ func NewDownloader(threads int, storage *Storage, bufferSize int64, torMgr *torr callbacks: make(map[RequestID][]DownloadCallback, 100), storage: storage, torMgr: torMgr, + client: client, log: log, } @@ -76,7 +79,6 @@ func (d *Downloader) thread(n int) { } func (d *Downloader) download(req *Request, buffer []byte) { - d.log.Debugf("Starting download %v (preload: %v)", req.id, req.preload) err := d.downloadFromAPI(req, buffer, 0) d.lock.Lock() @@ -137,7 +139,7 @@ func (d *Downloader) downloadFromAPI(request *Request, buffer []byte, delay int6 d.log.Debugf("response read error: %v", err) return fmt.Errorf("could not read objects %s %s API response", request.file.Path, request.file.Link) } - d.log.Debugf("Downloaded %v bytes of %s %s", n, request.file.Path, request.file.Link) + d.log.Debugf("Downloaded %v bytes of %s", n, request.file.Path) return nil } diff --git a/pkg/chunk/manager.go b/pkg/chunk/manager.go index 4ac1334..d18d333 100644 --- a/pkg/chunk/manager.go +++ b/pkg/chunk/manager.go @@ -6,8 +6,8 @@ import ( "hash/fnv" "os" - "github.com/debridmediamanager.com/zurg/internal/config" "github.com/debridmediamanager.com/zurg/internal/torrent" + zurghttp "github.com/debridmediamanager.com/zurg/pkg/http" ) // Manager manages chunks on disk @@ -59,7 +59,7 @@ func NewManager( loadThreads, maxChunks int, torMgr *torrent.TorrentManager, - cfg config.ConfigInterface) (*Manager, error) { + client *zurghttp.HTTPClient) (*Manager, error) { pageSize := int64(os.Getpagesize()) if chunkSize < pageSize { @@ -82,7 +82,7 @@ func NewManager( return nil, err } - downloader, err := NewDownloader(loadThreads, storage, chunkSize, torMgr) + downloader, err := NewDownloader(loadThreads, storage, chunkSize, torMgr, client) if nil != err { return nil, err } diff --git a/pkg/http/client.go b/pkg/http/client.go index bd86b16..0be76ee 100644 --- a/pkg/http/client.go +++ b/pkg/http/client.go @@ -14,7 +14,7 @@ type HTTPClient struct { Client *http.Client MaxRetries int Backoff func(attempt int) time.Duration - CheckRespStatus func(resp *http.Response, err error, log *zap.SugaredLogger) bool + CheckRespStatus func(resp *http.Response, err error) bool BearerToken string log *zap.SugaredLogger config config.ConfigInterface @@ -34,7 +34,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) { var err error for attempt := 0; attempt < r.MaxRetries; attempt++ { resp, err = r.Client.Do(req) - if !r.CheckRespStatus(resp, err, r.log) { + if !r.CheckRespStatus(resp, err) { return resp, err } time.Sleep(r.Backoff(attempt)) @@ -50,7 +50,7 @@ func NewHTTPClient(token string, maxRetries int, cfg config.ConfigInterface) *HT Backoff: func(attempt int) time.Duration { return time.Duration(attempt) * time.Second }, - CheckRespStatus: func(resp *http.Response, err error, log *zap.SugaredLogger) bool { + CheckRespStatus: func(resp *http.Response, err error) bool { if err != nil { return true } diff --git a/pkg/realdebrid/api.go b/pkg/realdebrid/api.go index 0a92666..8f66dee 100644 --- a/pkg/realdebrid/api.go +++ b/pkg/realdebrid/api.go @@ -10,7 +10,6 @@ import ( "strconv" "strings" - "github.com/debridmediamanager.com/zurg/internal/config" zurghttp "github.com/debridmediamanager.com/zurg/pkg/http" "go.uber.org/zap" ) @@ -20,7 +19,7 @@ type RealDebrid struct { client *zurghttp.HTTPClient } -func NewRealDebrid(accessToken string, config config.ConfigInterface, log *zap.SugaredLogger) *RealDebrid { +func NewRealDebrid(accessToken string, log *zap.SugaredLogger) *RealDebrid { maxRetries := 10 client := zurghttp.NewHTTPClient(accessToken, maxRetries, nil) return &RealDebrid{