hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use the chunk manager properly

Browse Source
This commit is contained in:
Ben Sarmiento
2023-11-18 01:51:25 +01:00
parent 4da9416bec
commit b669f8d673
10 changed files with 30 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -30,3 +30,5 @@ zurg
config.yml
error_videos/*.mp4
mnt/

21
cmd/zurg/main.go
View File

@@ -16,6 +16,7 @@ import (
"github.com/debridmediamanager.com/zurg/internal/version"
"github.com/debridmediamanager.com/zurg/internal/zfs"
"github.com/debridmediamanager.com/zurg/pkg/chunk"
zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
"github.com/hashicorp/golang-lru/v2/expirable"
@@ -44,7 +45,7 @@ func main() {
cache := expirable.NewLRU[string, string](1e4, nil, time.Hour)
rd := realdebrid.NewRealDebrid(config.GetToken(), config, logutil.NewLogger().Named("realdebrid"))
rd := realdebrid.NewRealDebrid(config.GetToken(), logutil.NewLogger().Named("realdebrid"))
torrentMgr := torrent.NewTorrentManager(config, rd)
@@ -65,16 +66,16 @@ func main() {
}()
log.Debugf("Initializing chunk manager, cores: %d", runtime.NumCPU())
// 64kb request size
client := zurghttp.NewHTTPClient(config.GetToken(), 10, config)
chunkMgr, err := chunk.NewManager(
"", // in-memory chunk file
5242880, // 10MB
max((runtime.NumCPU()/2)-1, 1), // 1 chunk - load ahead (1MB total)
max(runtime.NumCPU()/2, 1), // check threads
max(runtime.NumCPU()/2, 1), // load threads
runtime.NumCPU()*4,
torrentMgr, // max chunks
config)
"", // in-memory chunks
10485760, // 10MB chunk size
max(runtime.NumCPU()/2, 1), // 8 cores/2 = 4 chunks to load ahead
max(runtime.NumCPU()/2, 1), // 4 check threads
max(runtime.NumCPU()-1, 1), // number of chunks that should be read ahead
runtime.NumCPU()*2, // total chunks kept in memory
torrentMgr,
client)
if nil != err {
log.Panicf("Failed to initialize chunk manager: %v", err)
}

2
go.mod
View File

@@ -11,6 +11,6 @@ require (
)
require (
github.com/winfsp/cgofuse v1.5.0 // indirect
github.com/winfsp/cgofuse v1.5.0
go.uber.org/multierr v1.10.0 // indirect
)

4
go.sum
View File

@@ -1,5 +1,3 @@
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5 h1:A0NsYy4lDBZAC6QiYeJ4N+XuHIKBpyhAVRMHRQZKTeQ=
bazil.org/fuse v0.0.0-20230120002735-62a210ff1fd5/go.mod h1:gG3RZAMXCa/OTes6rr9EwusmR1OH1tDDy+cg9c5YliY=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
@@ -10,8 +8,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
github.com/winfsp/cgofuse v1.5.0 h1:MsBP7Mi/LiJf/7/F3O/7HjjR009ds6KCdqXzKpZSWxI=
github.com/winfsp/cgofuse v1.5.0/go.mod h1:h3awhoUOcn2VYVKCwDaYxSLlZwnyK+A8KaDoLUp2lbU=
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=

6
internal/net/router.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/dav"
intHttp "github.com/debridmediamanager.com/zurg/internal/http"
zurghttp "github.com/debridmediamanager.com/zurg/internal/http"
"github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/internal/universal"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
@@ -25,11 +25,11 @@ func Router(mux *http.ServeMux, c config.ConfigInterface, t *torrent.TorrentMana
if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 {
universal.HandleGetRequest(w, r, t, c, cache)
} else {
intHttp.HandleDirectoryListing(w, r, t, c)
zurghttp.HandleDirectoryListing(w, r, t, c)
}
case http.MethodHead:
universal.HandleHeadRequest(w, r, t, c, cache)
universal.HandleHeadRequest(w, r, t, cache)
default:
log.Errorf("Request %s %s not supported yet", r.Method, r.URL.Path)

3
internal/universal/head.go
View File

@@ -7,13 +7,12 @@ import (
"path/filepath"
"strings"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/torrent"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
"github.com/hashicorp/golang-lru/v2/expirable"
)
func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, c config.ConfigInterface, cache *expirable.LRU[string, string]) {
func HandleHeadRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, cache *expirable.LRU[string, string]) {
log := logutil.NewLogger().Named("unihead")
requestPath := path.Clean(r.URL.Path)

8
pkg/chunk/download.go
View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/debridmediamanager.com/zurg/internal/torrent"
zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
"go.uber.org/zap"
"golang.org/x/sys/unix"
@@ -23,12 +24,13 @@ type Downloader struct {
storage *Storage
torMgr *torrent.TorrentManager
log *zap.SugaredLogger
client *zurghttp.HTTPClient
}
type DownloadCallback func(error, []byte)
// NewDownloader creates a new download manager
func NewDownloader(threads int, storage *Storage, bufferSize int64, torMgr *torrent.TorrentManager) (*Downloader, error) {
func NewDownloader(threads int, storage *Storage, bufferSize int64, torMgr *torrent.TorrentManager, client *zurghttp.HTTPClient) (*Downloader, error) {
rlog := logutil.NewLogger()
log := rlog.Named("downloader")
@@ -38,6 +40,7 @@ func NewDownloader(threads int, storage *Storage, bufferSize int64, torMgr *torr
callbacks: make(map[RequestID][]DownloadCallback, 100),
storage: storage,
torMgr: torMgr,
client: client,
log: log,
}
@@ -76,7 +79,6 @@ func (d *Downloader) thread(n int) {
}
func (d *Downloader) download(req *Request, buffer []byte) {
d.log.Debugf("Starting download %v (preload: %v)", req.id, req.preload)
err := d.downloadFromAPI(req, buffer, 0)
d.lock.Lock()
@@ -137,7 +139,7 @@ func (d *Downloader) downloadFromAPI(request *Request, buffer []byte, delay int6
d.log.Debugf("response read error: %v", err)
return fmt.Errorf("could not read objects %s %s API response", request.file.Path, request.file.Link)
}
d.log.Debugf("Downloaded %v bytes of %s %s", n, request.file.Path, request.file.Link)
d.log.Debugf("Downloaded %v bytes of %s", n, request.file.Path)
return nil
}

6
pkg/chunk/manager.go
View File

@@ -6,8 +6,8 @@ import (
"hash/fnv"
"os"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/internal/torrent"
zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
)
// Manager manages chunks on disk
@@ -59,7 +59,7 @@ func NewManager(
loadThreads,
maxChunks int,
torMgr *torrent.TorrentManager,
cfg config.ConfigInterface) (*Manager, error) {
client *zurghttp.HTTPClient) (*Manager, error) {
pageSize := int64(os.Getpagesize())
if chunkSize < pageSize {
@@ -82,7 +82,7 @@ func NewManager(
return nil, err
}
downloader, err := NewDownloader(loadThreads, storage, chunkSize, torMgr)
downloader, err := NewDownloader(loadThreads, storage, chunkSize, torMgr, client)
if nil != err {
return nil, err
}

6
pkg/http/client.go
View File

@@ -14,7 +14,7 @@ type HTTPClient struct {
Client *http.Client
MaxRetries int
Backoff func(attempt int) time.Duration
CheckRespStatus func(resp *http.Response, err error, log *zap.SugaredLogger) bool
CheckRespStatus func(resp *http.Response, err error) bool
BearerToken string
log *zap.SugaredLogger
config config.ConfigInterface
@@ -34,7 +34,7 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
var err error
for attempt := 0; attempt < r.MaxRetries; attempt++ {
resp, err = r.Client.Do(req)
if !r.CheckRespStatus(resp, err, r.log) {
if !r.CheckRespStatus(resp, err) {
return resp, err
}
time.Sleep(r.Backoff(attempt))
@@ -50,7 +50,7 @@ func NewHTTPClient(token string, maxRetries int, cfg config.ConfigInterface) *HT
Backoff: func(attempt int) time.Duration {
return time.Duration(attempt) * time.Second
},
CheckRespStatus: func(resp *http.Response, err error, log *zap.SugaredLogger) bool {
CheckRespStatus: func(resp *http.Response, err error) bool {
if err != nil {
return true
}

3
pkg/realdebrid/api.go
View File

@@ -10,7 +10,6 @@ import (
"strconv"
"strings"
"github.com/debridmediamanager.com/zurg/internal/config"
zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
"go.uber.org/zap"
)
@@ -20,7 +19,7 @@ type RealDebrid struct {
client *zurghttp.HTTPClient
}
func NewRealDebrid(accessToken string, config config.ConfigInterface, log *zap.SugaredLogger) *RealDebrid {
func NewRealDebrid(accessToken string, log *zap.SugaredLogger) *RealDebrid {
maxRetries := 10
client := zurghttp.NewHTTPClient(accessToken, maxRetries, nil)
return &RealDebrid{