hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix traffic computation and optimize open ended requests

Browse Source
This commit is contained in:
Ben Adrian Sarmiento
2024-07-04 03:39:53 +02:00
parent 6deeb45db0
commit 49432dd810
5 changed files with 100 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build.yml vendored
View File

@@ -5,7 +5,7 @@ on:
tags: tags:
- 'v**' - 'v**'
schedule: schedule:
- cron: '50 1 * * *' - cron: '45 1 * * *'
jobs: jobs:
build: build:

108
internal/handlers/home.go
View File

@@ -22,29 +22,28 @@ type SponsorResponse struct {
} }
type RootResponse struct { type RootResponse struct {
Version string `json:"version"` Version string `json:"version"`
BuiltAt string `json:"built_at"` BuiltAt string `json:"built_at"`
GitCommit string `json:"git_commit"` GitCommit string `json:"git_commit"`
Html string `json:"html"` Html string `json:"html"`
Dav string `json:"dav"` Dav string `json:"dav"`
Infuse string `json:"infuse"` Infuse string `json:"infuse"`
Logs string `json:"logs"` Logs string `json:"logs"`
UserInfo *realdebrid.User `json:"user_info"` UserInfo *realdebrid.User `json:"user_info"`
APITraffic uint64 `json:"traffic_from_api"` LibrarySize int `json:"library_size"` // Number of torrents in the library
ServedMB uint64 `json:"served_mb"` TorrentsToRepair string `json:"repair_queue"` // List of torrents in the repair queue
LibrarySize int `json:"library_size"` // Number of torrents in the library MemAlloc uint64 `json:"mem_alloc"` // Memory allocation in MB
TorrentsToRepair string `json:"repair_queue"` // List of torrents in the repair queue TotalAlloc uint64 `json:"total_alloc"` // Total memory allocated in MB
MemAlloc uint64 `json:"mem_alloc"` // Memory allocation in MB Sys uint64 `json:"sys"` // System memory in MB
TotalAlloc uint64 `json:"total_alloc"` // Total memory allocated in MB NumGC uint32 `json:"num_gc"` // Number of completed GC cycles
Sys uint64 `json:"sys"` // System memory in MB PID int `json:"pid"` // Process ID
NumGC uint32 `json:"num_gc"` // Number of completed GC cycles Sponsor SponsorResponse `json:"sponsor_zurg"` // Sponsorship links
PID int `json:"pid"` // Process ID Config config.ZurgConfig `json:"config"`
Sponsor SponsorResponse `json:"sponsor_zurg"` // Sponsorship links Token string `json:"token"`
Config config.ZurgConfig `json:"config"` DownloadTokens []string `json:"download_tokens"`
Token string `json:"token"` IDsToDelete []string `json:"ids_to_delete"`
DownloadTokens []string `json:"download_tokens"` Hosts []string `json:"hosts"`
IDsToDelete []string `json:"ids_to_delete"` TrafficServedPerAPI uint64 `json:"traffic_served_per_api"`
Hosts []string `json:"hosts"`
} }
func (zr *Handlers) generateResponse(resp http.ResponseWriter, req *http.Request) (*RootResponse, error) { func (zr *Handlers) generateResponse(resp http.ResponseWriter, req *http.Request) (*RootResponse, error) {
@@ -98,23 +97,22 @@ func (zr *Handlers) generateResponse(resp http.ResponseWriter, req *http.Request
} }
return &RootResponse{ return &RootResponse{
Version: version.GetVersion(), Version: version.GetVersion(),
BuiltAt: version.GetBuiltAt(), BuiltAt: version.GetBuiltAt(),
GitCommit: version.GetGitCommit(), GitCommit: version.GetGitCommit(),
Html: fmt.Sprintf("//%s/http/", req.Host), Html: fmt.Sprintf("//%s/http/", req.Host),
Dav: fmt.Sprintf("//%s/dav/", req.Host), Dav: fmt.Sprintf("//%s/dav/", req.Host),
Infuse: fmt.Sprintf("//%s/infuse/", req.Host), Infuse: fmt.Sprintf("//%s/infuse/", req.Host),
Logs: fmt.Sprintf("//%s/logs/", req.Host), Logs: fmt.Sprintf("//%s/logs/", req.Host),
UserInfo: userInfo, UserInfo: userInfo,
APITraffic: uint64(trafficFromAPI), TrafficServedPerAPI: uint64(trafficFromAPI),
ServedMB: bToMb(zr.downloader.TotalBytes.Load()), LibrarySize: allTorrents.Count(),
LibrarySize: allTorrents.Count(), TorrentsToRepair: repairQueueStr,
TorrentsToRepair: repairQueueStr, MemAlloc: bToMb(mem.Alloc),
MemAlloc: bToMb(mem.Alloc), TotalAlloc: bToMb(mem.TotalAlloc),
TotalAlloc: bToMb(mem.TotalAlloc), Sys: bToMb(mem.Sys),
Sys: bToMb(mem.Sys), NumGC: mem.NumGC,
NumGC: mem.NumGC, PID: os.Getpid(),
PID: os.Getpid(),
Sponsor: SponsorResponse{ Sponsor: SponsorResponse{
Patreon: "https://www.patreon.com/debridmediamanager", Patreon: "https://www.patreon.com/debridmediamanager",
Github: "https://github.com/sponsors/debridmediamanager", Github: "https://github.com/sponsors/debridmediamanager",
@@ -197,18 +195,6 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
response.Logs, response.Logs,
) )
denominator := bToMb(response.APITraffic - zr.trafficOnStartup.Load())
if denominator == 0 {
denominator = 1
}
efficiency := response.ServedMB * 100 / denominator
if zr.trafficOnStartup.Load() > response.APITraffic {
// it cannot be bigger than traffic logged
// so it must be a reset back to 0
zr.trafficOnStartup.Store(0)
}
out += fmt.Sprintf(` out += fmt.Sprintf(`
<tr> <tr>
<td>Library Size</td> <td>Library Size</td>
@@ -235,16 +221,12 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
<td colspan="2">%d</td> <td colspan="2">%d</td>
</tr> </tr>
<tr> <tr>
<td>Traffic Logged</td> <td>Traffic Served (main token)</td>
<td colspan="2">%d MB (%d MB added)</td> <td colspan="2">%d MB (%d MB since startup)</td>
</tr> </tr>
<tr> <tr>
<td>Traffic Served</td> <td>Traffic Served (zurg)</td>
<td colspan="2">%d MB</td> <td colspan="2">%d MB</td>
</tr>
<tr>
<td>Traffic Efficiency</td>
<td colspan="2">%d%% (wasted %d MB) disclaimer: this assumes all RD traffic comes from this zurg instance</td>
</tr>`, </tr>`,
response.LibrarySize, response.LibrarySize,
response.MemAlloc, response.MemAlloc,
@@ -252,11 +234,9 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
response.Sys, response.Sys,
response.NumGC, response.NumGC,
response.PID, response.PID,
bToMb(response.APITraffic), bToMb(response.TrafficServedPerAPI), // traffic served *api*
bToMb(response.APITraffic-zr.trafficOnStartup.Load()), bToMb(response.TrafficServedPerAPI-zr.downloader.TrafficOnStartup.Load()), // traffic served *api* since startup
response.ServedMB, bToMb(zr.downloader.TrafficServed.Load()), // traffic served *zurg*
efficiency,
bToMb(response.APITraffic-zr.trafficOnStartup.Load())-response.ServedMB,
) )
out += fmt.Sprintf(` out += fmt.Sprintf(`

26
internal/handlers/router.go
View File

@@ -6,7 +6,6 @@ import (
"net/url" "net/url"
"path/filepath" "path/filepath"
"strings" "strings"
"sync/atomic"
"github.com/debridmediamanager/zurg/internal/config" "github.com/debridmediamanager/zurg/internal/config"
"github.com/debridmediamanager/zurg/internal/dav" "github.com/debridmediamanager/zurg/internal/dav"
@@ -21,14 +20,13 @@ import (
) )
type Handlers struct { type Handlers struct {
downloader *universal.Downloader downloader *universal.Downloader
torMgr *torrent.TorrentManager torMgr *torrent.TorrentManager
cfg config.ConfigInterface cfg config.ConfigInterface
rd *realdebrid.RealDebrid rd *realdebrid.RealDebrid
workerPool *ants.Pool workerPool *ants.Pool
hosts []string hosts []string
trafficOnStartup atomic.Uint64 log *logutil.Logger
log *logutil.Logger
} }
func init() { func init() {
@@ -48,16 +46,6 @@ func AttachHandlers(router *chi.Mux, downloader *universal.Downloader, torMgr *t
log: log, log: log,
} }
trafficDetails, err := rd.GetTrafficDetails()
if err != nil {
log.Errorf("Failed to get traffic details: %v", err)
trafficDetails = make(map[string]int64)
}
hs.trafficOnStartup.Store(uint64(0))
if _, ok := trafficDetails["real-debrid.com"]; ok {
hs.trafficOnStartup.Store(uint64(trafficDetails["real-debrid.com"]))
}
if cfg.GetUsername() != "" { if cfg.GetUsername() != "" {
router.Use(hs.basicAuth) router.Use(hs.basicAuth)
} }

44
internal/universal/downloader.go
View File

@@ -2,6 +2,7 @@ package universal
import ( import (
"context" "context"
"fmt"
"io" "io"
"net/http" "net/http"
"path/filepath" "path/filepath"
@@ -18,16 +19,28 @@ import (
) )
type Downloader struct { type Downloader struct {
rd *realdebrid.RealDebrid rd *realdebrid.RealDebrid
workerPool *ants.Pool workerPool *ants.Pool
TotalBytes atomic.Uint64 TrafficServed atomic.Uint64
TrafficOnStartup atomic.Uint64
} }
func NewDownloader(rd *realdebrid.RealDebrid, workerPool *ants.Pool) *Downloader { func NewDownloader(rd *realdebrid.RealDebrid, workerPool *ants.Pool) *Downloader {
return &Downloader{ dl := &Downloader{
rd: rd, rd: rd,
workerPool: workerPool, workerPool: workerPool,
} }
trafficDetails, err := dl.rd.GetTrafficDetails()
if err != nil {
trafficDetails = make(map[string]int64)
}
dl.TrafficOnStartup.Store(uint64(0))
if _, ok := trafficDetails["real-debrid.com"]; ok {
dl.TrafficOnStartup.Store(uint64(trafficDetails["real-debrid.com"]))
}
return dl
} }
// StartResetBandwidthCountersJob is a permanent job that resets the bandwidth counters at 12AM CET // StartResetBandwidthCountersJob is a permanent job that resets the bandwidth counters at 12AM CET
@@ -48,7 +61,8 @@ func (dl *Downloader) StartResetBandwidthCountersJob() {
ticker := time.NewTicker(24 * time.Hour) ticker := time.NewTicker(24 * time.Hour)
for { for {
dl.rd.TokenManager.ResetAllTokens() dl.rd.TokenManager.ResetAllTokens()
dl.TotalBytes.Store(0) dl.TrafficServed.Store(0)
dl.TrafficOnStartup.Store(0)
<-ticker.C <-ticker.C
} }
}) })
@@ -162,7 +176,12 @@ func (dl *Downloader) streamFileToResponse(
// Add the range header if it exists // Add the range header if it exists
if req.Header.Get("Range") != "" { if req.Header.Get("Range") != "" {
dlReq.Header.Add("Range", req.Header.Get("Range")) rangeVal := req.Header.Get("Range")
// check if open-ended range request (e.g. "bytes=999-")
if strings.HasSuffix(rangeVal, "-") {
rangeVal += fmt.Sprintf("%d", unrestrict.Filesize-1)
}
dlReq.Header.Add("Range", rangeVal)
} }
downloadResp, err := dl.rd.DownloadFile(dlReq) downloadResp, err := dl.rd.DownloadFile(dlReq)
@@ -227,13 +246,14 @@ func (dl *Downloader) streamFileToResponse(
n, _ = io.Copy(resp, downloadResp.Body) n, _ = io.Copy(resp, downloadResp.Body)
} }
if !strings.HasPrefix(unrestrict.Link, "https://real-debrid.com/d/") {
return
}
dl.workerPool.Submit(func() { dl.workerPool.Submit(func() {
if n == 0 { dl.TrafficServed.Add(uint64(n))
return
} if cfg.ShouldLogRequests() && bToMb(uint64(n)) > 0 {
// Update the download statistics
dl.TotalBytes.Add(uint64(n))
if cfg.ShouldLogRequests() {
log.Debugf("Served %d MB of file %s (range=%s)", bToMb(uint64(n)), unrestrict.Filename, req.Header.Get("Range")) log.Debugf("Served %d MB of file %s (range=%s)", bToMb(uint64(n)), unrestrict.Filename, req.Header.Get("Range"))
} }
}) })

26
pkg/realdebrid/api.go
View File

@@ -70,6 +70,10 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) {
tokenMap, _ := rd.UnrestrictMap.Get(token) tokenMap, _ := rd.UnrestrictMap.Get(token)
if tokenMap.Has(link) { if tokenMap.Has(link) {
download, _ := tokenMap.Get(link) download, _ := tokenMap.Get(link)
if !rd.cfg.ShouldServeFromRclone() {
return download, nil
}
// check if the link is in the verified links cache // check if the link is in the verified links cache
if expiry, ok := rd.verifiedLinks.Get(download.ID); ok && expiry > time.Now().Unix() { if expiry, ok := rd.verifiedLinks.Get(download.ID); ok && expiry > time.Now().Unix() {
return download, nil return download, nil
@@ -98,17 +102,19 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) {
tokenMap.Set(link, download) tokenMap.Set(link, download)
err = rd.downloadClient.VerifyLink(download.Download) if rd.cfg.ShouldServeFromRclone() {
if utils.IsBytesLimitReached(err) { err = rd.downloadClient.VerifyLink(download.Download)
rd.TokenManager.SetTokenAsExpired(token, "bandwidth limit exceeded") if utils.IsBytesLimitReached(err) {
continue rd.TokenManager.SetTokenAsExpired(token, "bandwidth limit exceeded")
} else if utils.IsInvalidDownloadCode(err) { continue
continue } else if utils.IsInvalidDownloadCode(err) {
} else if err != nil { continue
return nil, err } else if err != nil {
} return nil, err
}
rd.verifiedLinks.Set(download.ID, time.Now().Unix()+DOWNLOAD_LINK_EXPIRY) rd.verifiedLinks.Set(download.ID, time.Now().Unix()+DOWNLOAD_LINK_EXPIRY)
}
return download, err return download, err
} }