Fix traffic computation and optimize open ended requests

.github/workflows/build.yml

@@ -5,7 +5,7 @@ on:
     tags:
       - 'v**'
   schedule:
-    - cron: '50 1 * * *'
+    - cron: '45 1 * * *'
 
 jobs:
   build:

internal/handlers/home.go

@@ -30,8 +30,6 @@ type RootResponse struct {
 	Infuse              string            `json:"infuse"`
 	Logs                string            `json:"logs"`
 	UserInfo            *realdebrid.User  `json:"user_info"`
-	APITraffic       uint64            `json:"traffic_from_api"`
-	ServedMB         uint64            `json:"served_mb"`
 	LibrarySize         int               `json:"library_size"` // Number of torrents in the library
 	TorrentsToRepair    string            `json:"repair_queue"` // List of torrents in the repair queue
 	MemAlloc            uint64            `json:"mem_alloc"`    // Memory allocation in MB
@@ -45,6 +43,7 @@ type RootResponse struct {
 	DownloadTokens      []string          `json:"download_tokens"`
 	IDsToDelete         []string          `json:"ids_to_delete"`
 	Hosts               []string          `json:"hosts"`
+	TrafficServedPerAPI uint64            `json:"traffic_served_per_api"`
 }
 
 func (zr *Handlers) generateResponse(resp http.ResponseWriter, req *http.Request) (*RootResponse, error) {
@@ -106,8 +105,7 @@ func (zr *Handlers) generateResponse(resp http.ResponseWriter, req *http.Request
 		Infuse:              fmt.Sprintf("//%s/infuse/", req.Host),
 		Logs:                fmt.Sprintf("//%s/logs/", req.Host),
 		UserInfo:            userInfo,
-		APITraffic:       uint64(trafficFromAPI),
-		ServedMB:         bToMb(zr.downloader.TotalBytes.Load()),
+		TrafficServedPerAPI: uint64(trafficFromAPI),
 		LibrarySize:         allTorrents.Count(),
 		TorrentsToRepair:    repairQueueStr,
 		MemAlloc:            bToMb(mem.Alloc),
@@ -197,18 +195,6 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
 		response.Logs,
 	)
 
-	denominator := bToMb(response.APITraffic - zr.trafficOnStartup.Load())
-	if denominator == 0 {
-		denominator = 1
-	}
-	efficiency := response.ServedMB * 100 / denominator
-
-	if zr.trafficOnStartup.Load() > response.APITraffic {
-		// it cannot be bigger than traffic logged
-		// so it must be a reset back to 0
-		zr.trafficOnStartup.Store(0)
-	}
-
 	out += fmt.Sprintf(`
         <tr>
             <td>Library Size</td>
@@ -235,16 +221,12 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
             <td colspan="2">%d</td>
         </tr>
         <tr>
-            <td>Traffic Logged</td>
-            <td colspan="2">%d MB (%d MB added)</td>
+            <td>Traffic Served (main token)</td>
+            <td colspan="2">%d MB (%d MB since startup)</td>
         </tr>
         <tr>
-            <td>Traffic Served</td>
+            <td>Traffic Served (zurg)</td>
             <td colspan="2">%d MB</td>
-        </tr>
-        <tr>
-            <td>Traffic Efficiency</td>
-            <td colspan="2">%d%% (wasted %d MB) disclaimer: this assumes all RD traffic comes from this zurg instance</td>
         </tr>`,
 		response.LibrarySize,
 		response.MemAlloc,
@@ -252,11 +234,9 @@ func (zr *Handlers) handleHome(resp http.ResponseWriter, req *http.Request) {
 		response.Sys,
 		response.NumGC,
 		response.PID,
-		bToMb(response.APITraffic),
-		bToMb(response.APITraffic-zr.trafficOnStartup.Load()),
-		response.ServedMB,
-		efficiency,
-		bToMb(response.APITraffic-zr.trafficOnStartup.Load())-response.ServedMB,
+		bToMb(response.TrafficServedPerAPI), // traffic served *api*
+		bToMb(response.TrafficServedPerAPI-zr.downloader.TrafficOnStartup.Load()), // traffic served *api* since startup
+		bToMb(zr.downloader.TrafficServed.Load()),                                 // traffic served *zurg*
 	)
 
 	out += fmt.Sprintf(`

internal/handlers/router.go

@@ -6,7 +6,6 @@ import (
 	"net/url"
 	"path/filepath"
 	"strings"
-	"sync/atomic"
 
 	"github.com/debridmediamanager/zurg/internal/config"
 	"github.com/debridmediamanager/zurg/internal/dav"
@@ -27,7 +26,6 @@ type Handlers struct {
 	rd         *realdebrid.RealDebrid
 	workerPool *ants.Pool
 	hosts      []string
-	trafficOnStartup atomic.Uint64
 	log        *logutil.Logger
 }
 
@@ -48,16 +46,6 @@ func AttachHandlers(router *chi.Mux, downloader *universal.Downloader, torMgr *t
 		log:        log,
 	}
 
-	trafficDetails, err := rd.GetTrafficDetails()
-	if err != nil {
-		log.Errorf("Failed to get traffic details: %v", err)
-		trafficDetails = make(map[string]int64)
-	}
-	hs.trafficOnStartup.Store(uint64(0))
-	if _, ok := trafficDetails["real-debrid.com"]; ok {
-		hs.trafficOnStartup.Store(uint64(trafficDetails["real-debrid.com"]))
-	}
-
 	if cfg.GetUsername() != "" {
 		router.Use(hs.basicAuth)
 	}

internal/universal/downloader.go

@@ -2,6 +2,7 @@ package universal
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"path/filepath"
@@ -20,14 +21,26 @@ import (
 type Downloader struct {
 	rd               *realdebrid.RealDebrid
 	workerPool       *ants.Pool
-	TotalBytes atomic.Uint64
+	TrafficServed    atomic.Uint64
+	TrafficOnStartup atomic.Uint64
 }
 
 func NewDownloader(rd *realdebrid.RealDebrid, workerPool *ants.Pool) *Downloader {
-	return &Downloader{
+	dl := &Downloader{
 		rd:         rd,
 		workerPool: workerPool,
 	}
+
+	trafficDetails, err := dl.rd.GetTrafficDetails()
+	if err != nil {
+		trafficDetails = make(map[string]int64)
+	}
+	dl.TrafficOnStartup.Store(uint64(0))
+	if _, ok := trafficDetails["real-debrid.com"]; ok {
+		dl.TrafficOnStartup.Store(uint64(trafficDetails["real-debrid.com"]))
+	}
+
+	return dl
 }
 
 // StartResetBandwidthCountersJob is a permanent job that resets the bandwidth counters at 12AM CET
@@ -48,7 +61,8 @@ func (dl *Downloader) StartResetBandwidthCountersJob() {
 		ticker := time.NewTicker(24 * time.Hour)
 		for {
 			dl.rd.TokenManager.ResetAllTokens()
-			dl.TotalBytes.Store(0)
+			dl.TrafficServed.Store(0)
+			dl.TrafficOnStartup.Store(0)
 			<-ticker.C
 		}
 	})
@@ -162,7 +176,12 @@ func (dl *Downloader) streamFileToResponse(
 
 	// Add the range header if it exists
 	if req.Header.Get("Range") != "" {
-		dlReq.Header.Add("Range", req.Header.Get("Range"))
+		rangeVal := req.Header.Get("Range")
+		// check if open-ended range request (e.g. "bytes=999-")
+		if strings.HasSuffix(rangeVal, "-") {
+			rangeVal += fmt.Sprintf("%d", unrestrict.Filesize-1)
+		}
+		dlReq.Header.Add("Range", rangeVal)
 	}
 
 	downloadResp, err := dl.rd.DownloadFile(dlReq)
@@ -227,13 +246,14 @@ func (dl *Downloader) streamFileToResponse(
 		n, _ = io.Copy(resp, downloadResp.Body)
 	}
 
-	dl.workerPool.Submit(func() {
-		if n == 0 {
+	if !strings.HasPrefix(unrestrict.Link, "https://real-debrid.com/d/") {
 		return
 	}
-		// Update the download statistics
-		dl.TotalBytes.Add(uint64(n))
-		if cfg.ShouldLogRequests() {
+
+	dl.workerPool.Submit(func() {
+		dl.TrafficServed.Add(uint64(n))
+
+		if cfg.ShouldLogRequests() && bToMb(uint64(n)) > 0 {
 			log.Debugf("Served %d MB of file %s (range=%s)", bToMb(uint64(n)), unrestrict.Filename, req.Header.Get("Range"))
 		}
 	})

pkg/realdebrid/api.go

@@ -70,6 +70,10 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) {
 		tokenMap, _ := rd.UnrestrictMap.Get(token)
 		if tokenMap.Has(link) {
 			download, _ := tokenMap.Get(link)
+			if !rd.cfg.ShouldServeFromRclone() {
+				return download, nil
+			}
+
 			// check if the link is in the verified links cache
 			if expiry, ok := rd.verifiedLinks.Get(download.ID); ok && expiry > time.Now().Unix() {
 				return download, nil
@@ -98,6 +102,7 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) {
 
 		tokenMap.Set(link, download)
 
+		if rd.cfg.ShouldServeFromRclone() {
 			err = rd.downloadClient.VerifyLink(download.Download)
 			if utils.IsBytesLimitReached(err) {
 				rd.TokenManager.SetTokenAsExpired(token, "bandwidth limit exceeded")
@@ -109,6 +114,7 @@ func (rd *RealDebrid) UnrestrictAndVerify(link string) (*Download, error) {
 			}
 
 			rd.verifiedLinks.Set(download.ID, time.Now().Unix()+DOWNLOAD_LINK_EXPIRY)
+		}
 
 		return download, err
 	}