hpz937/zurg
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Hotfix

Browse Source
This commit is contained in:
Ben Sarmiento
2023-11-27 21:50:00 +01:00
parent a7623de410
commit c8334ecb3b
15 changed files with 277 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
pkg/dav/response.go
View File

@@ -1,39 +1,22 @@
package dav
import "fmt"
func DavDirectory(path, added string) Response {
return Response{
Href: "/" + customPathEscape(path),
Propstat: PropStat{
Prop: Prop{
ResourceType: ResourceType{Value: "<d:collection/>"},
LastModified: added,
},
Status: "HTTP/1.1 200 OK",
},
}
}
func DavFile(path string, fileSize int64, added string, link string) Response {
return Response{
Href: "/" + customPathEscape(path),
Propstat: PropStat{
Prop: Prop{
ContentLength: fileSize,
LastModified: added,
},
Status: "HTTP/1.1 200 OK",
},
}
}
import (
"fmt"
"path/filepath"
)
// optimized versions, no more marshalling
func Directory(path, added string) string {
func BaseDirectory(path, added string) string {
return fmt.Sprintf("<d:response><d:href>/%s</d:href><d:propstat><d:prop><d:resourcetype><d:collection/></d:resourcetype><d:getlastmodified>%s</d:getlastmodified></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>", customPathEscape(path), added)
}
func File(path string, fileSize int64, added string) string {
return fmt.Sprintf("<d:response><d:href>/%s</d:href><d:propstat><d:prop><d:resourcetype></d:resourcetype><d:getcontentlength>%d</d:getcontentlength><d:getlastmodified>%s</d:getlastmodified></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>", customPathEscape(path), fileSize, added)
func Directory(path, added string) string {
path = filepath.Base(path)
return fmt.Sprintf("<d:response><d:href>%s</d:href><d:propstat><d:prop><d:resourcetype><d:collection/></d:resourcetype><d:getlastmodified>%s</d:getlastmodified></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>", customPathEscape(path), added)
}
func File(path string, fileSize int64, added string) string {
path = filepath.Base(path)
return fmt.Sprintf("<d:response><d:href>%s</d:href><d:propstat><d:prop><d:resourcetype></d:resourcetype><d:getcontentlength>%d</d:getcontentlength><d:getlastmodified>%s</d:getlastmodified></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>", customPathEscape(path), fileSize, added)
}

1
pkg/dav/util.go
View File

@@ -18,5 +18,6 @@ func customPathEscape(input string) string {
escapedPath = strings.Replace(escapedPath, ">", "%3E", -1) // for >
escapedPath = strings.Replace(escapedPath, "\"", "%22", -1) // for "
escapedPath = strings.Replace(escapedPath, "'", "%27", -1) // for '
escapedPath = strings.Replace(escapedPath, ":", "%3A", -1) // for :
return escapedPath
}

79
pkg/http/client.go
View File

@@ -2,33 +2,34 @@ package http
import (
"context"
"math"
"net"
"net/http"
"strings"
"time"
"github.com/debridmediamanager.com/zurg/internal/config"
"github.com/debridmediamanager.com/zurg/pkg/logutil"
cmap "github.com/orcaman/concurrent-map/v2"
"go.uber.org/zap"
)
type HTTPClient struct {
Client *http.Client
MaxRetries int
Backoff func(attempt int) time.Duration
CheckRespStatus func(resp *http.Response, err error) bool
BearerToken string
log *zap.SugaredLogger
config config.ConfigInterface
IPv6 cmap.ConcurrentMap[string, net.IP]
Client *http.Client
MaxRetries int
Backoff func(attempt int) time.Duration
ShouldRetry func(resp *http.Response, err error) int
BearerToken string
log *zap.SugaredLogger
config config.ConfigInterface
IPv6 cmap.ConcurrentMap[string, net.IP]
}
func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
req.Close = true
if r.config != nil && strings.Contains(req.Host, "download.real-debrid.") {
prefHost := r.config.GetRandomPreferredHost()
if prefHost != "" {
req.Host = prefHost
req.URL.Host = prefHost
}
}
if r.BearerToken != "" {
@@ -72,35 +73,57 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
var resp *http.Response
var err error
for attempt := 0; attempt < r.MaxRetries; attempt++ {
attempt := 0
for {
resp, err = r.Client.Do(req)
if !r.CheckRespStatus(resp, err) {
if val := r.ShouldRetry(resp, err); val == -1 {
return resp, err
} else {
if val == 0 {
time.Sleep(8 * time.Second) // extra delay
} else {
attempt += val
if attempt > r.MaxRetries {
return resp, err
}
time.Sleep(r.Backoff(attempt))
}
}
time.Sleep(r.Backoff(attempt))
}
return resp, err
}
func NewHTTPClient(token string, maxRetries int, cfg config.ConfigInterface) *HTTPClient {
func NewHTTPClient(token string, maxRetries int, timeoutSecs int, cfg config.ConfigInterface, log *zap.SugaredLogger) *HTTPClient {
return &HTTPClient{
BearerToken: token,
Client: &http.Client{},
MaxRetries: maxRetries,
Client: &http.Client{
Timeout: time.Duration(timeoutSecs) * time.Second,
},
MaxRetries: maxRetries,
Backoff: func(attempt int) time.Duration {
return time.Duration(attempt) * time.Second
},
CheckRespStatus: func(resp *http.Response, err error) bool {
if err != nil {
return true
maxDuration := 60
backoff := int(math.Pow(2, float64(attempt)))
if backoff > maxDuration {
backoff = maxDuration
}
if resp.StatusCode == 429 {
return true
}
// no need to retry)
return false
return time.Duration(backoff) * time.Second
},
log: logutil.NewLogger().Named("client"),
ShouldRetry: func(resp *http.Response, err error) int {
if resp != nil {
if resp.StatusCode == 429 || resp.StatusCode == 400 {
return 0 // retry but don't increment attempt
}
return -1 // don't retry
} else if err != nil {
errStr := err.Error()
if strings.Contains(errStr, "EOF") || strings.Contains(errStr, "connection reset") || strings.Contains(errStr, "no such host") {
return 0 // retry but don't increment attempt
} else {
return 1
}
}
return 1 // retry and increment attempt
},
log: log,
config: cfg,
IPv6: cmap.New[net.IP](),
}

13
pkg/realdebrid/api.go
View File

@@ -19,7 +19,7 @@ type RealDebrid struct {
client *zurghttp.HTTPClient
}
func NewRealDebrid(accessToken string, client *zurghttp.HTTPClient, log *zap.SugaredLogger) *RealDebrid {
func NewRealDebrid(client *zurghttp.HTTPClient, log *zap.SugaredLogger) *RealDebrid {
return &RealDebrid{
log: log,
client: client,
@@ -69,7 +69,7 @@ func (rd *RealDebrid) GetTorrents(customLimit int) ([]Torrent, int, error) {
page := 1
limit := customLimit
if limit == 0 {
limit = 2500
limit = 1000
}
totalCount := 0
@@ -263,7 +263,7 @@ func (rd *RealDebrid) GetActiveTorrentCount() (*ActiveTorrentCountResponse, erro
return &response, nil
}
func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
func (rd *RealDebrid) UnrestrictLink(link string, checkFirstByte bool) (*UnrestrictResponse, error) {
data := url.Values{}
data.Set("link", link)
@@ -283,6 +283,11 @@ func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
rd.log.Errorf("Unrestrict link request returned status code %d for link %s", resp.StatusCode, link)
// return nil, fmt.Errorf("unrestrict link request returned status code %d so likely it has expired", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
// rd.log.Errorf("Error when reading the body of unrestrict link response: %v", err)
@@ -296,7 +301,7 @@ func (rd *RealDebrid) UnrestrictLink(link string) (*UnrestrictResponse, error) {
return nil, fmt.Errorf("undecodable response so likely it has expired")
}
if !rd.canFetchFirstByte(response.Download) {
if checkFirstByte && !rd.canFetchFirstByte(response.Download) {
return nil, fmt.Errorf("can't fetch first byte")
}

97
pkg/realdebrid/unrestrict.go
View File

@@ -3,91 +3,40 @@ package realdebrid
import (
"net/http"
"strings"
"time"
)
func (rd *RealDebrid) UnrestrictUntilOk(link string) *UnrestrictResponse {
func (rd *RealDebrid) UnrestrictUntilOk(link string, serveFromRclone bool) *UnrestrictResponse {
if !strings.HasPrefix(link, "http") {
return nil
}
unrestrictFn := func(link string) (*UnrestrictResponse, error) {
return rd.UnrestrictLink(link)
resp, _ := rd.UnrestrictLink(link, serveFromRclone)
if resp != nil {
return resp
}
return retryUntilOk(func() (*UnrestrictResponse, error) {
return unrestrictFn(link)
})
return nil
}
func (rd *RealDebrid) canFetchFirstByte(url string) bool {
const maxAttempts = 3
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return false
}
for i := 0; i < maxAttempts; i++ {
// Create a new HTTP request
req, err := http.NewRequest("GET", url, nil)
if err != nil {
continue
req.Header.Set("Range", "bytes=0-0")
resp, err := rd.client.Do(req)
if err != nil {
return false
}
defer resp.Body.Close()
// If server supports partial content
if resp.StatusCode == http.StatusPartialContent || resp.StatusCode == http.StatusOK {
buffer := make([]byte, 1)
_, err = resp.Body.Read(buffer)
if err == nil {
return true
}
// Set the Range header to request only the first byte
req.Header.Set("Range", "bytes=0-0")
// TODO set a proper client
resp, err := rd.client.Do(req)
if err != nil {
time.Sleep(1 * time.Second) // Add a delay before the next retry
continue
}
defer resp.Body.Close()
// If server supports partial content
if resp.StatusCode == http.StatusPartialContent {
buffer := make([]byte, 1)
_, err := resp.Body.Read(buffer)
if err == nil {
return true
}
} else if resp.StatusCode == http.StatusOK {
// If server doesn't support partial content, try reading the first byte and immediately close
buffer := make([]byte, 1)
_, err = resp.Body.Read(buffer)
if err == nil {
return true
}
}
time.Sleep(500 * time.Millisecond) // Add a delay before the next retry
}
return false
}
func retryUntilOk[T any](fn func() (T, error)) T {
// const initialDelay = 1 * time.Second
// const maxDelay = 128 * time.Second
const maxRetries = 2 // Maximum retries for non-429 errors
var result T
var err error
var retryCount int
for {
result, err = fn()
if err == nil {
return result
}
if strings.Contains(err.Error(), "first byte") || strings.Contains(err.Error(), "expired") {
return result
}
if !strings.Contains(err.Error(), "429") {
retryCount++
if retryCount >= maxRetries {
// If we've reached the maximum retries for errors other than 429, return the last result.
return result
}
}
// Calculate delay with exponential backoff
// delay := time.Duration(math.Min(float64(initialDelay)*math.Pow(2, float64(retryCount)), float64(maxDelay)))
delay := 500 * time.Millisecond
time.Sleep(delay)
}
}