zurg/pkg/http/client.go

package http

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/debridmediamanager/zurg/internal/config"
	"github.com/debridmediamanager/zurg/pkg/hosts"
	"github.com/debridmediamanager/zurg/pkg/logutil"
	http_dialer "github.com/mwitkow/go-http-dialer"
	"golang.org/x/net/proxy"

	cmap "github.com/orcaman/concurrent-map/v2"
)

const (
	RATE_LIMIT_FACTOR = 4 // should always be > 1
)

type HTTPClient struct {
	client         *http.Client
	maxRetries     int
	backoff        func(attempt int) time.Duration
	getRetryIncr   func(resp *http.Response, hasRangeHeader bool, err error) int
	bearerToken    string
	ensureIPv6Host bool
	cfg            config.ConfigInterface
	ipv6           cmap.ConcurrentMap[string, string]
	ipv6Hosts      []string
	log            *logutil.Logger
}

// {
// "error": "infringing_file",
// "error_code": 35
// }

type ErrorResponse struct {
	Message string `json:"error"`
	Code    int    `json:"error_code"`
}

func (e *ErrorResponse) Error() string {
	return fmt.Sprintf("api response error: %s (code: %d)", e.Message, e.Code)
}

func NewHTTPClient(token string, maxRetries int, timeoutSecs int, ensureIPv6Host bool, cfg config.ConfigInterface, log *logutil.Logger) *HTTPClient {
	client := HTTPClient{
		bearerToken: token,
		client: &http.Client{
			Timeout: time.Duration(timeoutSecs) * time.Second,
		},
		maxRetries: maxRetries * RATE_LIMIT_FACTOR,
		backoff: func(attempt int) time.Duration {
			maxDuration := 60
			backoff := int(math.Pow(2, float64(attempt)))
			if backoff > maxDuration {
				backoff = maxDuration
			}
			return time.Duration(backoff) * time.Second
		},
		getRetryIncr: func(resp *http.Response, hasRangeHeader bool, err error) int {
			if resp != nil {
				if resp.StatusCode == 429 {
					return 1
				}
				if resp.StatusCode == http.StatusOK && hasRangeHeader {
					return 1
				}
				return 0 // don't retry
			} else if err != nil {
				log.Errorf("Client request error: %s", err.Error())
				if strings.Contains(err.Error(), "api response error") {
					if apiErr, ok := err.(*ErrorResponse); ok {
						switch apiErr.Code {
						case -1: // Internal error
							return 1
						case 5: // Slow down
							return 1
						case 6: // Ressource unreachable
							return 1
						case 17: // Hoster in maintenance
							return 1
						case 19: // Hoster temporarily unavailable
							return 1
						case 25: // Service unavailable
							return 1
						case 34: // Too many requests
							return 1
						case 36: // Fair Usage Limit
							return 1
						default:
							return 0 // don't retry
						}
					}
				}
				return 1
			}
			return RATE_LIMIT_FACTOR
		},
		ensureIPv6Host: ensureIPv6Host,
		cfg:            cfg,
		ipv6:           cmap.New[string](),
		log:            log,
	}

	var dialer proxy.Dialer = &net.Dialer{}
	if proxyURLString := cfg.GetProxy(); proxyURLString != "" {
		proxyURL, err := url.Parse(proxyURLString)
		if err != nil {
			log.Errorf("Failed to parse proxy URL: %v", err)
			return nil
		}
		dialer, err = proxyDialer(proxyURL)
		if err != nil {
			log.Errorf("Failed to create proxy dialer: %v", err)
			return nil
		}
	}

	if cfg.ShouldForceIPv6() {
		ipv6List, err := hosts.FetchHosts(hosts.IPV6)
		if err != nil {
			log.Warnf("Failed to fetch IPv6 hosts: %v", err)
			// Decide if you want to return nil here or continue without IPv6
		} else {
			client.ipv6Hosts = ipv6List
			log.Debugf("Fetched %d IPv6 hosts", len(ipv6List))
		}

		client.client.Transport = &http.Transport{
			DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
				if ipv6Address, ok := client.ipv6.Get(address); ok {
					return dialer.Dial(network, ipv6Address)
				}
				host, port, err := net.SplitHostPort(address)
				if err != nil {
					return nil, err
				}
				ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
				if err != nil {
					return nil, err
				}
				for _, ip := range ips {
					if ip.IP.To4() == nil { // IPv6 address found
						ipv6Address := net.JoinHostPort(ip.IP.String(), port)
						client.ipv6.Set(address, ipv6Address)
						return dialer.Dial(network, ipv6Address)
					}
				}
				return dialer.Dial(network, address)
			},
		}
	} else {
		client.client.Transport = &http.Transport{
			DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
				return dialer.Dial(network, address)
			},
		}
	}

	return &client
}

func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
	if r.bearerToken != "" {
		req.Header.Set("Authorization", "Bearer "+r.bearerToken)
	}
	r.replaceHostIfNeeded(req)
	// check if Range header is set
	hasRangeHeader := req.Header.Get("Range") != "" && req.Header.Get("Range") != "bytes=0-"

	var resp *http.Response
	var err error
	attempt := 0
	for {
		resp, err = r.client.Do(req)
		if resp != nil && (resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusPartialContent) {
			body, _ := io.ReadAll(resp.Body)
			if body != nil {
				var errResp ErrorResponse
				jsonErr := json.Unmarshal(body, &errResp)
				if jsonErr == nil {
					errResp.Message += fmt.Sprintf(" (status code: %d)", resp.StatusCode)
					err = &errResp
				}
			}
		}
		if incr := r.getRetryIncr(resp, hasRangeHeader, err); incr > 0 {
			attempt += incr
			if attempt > r.maxRetries {
				break
			}
			if incr >= RATE_LIMIT_FACTOR {
				time.Sleep(r.backoff(attempt))
			} else {
				time.Sleep(time.Duration(r.cfg.GetRateLimitSleepSeconds()) * time.Second) // extra delay
			}
			if resp != nil {
				resp.Body.Close()
			}
		} else {
			// if incr == 0, don't retry anymore
			break
		}
	}
	return resp, err
}

func (r *HTTPClient) replaceHostIfNeeded(req *http.Request) {
	if !r.ensureIPv6Host || !r.cfg.ShouldForceIPv6() {
		return
	}
	// if no hosts are found, just replace .com with .cloud
	if len(r.ipv6Hosts) == 0 {
		host := req.URL.Host
		if strings.HasSuffix(host, ".com") {
			newHost := strings.Replace(host, ".com", ".cloud", 1)
			req.Host = newHost
			req.URL.Host = newHost
		}
	}
	// if hosts are found, ensure the host is an IPv6 host
	found := false
	for _, h := range r.ipv6Hosts {
		if h == req.URL.Host {
			found = true
			break
		}
	}
	// if host is not an IPv6 host, replace it with a random IPv6 host
	if !found {
		r.log.Warnf("Host %s is not an IPv6 host, replacing with a random IPv6 host (ensure you have preferred_hosts properly set in your config.yml, if unset, run `zurg network-test -t ipv6`)", req.URL.Host)
		newHost := r.ipv6Hosts[rand.Intn(len(r.ipv6Hosts))]
		req.Host = newHost
		req.URL.Host = newHost
	}
}

func proxyDialer(proxyURL *url.URL) (proxy.Dialer, error) {
	if proxyURL.Scheme == "http" || proxyURL.Scheme == "https" {
		// Create a new HTTP proxy dialer
		httpProxyDialer := http_dialer.New(proxyURL)
		return httpProxyDialer, nil
	} else if proxyURL.Scheme == "socks5" {
		// For SOCKS5 proxies, use the proxy package's FromURL
		return proxy.FromURL(proxyURL, proxy.Direct)
	}
	return nil, fmt.Errorf("unsupported proxy scheme: %s", proxyURL.Scheme)
}