Add rate limiter

pkg/http/client.go

@@ -30,6 +30,7 @@ type HTTPClient struct {
 	backoff            func(int, int) time.Duration
 	dnsCache           cmap.ConcurrentMap[string, string]
 	hosts              []string
+	rateLimiter        *RateLimiter
 	log                *logutil.Logger
 }
 
@@ -58,6 +59,7 @@ func NewHTTPClient(
 	forceIPv6 bool,
 	hosts []string,
 	proxyURL string,
+	rateLimiter *RateLimiter,
 	log *logutil.Logger,
 ) *HTTPClient {
 	client := HTTPClient{
@@ -69,6 +71,7 @@ func NewHTTPClient(
 		backoff:            backoffFunc,
 		dnsCache:           cmap.New[string](),
 		hosts:              hosts,
+		rateLimiter:        rateLimiter,
 		log:                log,
 	}
 
@@ -154,6 +157,8 @@ func (r *HTTPClient) Do(req *http.Request) (*http.Response, error) {
 			r.ensureReachableHost(req)
 		}
 
+		r.rateLimiter.Wait()
+
 		resp, err = r.client.Do(req)
 
 		// http 4xx and 5xx errors