Optimizations

cmd/zurg/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/internal/universal"
 	"github.com/debridmediamanager.com/zurg/internal/version"
+	"github.com/dgraph-io/ristretto"
 	"github.com/panjf2000/ants/v2"
 
 	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
@@ -52,17 +53,27 @@ func main() {
 	}
 	defer p.Release()
 
-	torrentMgr := torrent.NewTorrentManager(config, rd, p, log.Named("manager"))
+	cache, err := ristretto.NewCache(&ristretto.Config{
+		NumCounters: 1e5,       // 200,000 to track frequency for 100,000 items.
+		MaxCost:     100 << 20, // maximum cost of cache (100MB).
+		BufferItems: 1 << 10,   // number of keys per Get buffer, can be adjusted.
+	})
+	if err != nil {
+		zurglog.Errorf("Failed to create cache: %v", err)
+		os.Exit(1)
+	}
+
+	torrentMgr := torrent.NewTorrentManager(config, rd, p, cache, log.Named("manager"))
 
 	downloadClient := zurghttp.NewHTTPClient(config.GetToken(), config.GetRetriesUntilFailed(), 0, config, log.Named("dlclient"))
 	getfile := universal.NewGetFile(downloadClient)
 
 	go func() {
-		http.ListenAndServe("localhost:6060", nil)
+		http.ListenAndServe("[::]:6060", nil)
 	}()
 
 	mux := http.NewServeMux()
-	net.Router(mux, getfile, config, torrentMgr, log.Named("net"))
+	net.Router(mux, getfile, config, torrentMgr, cache, log.Named("net"))
 
 	addr := fmt.Sprintf("%s:%s", config.GetHost(), config.GetPort())
 	server := &http.Server{Addr: addr, Handler: mux}

go.mod

@@ -11,4 +11,12 @@ require github.com/orcaman/concurrent-map/v2 v2.0.1
 
 require github.com/panjf2000/ants/v2 v2.8.2
 
-require go.uber.org/multierr v1.10.0 // indirect
+require (
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
+)

go.sum

@@ -1,15 +1,27 @@
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
 github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
 github.com/panjf2000/ants/v2 v2.8.2 h1:D1wfANttg8uXhC9149gRt1PDQ+dLVFjNXkCEycMcvQQ=
 github.com/panjf2000/ants/v2 v2.8.2/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
@@ -22,8 +34,11 @@ go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
 go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/dav/listing.go

@@ -9,21 +9,24 @@ import (
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/pkg/dav"
+	"github.com/dgraph-io/ristretto"
 	"go.uber.org/zap"
 )
 
-func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, log *zap.SugaredLogger) {
+func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, cache *ristretto.Cache, log *zap.SugaredLogger) {
 	requestPath := path.Clean(r.URL.Path)
-	filteredSegments := splitIntoSegments(requestPath)
 
+	var output *string
 	var err error
+
+	filteredSegments := splitIntoSegments(requestPath)
 	switch {
 	case len(filteredSegments) == 0:
-		err = handleListDirectories(w, t)
+		output, err = handleListDirectories(w, t)
 	case len(filteredSegments) == 1:
-		err = handleListTorrents(w, requestPath, t)
+		output, err = handleListTorrents(w, requestPath, t)
 	case len(filteredSegments) == 2:
-		err = handleListFiles(w, requestPath, t)
+		output, err = handleListFiles(w, requestPath, t)
 	default:
 		log.Warnf("Request %s %s not found", r.Method, requestPath)
 		http.Error(w, "Not Found", http.StatusNotFound)
@@ -40,13 +43,17 @@ func HandlePropfindRequest(w http.ResponseWriter, r *http.Request, t *torrent.To
 		return
 	}
 
+	if output != nil {
 		w.Header().Set("Content-Type", "text/xml; charset=\"utf-8\"")
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprint(w, *output)
+	}
 }
 
-func handleListDirectories(w http.ResponseWriter, t *torrent.TorrentManager) error {
+func handleListDirectories(w http.ResponseWriter, t *torrent.TorrentManager) (*string, error) {
-	fmt.Fprint(w, "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
+	davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">"
 	// initial response is the directory itself
-	fmt.Fprint(w, dav.BaseDirectory("", ""))
+	davDoc += dav.BaseDirectory("", "")
 
 	directories := t.DirectoryMap.Keys()
 	sort.Strings(directories)
@@ -54,61 +61,40 @@ func handleListDirectories(w http.ResponseWriter, t *torrent.TorrentManager) err
 		if strings.HasPrefix(directory, "int__") {
 			continue
 		}
-		fmt.Fprint(w, dav.Directory(directory, ""))
+		davDoc += dav.Directory(directory, "")
 	}
 
-	fmt.Fprint(w, "</d:multistatus>")
+	davDoc += "</d:multistatus>"
-	return nil
+	return &davDoc, nil
 }
 
-func handleListTorrents(w http.ResponseWriter, requestPath string, t *torrent.TorrentManager) error {
+func handleListTorrents(w http.ResponseWriter, requestPath string, t *torrent.TorrentManager) (*string, error) {
 	basePath := path.Base(requestPath)
-	torrents, ok := t.DirectoryMap.Get(basePath)
+	_, ok := t.DirectoryMap.Get(basePath)
 	if !ok {
-		return fmt.Errorf("cannot find directory %s", basePath)
+		return nil, fmt.Errorf("cannot find directory %s", basePath)
 	}
 
-	fmt.Fprint(w, "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
+	resp, _ := t.ResponseCache.Get(basePath + ".dav")
-	// initial response is the directory itself
+	davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">" + dav.BaseDirectory(basePath, "") + dav.BaseDirectory(basePath, "") + resp.(string) + "</d:multistatus>"
-	fmt.Fprint(w, dav.BaseDirectory(basePath, ""))
 
-	var allTorrents []torrent.Torrent
+	return &davDoc, nil
-	torrents.IterCb(func(key string, tor *torrent.Torrent) {
-		if tor.AllInProgress() {
-			return
-		}
-		copy := *tor
-		copy.AccessKey = key
-		allTorrents = append(allTorrents, copy)
-	})
-	sort.Slice(allTorrents, func(i, j int) bool {
-		return allTorrents[i].AccessKey < allTorrents[j].AccessKey
-	})
-
-	for _, tor := range allTorrents {
-		fmt.Fprint(w, dav.Directory(tor.AccessKey, tor.LatestAdded))
 }
 
-	fmt.Fprint(w, "</d:multistatus>")
+func handleListFiles(w http.ResponseWriter, requestPath string, t *torrent.TorrentManager) (*string, error) {
-	return nil
-}
-
-func handleListFiles(w http.ResponseWriter, requestPath string, t *torrent.TorrentManager) error {
 	requestPath = strings.Trim(requestPath, "/")
 	basePath := path.Base(path.Dir(requestPath))
 	torrents, ok := t.DirectoryMap.Get(basePath)
 	if !ok {
-		return fmt.Errorf("cannot find directory %s", basePath)
+		return nil, fmt.Errorf("cannot find directory %s", basePath)
 	}
 	accessKey := path.Base(requestPath)
 	tor, ok := torrents.Get(accessKey)
 	if !ok {
-		return fmt.Errorf("cannot find torrent %s", accessKey)
+		return nil, fmt.Errorf("cannot find torrent %s", accessKey)
 	}
 
-	fmt.Fprint(w, "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">")
+	davDoc := "<?xml version=\"1.0\" encoding=\"utf-8\"?><d:multistatus xmlns:d=\"DAV:\">" + dav.BaseDirectory(requestPath, tor.LatestAdded)
-	// initial response is the directory itself
-	fmt.Fprint(w, dav.BaseDirectory(requestPath, tor.LatestAdded))
 
 	filenames := tor.SelectedFiles.Keys()
 	sort.Strings(filenames)
@@ -117,9 +103,9 @@ func handleListFiles(w http.ResponseWriter, requestPath string, t *torrent.Torre
 		if file == nil || !strings.HasPrefix(file.Link, "http") {
 			continue
 		}
-		fmt.Fprint(w, dav.File(filename, file.Bytes, file.Ended))
+		davDoc += dav.File(filename, file.Bytes, file.Ended)
 	}
 
-	fmt.Fprint(w, "</d:multistatus>")
+	davDoc += "</d:multistatus>"
-	return nil
+	return &davDoc, nil
 }

internal/http/listing.go

@@ -10,10 +10,11 @@ import (
 	"strings"
 
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
+	"github.com/dgraph-io/ristretto"
 	"go.uber.org/zap"
 )
 
-func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, log *zap.SugaredLogger) {
+func HandleDirectoryListing(w http.ResponseWriter, r *http.Request, t *torrent.TorrentManager, cache *ristretto.Cache, log *zap.SugaredLogger) {
 	requestPath := path.Clean(r.URL.Path)
 
 	var output *string
@@ -66,29 +67,14 @@ func handleRoot(t *torrent.TorrentManager) (*string, error) {
 
 func handleListOfTorrents(requestPath string, t *torrent.TorrentManager) (*string, error) {
 	basePath := path.Base(requestPath)
-	torrents, ok := t.DirectoryMap.Get(basePath)
+	_, ok := t.DirectoryMap.Get(basePath)
 	if !ok {
 		return nil, fmt.Errorf("cannot find directory %s", basePath)
 	}
 
-	htmlDoc := "<ol>"
+	resp, _ := t.ResponseCache.Get(basePath + ".html")
+	htmlDoc := resp.(string)
 
-	var allTorrents []torrent.Torrent
-	torrents.IterCb(func(key string, tor *torrent.Torrent) {
-		if tor.AllInProgress() {
-			return
-		}
-		copy := *tor
-		copy.AccessKey = key
-		allTorrents = append(allTorrents, copy)
-	})
-	sort.Slice(allTorrents, func(i, j int) bool {
-		return allTorrents[i].AccessKey < allTorrents[j].AccessKey
-	})
-
-	for _, tor := range allTorrents {
-		htmlDoc = htmlDoc + fmt.Sprintf("<li><a href=\"%s/\">%s</a></li>", filepath.Join(requestPath, url.PathEscape(tor.AccessKey)), tor.AccessKey)
-	}
 	return &htmlDoc, nil
 }
 

internal/net/router.go

@@ -10,19 +10,20 @@ import (
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
 	"github.com/debridmediamanager.com/zurg/internal/torrent"
 	"github.com/debridmediamanager.com/zurg/internal/universal"
+	"github.com/dgraph-io/ristretto"
 	"go.uber.org/zap"
 )
 
 // Router creates a WebDAV router
-func Router(mux *http.ServeMux, getfile *universal.GetFile, c config.ConfigInterface, t *torrent.TorrentManager, log *zap.SugaredLogger) {
+func Router(mux *http.ServeMux, getfile *universal.GetFile, c config.ConfigInterface, t *torrent.TorrentManager, cache *ristretto.Cache, log *zap.SugaredLogger) {
 	mux.HandleFunc("/http/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
 		case http.MethodGet:
 			requestPath := path.Clean(r.URL.Path)
 			if countNonEmptySegments(strings.Split(requestPath, "/")) > 3 {
-				getfile.HandleGetRequest(w, r, t, c, log)
+				getfile.HandleGetRequest(w, r, t, c, cache, log)
 			} else {
-				intHttp.HandleDirectoryListing(w, r, t, log)
+				intHttp.HandleDirectoryListing(w, r, t, cache, log)
 			}
 
 		case http.MethodHead:
@@ -37,13 +38,13 @@ func Router(mux *http.ServeMux, getfile *universal.GetFile, c config.ConfigInter
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
 		case "PROPFIND":
-			dav.HandlePropfindRequest(w, r, t, log)
+			dav.HandlePropfindRequest(w, r, t, cache, log)
 
 		case http.MethodDelete:
 			dav.HandleDeleteRequest(w, r, t, log)
 
 		case http.MethodGet:
-			getfile.HandleGetRequest(w, r, t, c, log)
+			getfile.HandleGetRequest(w, r, t, c, cache, log)
 
 		case http.MethodOptions:
 			w.WriteHeader(http.StatusOK)

internal/torrent/manager.go

@@ -7,13 +7,16 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/debridmediamanager.com/zurg/internal/config"
+	"github.com/debridmediamanager.com/zurg/pkg/dav"
 	"github.com/debridmediamanager.com/zurg/pkg/realdebrid"
 	"github.com/debridmediamanager.com/zurg/pkg/utils"
+	"github.com/dgraph-io/ristretto"
 	cmap "github.com/orcaman/concurrent-map/v2"
 	"github.com/panjf2000/ants/v2"
 	"go.uber.org/zap"
@@ -26,13 +29,14 @@ const (
 )
 
 type TorrentManager struct {
+	Config          config.ConfigInterface
+	Api             *realdebrid.RealDebrid
 	DirectoryMap    cmap.ConcurrentMap[string, cmap.ConcurrentMap[string, *Torrent]] // directory -> accessKey -> Torrent
 	DownloadCache   cmap.ConcurrentMap[string, *realdebrid.Download]
+	ResponseCache   *ristretto.Cache
 	checksum        string
 	latestAdded     string
 	requiredVersion string
-	cfg             config.ConfigInterface
-	Api             *realdebrid.RealDebrid
 	antsPool        *ants.Pool
 	unrestrictPool  *ants.Pool
 	log             *zap.SugaredLogger
@@ -42,18 +46,19 @@ type TorrentManager struct {
 // NewTorrentManager creates a new torrent manager
 // it will fetch all torrents and their info in the background
 // and store them in-memory and cached in files
-func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, log *zap.SugaredLogger) *TorrentManager {
+func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p *ants.Pool, cache *ristretto.Cache, log *zap.SugaredLogger) *TorrentManager {
 	t := &TorrentManager{
-		cfg:             cfg,
+		Config:          cfg,
-		DirectoryMap:    cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
-		requiredVersion: "18.11.2023",
 		Api:             api,
+		DirectoryMap:    cmap.New[cmap.ConcurrentMap[string, *Torrent]](),
+		ResponseCache:   cache,
+		requiredVersion: "18.11.2023",
 		antsPool:        p,
 		log:             log,
 		mu:              &sync.Mutex{},
 	}
 
-	unrestrictPool, err := ants.NewPool(t.cfg.GetUnrestrictWorkers())
+	unrestrictPool, err := ants.NewPool(t.Config.GetUnrestrictWorkers())
 	if err != nil {
 		t.unrestrictPool = t.antsPool
 	} else {
@@ -145,12 +150,12 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p
 		// get filenames
 		filenames := torrent.SelectedFiles.Keys()
 		// Map torrents to directories
-		switch t.cfg.GetVersion() {
+		switch t.Config.GetVersion() {
 		case "v1":
-			configV1 := t.cfg.(*config.ZurgConfigV1)
+			configV1 := t.Config.(*config.ZurgConfigV1)
 			for _, directories := range configV1.GetGroupMap() {
 				for _, directory := range directories {
-					if t.cfg.MeetsConditions(directory, torrent.AccessKey, torrentIDs, filenames) {
+					if t.Config.MeetsConditions(directory, torrent.AccessKey, torrentIDs, filenames) {
 						torrents, _ := t.DirectoryMap.Get(directory)
 						torrents.Set(accessKey, torrent)
 						break
@@ -159,12 +164,13 @@ func NewTorrentManager(cfg config.ConfigInterface, api *realdebrid.RealDebrid, p
 			}
 		}
 	})
+	t.updateSortedKeys()
 
 	t.log.Infof("Compiled into %d torrents, %d were missing info", allTorrents.Count(), noInfoCount)
 
 	t.SetChecksum(t.getChecksum())
 
-	if t.cfg.EnableRepair() {
+	if t.Config.EnableRepair() {
 		t.log.Info("Checking for torrents to repair")
 		t.repairAll()
 		t.log.Info("Finished checking for torrents to repair")
@@ -209,8 +215,8 @@ func (t *TorrentManager) mergeToMain(mainTorrent, torrentToMerge *Torrent) *Torr
 func (t *TorrentManager) UnrestrictUntilOk(link string) *realdebrid.Download {
 	retChan := make(chan *realdebrid.Download, 1)
 	t.unrestrictPool.Submit(func() {
-		retChan <- t.Api.UnrestrictUntilOk(link, t.cfg.ShouldServeFromRclone())
+		retChan <- t.Api.UnrestrictUntilOk(link, t.Config.ShouldServeFromRclone())
-		time.Sleep(time.Duration(t.cfg.GetReleaseUnrestrictAfterMs()) * time.Millisecond)
+		time.Sleep(time.Duration(t.Config.GetReleaseUnrestrictAfterMs()) * time.Millisecond)
 	})
 	defer close(retChan)
 	return <-retChan
@@ -282,7 +288,7 @@ func (t *TorrentManager) getChecksum() string {
 func (t *TorrentManager) startRefreshJob() {
 	t.log.Info("Starting periodic refresh")
 	for {
-		<-time.After(time.Duration(t.cfg.GetRefreshEverySeconds()) * time.Second)
+		<-time.After(time.Duration(t.Config.GetRefreshEverySeconds()) * time.Second)
 
 		checksum := t.getChecksum()
 		if checksum == t.checksum {
@@ -358,12 +364,12 @@ func (t *TorrentManager) startRefreshJob() {
 			// get filenames
 			filenames := torrent.SelectedFiles.Keys()
 			// Map torrents to directories
-			switch t.cfg.GetVersion() {
+			switch t.Config.GetVersion() {
 			case "v1":
-				configV1 := t.cfg.(*config.ZurgConfigV1)
+				configV1 := t.Config.(*config.ZurgConfigV1)
 				for _, directories := range configV1.GetGroupMap() {
 					for _, directory := range directories {
-						if t.cfg.MeetsConditions(directory, torrent.AccessKey, torrentIDs, filenames) {
+						if t.Config.MeetsConditions(directory, torrent.AccessKey, torrentIDs, filenames) {
 							torrents, _ := t.DirectoryMap.Get(directory)
 							torrents.Set(torrent.AccessKey, torrent)
 							if torrent.LatestAdded > t.latestAdded {
@@ -386,19 +392,20 @@ func (t *TorrentManager) startRefreshJob() {
 				t.log.Infof("Deleted torrent: %s\n", oldAccessKey)
 			}
 		}
+		t.updateSortedKeys()
 
 		t.log.Infof("Compiled into %d torrents, %d were missing info", oldTorrents.Count(), noInfoCount)
 
 		t.SetChecksum(t.getChecksum())
 
-		if t.cfg.EnableRepair() {
+		if t.Config.EnableRepair() {
 			t.log.Info("Checking for torrents to repair")
 			t.repairAll()
 			t.log.Info("Finished checking for torrents to repair")
 		} else {
 			t.log.Info("Repair is disabled, skipping repair check")
 		}
-		go OnLibraryUpdateHook(updatedPaths, t.cfg, t.log)
+		go OnLibraryUpdateHook(updatedPaths, t.Config, t.log)
 
 		t.latestAdded = newTorrents[0].Added
 		t.log.Info("Finished refreshing torrents")
@@ -477,11 +484,11 @@ func (t *TorrentManager) getMoreInfo(rdTorrent realdebrid.Torrent) *Torrent {
 }
 
 func (t *TorrentManager) getName(name, originalName string) string {
-	if t.cfg.EnableRetainRDTorrentName() {
+	if t.Config.EnableRetainRDTorrentName() {
 		return name
 	}
 	// drop the extension from the name
-	if t.cfg.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
+	if t.Config.EnableRetainFolderNameExtension() && strings.Contains(name, originalName) {
 		return name
 	} else {
 		ret := strings.TrimSuffix(originalName, ".mp4")
@@ -551,9 +558,9 @@ func (t *TorrentManager) organizeChaos(links []string, selectedFiles []*File) ([
 				resultsChan <- Result{Response: download}
 				return
 			}
-			resp := t.Api.UnrestrictUntilOk(link, t.cfg.ShouldServeFromRclone())
+			resp := t.Api.UnrestrictUntilOk(link, t.Config.ShouldServeFromRclone())
 			resultsChan <- Result{Response: resp}
-			time.Sleep(time.Duration(t.cfg.GetReleaseUnrestrictAfterMs()) * time.Millisecond)
+			time.Sleep(time.Duration(t.Config.GetReleaseUnrestrictAfterMs()) * time.Millisecond)
 		})
 	}
 
@@ -649,10 +656,11 @@ func (t *TorrentManager) Delete(accessKey string) {
 			torrents.Remove(accessKey)
 		}
 	})
+	t.updateSortedKeys()
 }
 
 func (t *TorrentManager) Repair(accessKey string) {
-	if !t.cfg.EnableRepair() {
+	if !t.Config.EnableRepair() {
 		t.log.Warn("Repair is disabled; if you do not have other zurg instances running, you should enable repair")
 		return
 	}
@@ -700,6 +708,7 @@ func (t *TorrentManager) Repair(accessKey string) {
 		t.DirectoryMap.IterCb(func(_ string, torrents cmap.ConcurrentMap[string, *Torrent]) {
 			torrents.Remove(torrent.AccessKey)
 		})
+		t.updateSortedKeys()
 		// t.log.Debugf("You can try fixing it yourself magnet:?xt=urn:btih:%s", info.Hash)
 		return
 	} else if streamableCount == 1 {
@@ -708,6 +717,7 @@ func (t *TorrentManager) Repair(accessKey string) {
 		t.DirectoryMap.IterCb(func(_ string, torrents cmap.ConcurrentMap[string, *Torrent]) {
 			torrents.Remove(torrent.AccessKey)
 		})
+		t.updateSortedKeys()
 		return
 	}
 	// t.log.Debugf("Identified the expired files of torrent id=%s", info.ID)
@@ -876,3 +886,23 @@ func (t *TorrentManager) canCapacityHandle() bool {
 		retryCount++
 	}
 }
+
+func (t *TorrentManager) updateSortedKeys() {
+	t.DirectoryMap.IterCb(func(directory string, torrents cmap.ConcurrentMap[string, *Torrent]) {
+		allKeys := torrents.Keys()
+		sort.Strings(allKeys)
+		davRet := ""
+		htmlRet := ""
+		for _, accessKey := range allKeys {
+			if tor, ok := torrents.Get(accessKey); ok {
+				if tor.AnyInProgress() {
+					continue
+				}
+				davRet += dav.Directory(tor.AccessKey, tor.LatestAdded)
+				htmlRet += fmt.Sprintf("<a href=\"/%s/%s\">%s</a><br>", directory, tor.AccessKey, tor.AccessKey)
+			}
+		}
+		t.ResponseCache.Set(directory+".dav", davRet, 1)
+		t.ResponseCache.Set(directory+".html", "<ol>"+htmlRet, 1)
+	})
+}

internal/universal/get.go

@@ -14,6 +14,7 @@ import (
 	intHttp "github.com/debridmediamanager.com/zurg/internal/http"
 	intTor "github.com/debridmediamanager.com/zurg/internal/torrent"
 	zurghttp "github.com/debridmediamanager.com/zurg/pkg/http"
+	"github.com/dgraph-io/ristretto"
 	"go.uber.org/zap"
 )
 
@@ -26,7 +27,7 @@ func NewGetFile(client *zurghttp.HTTPClient) *GetFile {
 }
 
 // HandleGetRequest handles a GET request universally for both WebDAV and HTTP
-func (gf *GetFile) HandleGetRequest(w http.ResponseWriter, r *http.Request, t *intTor.TorrentManager, c config.ConfigInterface, log *zap.SugaredLogger) {
+func (gf *GetFile) HandleGetRequest(w http.ResponseWriter, r *http.Request, t *intTor.TorrentManager, c config.ConfigInterface, cache *ristretto.Cache, log *zap.SugaredLogger) {
 	requestPath := path.Clean(r.URL.Path)
 	isDav := true
 	if strings.Contains(requestPath, "/http") {
@@ -40,9 +41,9 @@ func (gf *GetFile) HandleGetRequest(w http.ResponseWriter, r *http.Request, t *i
 	// If there are less than 3 segments, return an error or adjust as needed
 	if len(segments) <= 3 {
 		if isDav {
-			dav.HandlePropfindRequest(w, r, t, log)
+			dav.HandlePropfindRequest(w, r, t, cache, log)
 		} else {
-			intHttp.HandleDirectoryListing(w, r, t, log)
+			intHttp.HandleDirectoryListing(w, r, t, cache, log)
 		}
 		return
 	}